blob: 87937fb6a85d1bc5769b86519b282e9326b1a0fa [file] [log] [blame]
Josh Gaoc51726c2018-10-11 16:33:05 -07001/*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define TRACE_TAG USB
18
19#include "sysdeps.h"
20
21#include <errno.h>
Josh Gaoa9b62d52020-02-19 13:50:57 -080022#include <inttypes.h>
Josh Gaoc51726c2018-10-11 16:33:05 -070023#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26#include <sys/ioctl.h>
27#include <sys/types.h>
28#include <unistd.h>
29
30#include <linux/usb/functionfs.h>
31#include <sys/eventfd.h>
32
Josh Gao86b33be2019-02-26 17:53:52 -080033#include <algorithm>
Josh Gaoc51726c2018-10-11 16:33:05 -070034#include <array>
35#include <future>
36#include <memory>
37#include <mutex>
38#include <optional>
39#include <vector>
40
41#include <asyncio/AsyncIO.h>
42
43#include <android-base/logging.h>
44#include <android-base/macros.h>
45#include <android-base/properties.h>
46#include <android-base/thread_annotations.h>
47
48#include <adbd/usb.h>
49
50#include "adb_unique_fd.h"
51#include "adb_utils.h"
52#include "sysdeps/chrono.h"
53#include "transport.h"
54#include "types.h"
55
56using android::base::StringPrintf;
57
Josh Gaoc0b831b2019-02-13 15:27:28 -080058// We can't find out whether we have support for AIO on ffs endpoints until we submit a read.
59static std::optional<bool> gFfsAioSupported;
60
Josh Gao08ccc732019-04-16 11:20:04 -070061// Not all USB controllers support operations larger than 16k, so don't go above that.
Josh Gaod0feaf92019-04-24 14:28:25 -070062// Also, each submitted operation does an allocation in the kernel of that size, so we want to
63// minimize our queue depth while still maintaining a deep enough queue to keep the USB stack fed.
64static constexpr size_t kUsbReadQueueDepth = 8;
Josh Gao08ccc732019-04-16 11:20:04 -070065static constexpr size_t kUsbReadSize = 4 * PAGE_SIZE;
Josh Gaoc51726c2018-10-11 16:33:05 -070066
Josh Gaod0feaf92019-04-24 14:28:25 -070067static constexpr size_t kUsbWriteQueueDepth = 8;
Josh Gao08ccc732019-04-16 11:20:04 -070068static constexpr size_t kUsbWriteSize = 4 * PAGE_SIZE;
Josh Gaoc51726c2018-10-11 16:33:05 -070069
Dan Albert782036b2019-06-24 14:35:35 -070070static const char* to_string(enum usb_functionfs_event_type type) {
71 switch (type) {
72 case FUNCTIONFS_BIND:
73 return "FUNCTIONFS_BIND";
74 case FUNCTIONFS_UNBIND:
75 return "FUNCTIONFS_UNBIND";
76 case FUNCTIONFS_ENABLE:
77 return "FUNCTIONFS_ENABLE";
78 case FUNCTIONFS_DISABLE:
79 return "FUNCTIONFS_DISABLE";
80 case FUNCTIONFS_SETUP:
81 return "FUNCTIONFS_SETUP";
82 case FUNCTIONFS_SUSPEND:
83 return "FUNCTIONFS_SUSPEND";
84 case FUNCTIONFS_RESUME:
85 return "FUNCTIONFS_RESUME";
86 }
87}
88
Josh Gaoc51726c2018-10-11 16:33:05 -070089enum class TransferDirection : uint64_t {
90 READ = 0,
91 WRITE = 1,
92};
93
94struct TransferId {
95 TransferDirection direction : 1;
96 uint64_t id : 63;
97
98 TransferId() : TransferId(TransferDirection::READ, 0) {}
99
100 private:
101 TransferId(TransferDirection direction, uint64_t id) : direction(direction), id(id) {}
102
103 public:
104 explicit operator uint64_t() const {
105 uint64_t result;
106 static_assert(sizeof(*this) == sizeof(result));
107 memcpy(&result, this, sizeof(*this));
108 return result;
109 }
110
111 static TransferId read(uint64_t id) { return TransferId(TransferDirection::READ, id); }
112 static TransferId write(uint64_t id) { return TransferId(TransferDirection::WRITE, id); }
113
114 static TransferId from_value(uint64_t value) {
115 TransferId result;
116 memcpy(&result, &value, sizeof(value));
117 return result;
118 }
119};
120
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700121template <class Payload>
Josh Gaoc51726c2018-10-11 16:33:05 -0700122struct IoBlock {
Josh Gaob0195742019-03-18 14:11:28 -0700123 bool pending = false;
Evgenii Stepanov9da358d2019-05-15 18:45:01 -0700124 struct iocb control = {};
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700125 Payload payload;
Josh Gaoc51726c2018-10-11 16:33:05 -0700126
127 TransferId id() const { return TransferId::from_value(control.aio_data); }
128};
129
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700130using IoReadBlock = IoBlock<Block>;
131using IoWriteBlock = IoBlock<std::shared_ptr<Block>>;
132
Josh Gaoc51726c2018-10-11 16:33:05 -0700133struct ScopedAioContext {
134 ScopedAioContext() = default;
135 ~ScopedAioContext() { reset(); }
136
137 ScopedAioContext(ScopedAioContext&& move) { reset(move.release()); }
138 ScopedAioContext(const ScopedAioContext& copy) = delete;
139
140 ScopedAioContext& operator=(ScopedAioContext&& move) {
141 reset(move.release());
142 return *this;
143 }
144 ScopedAioContext& operator=(const ScopedAioContext& copy) = delete;
145
146 static ScopedAioContext Create(size_t max_events) {
147 aio_context_t ctx = 0;
148 if (io_setup(max_events, &ctx) != 0) {
149 PLOG(FATAL) << "failed to create aio_context_t";
150 }
151 ScopedAioContext result;
152 result.reset(ctx);
153 return result;
154 }
155
156 aio_context_t release() {
157 aio_context_t result = context_;
158 context_ = 0;
159 return result;
160 }
161
162 void reset(aio_context_t new_context = 0) {
163 if (context_ != 0) {
164 io_destroy(context_);
165 }
166
167 context_ = new_context;
168 }
169
170 aio_context_t get() { return context_; }
171
172 private:
173 aio_context_t context_ = 0;
174};
175
176struct UsbFfsConnection : public Connection {
Dan Albert782036b2019-06-24 14:35:35 -0700177 UsbFfsConnection(unique_fd control, unique_fd read, unique_fd write,
Josh Gaoc51726c2018-10-11 16:33:05 -0700178 std::promise<void> destruction_notifier)
Josh Gao19dc2962019-03-26 18:47:45 -0700179 : worker_started_(false),
180 stopped_(false),
Josh Gaoc51726c2018-10-11 16:33:05 -0700181 destruction_notifier_(std::move(destruction_notifier)),
Dan Albert782036b2019-06-24 14:35:35 -0700182 control_fd_(std::move(control)),
Josh Gaoc51726c2018-10-11 16:33:05 -0700183 read_fd_(std::move(read)),
184 write_fd_(std::move(write)) {
185 LOG(INFO) << "UsbFfsConnection constructed";
Josh Gaoc0b831b2019-02-13 15:27:28 -0800186 worker_event_fd_.reset(eventfd(0, EFD_CLOEXEC));
187 if (worker_event_fd_ == -1) {
188 PLOG(FATAL) << "failed to create eventfd";
189 }
190
Dan Albert782036b2019-06-24 14:35:35 -0700191 monitor_event_fd_.reset(eventfd(0, EFD_CLOEXEC));
192 if (monitor_event_fd_ == -1) {
193 PLOG(FATAL) << "failed to create eventfd";
194 }
195
Josh Gaoc51726c2018-10-11 16:33:05 -0700196 aio_context_ = ScopedAioContext::Create(kUsbReadQueueDepth + kUsbWriteQueueDepth);
197 }
198
199 ~UsbFfsConnection() {
200 LOG(INFO) << "UsbFfsConnection being destroyed";
201 Stop();
202 monitor_thread_.join();
Josh Gaoc0b831b2019-02-13 15:27:28 -0800203
204 // We need to explicitly close our file descriptors before we notify our destruction,
205 // because the thread listening on the future will immediately try to reopen the endpoint.
Josh Gao19dc2962019-03-26 18:47:45 -0700206 aio_context_.reset();
Dan Albert782036b2019-06-24 14:35:35 -0700207 control_fd_.reset();
Josh Gaoc0b831b2019-02-13 15:27:28 -0800208 read_fd_.reset();
209 write_fd_.reset();
210
Josh Gaoc51726c2018-10-11 16:33:05 -0700211 destruction_notifier_.set_value();
212 }
213
214 virtual bool Write(std::unique_ptr<apacket> packet) override final {
215 LOG(DEBUG) << "USB write: " << dump_header(&packet->msg);
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700216 auto header = std::make_shared<Block>(sizeof(packet->msg));
217 memcpy(header->data(), &packet->msg, sizeof(packet->msg));
Josh Gaoc51726c2018-10-11 16:33:05 -0700218
219 std::lock_guard<std::mutex> lock(write_mutex_);
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700220 write_requests_.push_back(
221 CreateWriteBlock(std::move(header), 0, sizeof(packet->msg), next_write_id_++));
Josh Gaoc51726c2018-10-11 16:33:05 -0700222 if (!packet->payload.empty()) {
Josh Gao86b33be2019-02-26 17:53:52 -0800223 // The kernel attempts to allocate a contiguous block of memory for each write,
224 // which can fail if the write is large and the kernel heap is fragmented.
225 // Split large writes into smaller chunks to avoid this.
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700226 auto payload = std::make_shared<Block>(std::move(packet->payload));
Josh Gao86b33be2019-02-26 17:53:52 -0800227 size_t offset = 0;
228 size_t len = payload->size();
229
230 while (len > 0) {
231 size_t write_size = std::min(kUsbWriteSize, len);
232 write_requests_.push_back(
233 CreateWriteBlock(payload, offset, write_size, next_write_id_++));
234 len -= write_size;
235 offset += write_size;
236 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700237 }
238 SubmitWrites();
239 return true;
240 }
241
242 virtual void Start() override final { StartMonitor(); }
243
244 virtual void Stop() override final {
245 if (stopped_.exchange(true)) {
246 return;
247 }
248 stopped_ = true;
249 uint64_t notify = 1;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800250 ssize_t rc = adb_write(worker_event_fd_.get(), &notify, sizeof(notify));
Josh Gaoc51726c2018-10-11 16:33:05 -0700251 if (rc < 0) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800252 PLOG(FATAL) << "failed to notify worker eventfd to stop UsbFfsConnection";
Josh Gaoc51726c2018-10-11 16:33:05 -0700253 }
254 CHECK_EQ(static_cast<size_t>(rc), sizeof(notify));
Dan Albert782036b2019-06-24 14:35:35 -0700255
256 rc = adb_write(monitor_event_fd_.get(), &notify, sizeof(notify));
257 if (rc < 0) {
258 PLOG(FATAL) << "failed to notify monitor eventfd to stop UsbFfsConnection";
259 }
260
261 CHECK_EQ(static_cast<size_t>(rc), sizeof(notify));
Josh Gaoc51726c2018-10-11 16:33:05 -0700262 }
263
Joshua Duong5cf78682020-01-21 13:19:42 -0800264 virtual bool DoTlsHandshake(RSA* key, std::string* auth_key) override final {
265 // TODO: support TLS for usb connections.
266 LOG(FATAL) << "Not supported yet.";
267 return false;
268 }
269
Josh Gaoc51726c2018-10-11 16:33:05 -0700270 private:
271 void StartMonitor() {
272 // This is a bit of a mess.
273 // It's possible for io_submit to end up blocking, if we call it as the endpoint
274 // becomes disabled. Work around this by having a monitor thread to listen for functionfs
275 // lifecycle events. If we notice an error condition (either we've become disabled, or we
276 // were never enabled in the first place), we send interruption signals to the worker thread
277 // until it dies, and then report failure to the transport via HandleError, which will
278 // eventually result in the transport being destroyed, which will result in UsbFfsConnection
279 // being destroyed, which unblocks the open thread and restarts this entire process.
Josh Gaoc51726c2018-10-11 16:33:05 -0700280 static std::once_flag handler_once;
281 std::call_once(handler_once, []() { signal(kInterruptionSignal, [](int) {}); });
282
283 monitor_thread_ = std::thread([this]() {
284 adb_thread_setname("UsbFfs-monitor");
Josh Gaoe3d34e12020-02-26 15:44:30 -0800285 LOG(INFO) << "UsbFfs-monitor thread spawned";
Josh Gaoc51726c2018-10-11 16:33:05 -0700286
Dan Albert782036b2019-06-24 14:35:35 -0700287 bool bound = false;
Josh Gao6933d542019-03-26 13:21:42 -0700288 bool enabled = false;
Josh Gaoc51726c2018-10-11 16:33:05 -0700289 bool running = true;
290 while (running) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800291 adb_pollfd pfd[2] = {
Dan Albert782036b2019-06-24 14:35:35 -0700292 { .fd = control_fd_.get(), .events = POLLIN, .revents = 0 },
293 { .fd = monitor_event_fd_.get(), .events = POLLIN, .revents = 0 },
Josh Gaoc0b831b2019-02-13 15:27:28 -0800294 };
Josh Gao19dc2962019-03-26 18:47:45 -0700295
Dan Albert782036b2019-06-24 14:35:35 -0700296 // If we don't see our first bind within a second, try again.
297 int timeout_ms = bound ? -1 : 1000;
298
299 int rc = TEMP_FAILURE_RETRY(adb_poll(pfd, 2, timeout_ms));
Josh Gaoc0b831b2019-02-13 15:27:28 -0800300 if (rc == -1) {
301 PLOG(FATAL) << "poll on USB control fd failed";
Dan Albert782036b2019-06-24 14:35:35 -0700302 } else if (rc == 0) {
303 LOG(WARNING) << "timed out while waiting for FUNCTIONFS_BIND, trying again";
304 break;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800305 }
306
307 if (pfd[1].revents) {
Dan Albert782036b2019-06-24 14:35:35 -0700308 // We were told to die.
309 break;
Josh Gaoc51726c2018-10-11 16:33:05 -0700310 }
311
312 struct usb_functionfs_event event;
Dan Albert782036b2019-06-24 14:35:35 -0700313 rc = TEMP_FAILURE_RETRY(adb_read(control_fd_.get(), &event, sizeof(event)));
Josh Gao1f7ae9d2019-05-10 11:37:34 -0700314 if (rc == -1) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700315 PLOG(FATAL) << "failed to read functionfs event";
Josh Gao1f7ae9d2019-05-10 11:37:34 -0700316 } else if (rc == 0) {
317 LOG(WARNING) << "hit EOF on functionfs control fd";
318 break;
319 } else if (rc != sizeof(event)) {
320 LOG(FATAL) << "read functionfs event of unexpected size, expected "
321 << sizeof(event) << ", got " << rc;
Josh Gaoc51726c2018-10-11 16:33:05 -0700322 }
323
324 LOG(INFO) << "USB event: "
Dan Albert782036b2019-06-24 14:35:35 -0700325 << to_string(static_cast<usb_functionfs_event_type>(event.type));
Josh Gaoc51726c2018-10-11 16:33:05 -0700326
327 switch (event.type) {
328 case FUNCTIONFS_BIND:
Dan Albert782036b2019-06-24 14:35:35 -0700329 if (bound) {
330 LOG(WARNING) << "received FUNCTIONFS_BIND while already bound?";
331 running = false;
332 break;
333 }
334
335 if (enabled) {
336 LOG(WARNING) << "received FUNCTIONFS_BIND while already enabled?";
337 running = false;
338 break;
339 }
340
341 bound = true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700342 break;
343
344 case FUNCTIONFS_ENABLE:
Dan Albert782036b2019-06-24 14:35:35 -0700345 if (!bound) {
346 LOG(WARNING) << "received FUNCTIONFS_ENABLE while not bound?";
347 running = false;
348 break;
349 }
350
Josh Gao87afd522019-03-28 11:05:53 -0700351 if (enabled) {
352 LOG(WARNING) << "received FUNCTIONFS_ENABLE while already enabled?";
353 running = false;
Josh Gao94fb36b2019-05-01 16:53:53 -0700354 break;
Josh Gao87afd522019-03-28 11:05:53 -0700355 }
356
357 enabled = true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700358 StartWorker();
359 break;
360
361 case FUNCTIONFS_DISABLE:
Dan Albert782036b2019-06-24 14:35:35 -0700362 if (!bound) {
363 LOG(WARNING) << "received FUNCTIONFS_DISABLE while not bound?";
364 }
365
Josh Gao87afd522019-03-28 11:05:53 -0700366 if (!enabled) {
367 LOG(WARNING) << "received FUNCTIONFS_DISABLE while not enabled?";
368 }
369
370 enabled = false;
Josh Gao6933d542019-03-26 13:21:42 -0700371 running = false;
372 break;
373
374 case FUNCTIONFS_UNBIND:
Josh Gao87afd522019-03-28 11:05:53 -0700375 if (enabled) {
376 LOG(WARNING) << "received FUNCTIONFS_UNBIND while still enabled?";
377 }
Josh Gao6933d542019-03-26 13:21:42 -0700378
Dan Albert782036b2019-06-24 14:35:35 -0700379 if (!bound) {
380 LOG(WARNING) << "received FUNCTIONFS_UNBIND when not bound?";
381 }
382
383 bound = false;
Josh Gaoc51726c2018-10-11 16:33:05 -0700384 running = false;
385 break;
Josh Gao12807c72019-05-15 18:03:29 -0700386
387 case FUNCTIONFS_SETUP: {
Dan Albert782036b2019-06-24 14:35:35 -0700388 LOG(INFO) << "received FUNCTIONFS_SETUP control transfer: bRequestType = "
389 << static_cast<int>(event.u.setup.bRequestType)
390 << ", bRequest = " << static_cast<int>(event.u.setup.bRequest)
391 << ", wValue = " << static_cast<int>(event.u.setup.wValue)
392 << ", wIndex = " << static_cast<int>(event.u.setup.wIndex)
393 << ", wLength = " << static_cast<int>(event.u.setup.wLength);
394
395 if ((event.u.setup.bRequestType & USB_DIR_IN)) {
396 LOG(INFO) << "acking device-to-host control transfer";
397 ssize_t rc = adb_write(control_fd_.get(), "", 0);
398 if (rc != 0) {
399 PLOG(ERROR) << "failed to write empty packet to host";
400 break;
401 }
402 } else {
403 std::string buf;
404 buf.resize(event.u.setup.wLength + 1);
405
406 ssize_t rc = adb_read(control_fd_.get(), buf.data(), buf.size());
407 if (rc != event.u.setup.wLength) {
408 LOG(ERROR)
409 << "read " << rc
410 << " bytes when trying to read control request, expected "
411 << event.u.setup.wLength;
412 }
413
414 LOG(INFO) << "control request contents: " << buf;
415 break;
416 }
Josh Gao12807c72019-05-15 18:03:29 -0700417 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700418 }
419 }
420
Josh Gaoe778b3a2019-02-28 13:29:32 -0800421 StopWorker();
Josh Gao19dc2962019-03-26 18:47:45 -0700422 HandleError("monitor thread finished");
Josh Gaoc51726c2018-10-11 16:33:05 -0700423 });
424 }
425
426 void StartWorker() {
Josh Gao19dc2962019-03-26 18:47:45 -0700427 CHECK(!worker_started_);
428 worker_started_ = true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700429 worker_thread_ = std::thread([this]() {
430 adb_thread_setname("UsbFfs-worker");
Josh Gaoe3d34e12020-02-26 15:44:30 -0800431 LOG(INFO) << "UsbFfs-worker thread spawned";
432
Josh Gaoc51726c2018-10-11 16:33:05 -0700433 for (size_t i = 0; i < kUsbReadQueueDepth; ++i) {
434 read_requests_[i] = CreateReadBlock(next_read_id_++);
Josh Gaoc0b831b2019-02-13 15:27:28 -0800435 if (!SubmitRead(&read_requests_[i])) {
436 return;
437 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700438 }
439
440 while (!stopped_) {
441 uint64_t dummy;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800442 ssize_t rc = adb_read(worker_event_fd_.get(), &dummy, sizeof(dummy));
Josh Gaoc51726c2018-10-11 16:33:05 -0700443 if (rc == -1) {
444 PLOG(FATAL) << "failed to read from eventfd";
445 } else if (rc == 0) {
446 LOG(FATAL) << "hit EOF on eventfd";
447 }
448
Josh Gao6933d542019-03-26 13:21:42 -0700449 ReadEvents();
Josh Gaoc51726c2018-10-11 16:33:05 -0700450 }
451 });
452 }
453
Josh Gaoe778b3a2019-02-28 13:29:32 -0800454 void StopWorker() {
Josh Gao19dc2962019-03-26 18:47:45 -0700455 if (!worker_started_) {
456 return;
457 }
458
Josh Gaoe778b3a2019-02-28 13:29:32 -0800459 pthread_t worker_thread_handle = worker_thread_.native_handle();
460 while (true) {
461 int rc = pthread_kill(worker_thread_handle, kInterruptionSignal);
462 if (rc != 0) {
463 LOG(ERROR) << "failed to send interruption signal to worker: " << strerror(rc);
464 break;
465 }
466
467 std::this_thread::sleep_for(100ms);
468
469 rc = pthread_kill(worker_thread_handle, 0);
470 if (rc == 0) {
471 continue;
472 } else if (rc == ESRCH) {
473 break;
474 } else {
475 LOG(ERROR) << "failed to send interruption signal to worker: " << strerror(rc);
476 }
477 }
478
479 worker_thread_.join();
480 }
481
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700482 void PrepareReadBlock(IoReadBlock* block, uint64_t id) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700483 block->pending = false;
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700484 if (block->payload.capacity() >= kUsbReadSize) {
485 block->payload.resize(kUsbReadSize);
486 } else {
487 block->payload = Block(kUsbReadSize);
488 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700489 block->control.aio_data = static_cast<uint64_t>(TransferId::read(id));
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700490 block->control.aio_buf = reinterpret_cast<uintptr_t>(block->payload.data());
491 block->control.aio_nbytes = block->payload.size();
Josh Gaoc51726c2018-10-11 16:33:05 -0700492 }
493
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700494 IoReadBlock CreateReadBlock(uint64_t id) {
495 IoReadBlock block;
Josh Gaoc51726c2018-10-11 16:33:05 -0700496 PrepareReadBlock(&block, id);
497 block.control.aio_rw_flags = 0;
498 block.control.aio_lio_opcode = IOCB_CMD_PREAD;
499 block.control.aio_reqprio = 0;
500 block.control.aio_fildes = read_fd_.get();
501 block.control.aio_offset = 0;
502 block.control.aio_flags = IOCB_FLAG_RESFD;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800503 block.control.aio_resfd = worker_event_fd_.get();
Josh Gaoc51726c2018-10-11 16:33:05 -0700504 return block;
505 }
506
Josh Gao6933d542019-03-26 13:21:42 -0700507 void ReadEvents() {
Josh Gaoc51726c2018-10-11 16:33:05 -0700508 static constexpr size_t kMaxEvents = kUsbReadQueueDepth + kUsbWriteQueueDepth;
509 struct io_event events[kMaxEvents];
510 struct timespec timeout = {.tv_sec = 0, .tv_nsec = 0};
511 int rc = io_getevents(aio_context_.get(), 0, kMaxEvents, events, &timeout);
512 if (rc == -1) {
513 HandleError(StringPrintf("io_getevents failed while reading: %s", strerror(errno)));
514 return;
515 }
516
517 for (int event_idx = 0; event_idx < rc; ++event_idx) {
518 auto& event = events[event_idx];
519 TransferId id = TransferId::from_value(event.data);
520
521 if (event.res < 0) {
522 std::string error =
523 StringPrintf("%s %" PRIu64 " failed with error %s",
524 id.direction == TransferDirection::READ ? "read" : "write",
525 id.id, strerror(-event.res));
526 HandleError(error);
527 return;
528 }
529
530 if (id.direction == TransferDirection::READ) {
Josh Gao7b304842020-02-28 14:53:56 -0800531 if (!HandleRead(id, event.res)) {
532 return;
533 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700534 } else {
535 HandleWrite(id);
536 }
537 }
538 }
539
Josh Gao7b304842020-02-28 14:53:56 -0800540 bool HandleRead(TransferId id, int64_t size) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700541 uint64_t read_idx = id.id % kUsbReadQueueDepth;
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700542 IoReadBlock* block = &read_requests_[read_idx];
Josh Gaoc51726c2018-10-11 16:33:05 -0700543 block->pending = false;
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700544 block->payload.resize(size);
Josh Gaoc51726c2018-10-11 16:33:05 -0700545
546 // Notification for completed reads can be received out of order.
547 if (block->id().id != needed_read_id_) {
548 LOG(VERBOSE) << "read " << block->id().id << " completed while waiting for "
549 << needed_read_id_;
Josh Gao7b304842020-02-28 14:53:56 -0800550 return true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700551 }
552
553 for (uint64_t id = needed_read_id_;; ++id) {
554 size_t read_idx = id % kUsbReadQueueDepth;
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700555 IoReadBlock* current_block = &read_requests_[read_idx];
Josh Gaoc51726c2018-10-11 16:33:05 -0700556 if (current_block->pending) {
557 break;
558 }
Josh Gao7b304842020-02-28 14:53:56 -0800559 if (!ProcessRead(current_block)) {
560 return false;
561 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700562 ++needed_read_id_;
563 }
Josh Gao7b304842020-02-28 14:53:56 -0800564
565 return true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700566 }
567
Josh Gao7b304842020-02-28 14:53:56 -0800568 bool ProcessRead(IoReadBlock* block) {
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700569 if (!block->payload.empty()) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700570 if (!incoming_header_.has_value()) {
Josh Gao7b304842020-02-28 14:53:56 -0800571 if (block->payload.size() != sizeof(amessage)) {
572 HandleError("received packet of unexpected length while reading header");
573 return false;
574 }
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700575 amessage& msg = incoming_header_.emplace();
576 memcpy(&msg, block->payload.data(), sizeof(msg));
Josh Gaoc51726c2018-10-11 16:33:05 -0700577 LOG(DEBUG) << "USB read:" << dump_header(&msg);
578 incoming_header_ = msg;
579 } else {
580 size_t bytes_left = incoming_header_->data_length - incoming_payload_.size();
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700581 Block payload = std::move(block->payload);
Josh Gao7b304842020-02-28 14:53:56 -0800582 if (block->payload.size() > bytes_left) {
583 HandleError("received too many bytes while waiting for payload");
584 return false;
585 }
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700586 incoming_payload_.append(std::move(payload));
Josh Gaoc51726c2018-10-11 16:33:05 -0700587 }
588
589 if (incoming_header_->data_length == incoming_payload_.size()) {
590 auto packet = std::make_unique<apacket>();
591 packet->msg = *incoming_header_;
592
593 // TODO: Make apacket contain an IOVector so we don't have to coalesce.
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700594 packet->payload = std::move(incoming_payload_).coalesce();
Josh Gaoc51726c2018-10-11 16:33:05 -0700595 read_callback_(this, std::move(packet));
596
597 incoming_header_.reset();
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700598 // reuse the capacity of the incoming payload while we can.
599 auto free_block = incoming_payload_.clear();
600 if (block->payload.capacity() == 0) {
601 block->payload = std::move(free_block);
602 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700603 }
604 }
605
606 PrepareReadBlock(block, block->id().id + kUsbReadQueueDepth);
607 SubmitRead(block);
Josh Gao7b304842020-02-28 14:53:56 -0800608 return true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700609 }
610
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700611 bool SubmitRead(IoReadBlock* block) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700612 block->pending = true;
613 struct iocb* iocb = &block->control;
614 if (io_submit(aio_context_.get(), 1, &iocb) != 1) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800615 if (errno == EINVAL && !gFfsAioSupported.has_value()) {
616 HandleError("failed to submit first read, AIO on FFS not supported");
617 gFfsAioSupported = false;
618 return false;
619 }
620
Josh Gaoc51726c2018-10-11 16:33:05 -0700621 HandleError(StringPrintf("failed to submit read: %s", strerror(errno)));
Josh Gaoc0b831b2019-02-13 15:27:28 -0800622 return false;
Josh Gaoc51726c2018-10-11 16:33:05 -0700623 }
Josh Gaoc0b831b2019-02-13 15:27:28 -0800624
625 gFfsAioSupported = true;
626 return true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700627 }
628
629 void HandleWrite(TransferId id) {
630 std::lock_guard<std::mutex> lock(write_mutex_);
631 auto it =
632 std::find_if(write_requests_.begin(), write_requests_.end(), [id](const auto& req) {
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700633 return static_cast<uint64_t>(req.id()) == static_cast<uint64_t>(id);
Josh Gaoc51726c2018-10-11 16:33:05 -0700634 });
635 CHECK(it != write_requests_.end());
636
637 write_requests_.erase(it);
638 size_t outstanding_writes = --writes_submitted_;
639 LOG(DEBUG) << "USB write: reaped, down to " << outstanding_writes;
640
641 SubmitWrites();
642 }
643
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700644 IoWriteBlock CreateWriteBlock(std::shared_ptr<Block> payload, size_t offset, size_t len,
645 uint64_t id) {
646 auto block = IoWriteBlock();
647 block.payload = std::move(payload);
648 block.control.aio_data = static_cast<uint64_t>(TransferId::write(id));
649 block.control.aio_rw_flags = 0;
650 block.control.aio_lio_opcode = IOCB_CMD_PWRITE;
651 block.control.aio_reqprio = 0;
652 block.control.aio_fildes = write_fd_.get();
653 block.control.aio_buf = reinterpret_cast<uintptr_t>(block.payload->data() + offset);
654 block.control.aio_nbytes = len;
655 block.control.aio_offset = 0;
656 block.control.aio_flags = IOCB_FLAG_RESFD;
657 block.control.aio_resfd = worker_event_fd_.get();
Josh Gaoc51726c2018-10-11 16:33:05 -0700658 return block;
659 }
660
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700661 IoWriteBlock CreateWriteBlock(Block&& payload, uint64_t id) {
662 size_t len = payload.size();
663 return CreateWriteBlock(std::make_shared<Block>(std::move(payload)), 0, len, id);
Josh Gao86b33be2019-02-26 17:53:52 -0800664 }
665
Josh Gaoc51726c2018-10-11 16:33:05 -0700666 void SubmitWrites() REQUIRES(write_mutex_) {
667 if (writes_submitted_ == kUsbWriteQueueDepth) {
668 return;
669 }
670
671 ssize_t writes_to_submit = std::min(kUsbWriteQueueDepth - writes_submitted_,
672 write_requests_.size() - writes_submitted_);
673 CHECK_GE(writes_to_submit, 0);
674 if (writes_to_submit == 0) {
675 return;
676 }
677
678 struct iocb* iocbs[kUsbWriteQueueDepth];
679 for (int i = 0; i < writes_to_submit; ++i) {
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700680 CHECK(!write_requests_[writes_submitted_ + i].pending);
681 write_requests_[writes_submitted_ + i].pending = true;
682 iocbs[i] = &write_requests_[writes_submitted_ + i].control;
Josh Gaoc51726c2018-10-11 16:33:05 -0700683 LOG(VERBOSE) << "submitting write_request " << static_cast<void*>(iocbs[i]);
684 }
685
Josh Gao63b52ec2019-03-26 13:06:38 -0700686 writes_submitted_ += writes_to_submit;
687
Josh Gaoc51726c2018-10-11 16:33:05 -0700688 int rc = io_submit(aio_context_.get(), writes_to_submit, iocbs);
689 if (rc == -1) {
690 HandleError(StringPrintf("failed to submit write requests: %s", strerror(errno)));
691 return;
692 } else if (rc != writes_to_submit) {
693 LOG(FATAL) << "failed to submit all writes: wanted to submit " << writes_to_submit
694 << ", actually submitted " << rc;
695 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700696 }
697
698 void HandleError(const std::string& error) {
699 std::call_once(error_flag_, [&]() {
700 error_callback_(this, error);
701 if (!stopped_) {
702 Stop();
703 }
704 });
705 }
706
707 std::thread monitor_thread_;
Josh Gao19dc2962019-03-26 18:47:45 -0700708
709 bool worker_started_;
Josh Gaoc51726c2018-10-11 16:33:05 -0700710 std::thread worker_thread_;
711
712 std::atomic<bool> stopped_;
713 std::promise<void> destruction_notifier_;
714 std::once_flag error_flag_;
715
Josh Gaoc0b831b2019-02-13 15:27:28 -0800716 unique_fd worker_event_fd_;
Dan Albert782036b2019-06-24 14:35:35 -0700717 unique_fd monitor_event_fd_;
Josh Gaoc51726c2018-10-11 16:33:05 -0700718
719 ScopedAioContext aio_context_;
Dan Albert782036b2019-06-24 14:35:35 -0700720 unique_fd control_fd_;
Josh Gaoc51726c2018-10-11 16:33:05 -0700721 unique_fd read_fd_;
722 unique_fd write_fd_;
723
724 std::optional<amessage> incoming_header_;
725 IOVector incoming_payload_;
726
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700727 std::array<IoReadBlock, kUsbReadQueueDepth> read_requests_;
Josh Gaoc51726c2018-10-11 16:33:05 -0700728 IOVector read_data_;
729
730 // ID of the next request that we're going to send out.
731 size_t next_read_id_ = 0;
732
733 // ID of the next packet we're waiting for.
734 size_t needed_read_id_ = 0;
735
736 std::mutex write_mutex_;
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700737 std::deque<IoWriteBlock> write_requests_ GUARDED_BY(write_mutex_);
Josh Gaoc51726c2018-10-11 16:33:05 -0700738 size_t next_write_id_ GUARDED_BY(write_mutex_) = 0;
739 size_t writes_submitted_ GUARDED_BY(write_mutex_) = 0;
Josh Gaoe778b3a2019-02-28 13:29:32 -0800740
741 static constexpr int kInterruptionSignal = SIGUSR1;
Josh Gaoc51726c2018-10-11 16:33:05 -0700742};
743
Josh Gaoc0b831b2019-02-13 15:27:28 -0800744void usb_init_legacy();
745
Josh Gaoc51726c2018-10-11 16:33:05 -0700746static void usb_ffs_open_thread() {
747 adb_thread_setname("usb ffs open");
748
749 while (true) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800750 if (gFfsAioSupported.has_value() && !gFfsAioSupported.value()) {
751 LOG(INFO) << "failed to use nonblocking ffs, falling back to legacy";
752 return usb_init_legacy();
753 }
754
Dan Albert782036b2019-06-24 14:35:35 -0700755 unique_fd control;
756 unique_fd bulk_out;
757 unique_fd bulk_in;
Josh Gaoc51726c2018-10-11 16:33:05 -0700758 if (!open_functionfs(&control, &bulk_out, &bulk_in)) {
759 std::this_thread::sleep_for(1s);
760 continue;
761 }
762
763 atransport* transport = new atransport();
764 transport->serial = "UsbFfs";
765 std::promise<void> destruction_notifier;
766 std::future<void> future = destruction_notifier.get_future();
767 transport->SetConnection(std::make_unique<UsbFfsConnection>(
Dan Albert782036b2019-06-24 14:35:35 -0700768 std::move(control), std::move(bulk_out), std::move(bulk_in),
Josh Gaoc51726c2018-10-11 16:33:05 -0700769 std::move(destruction_notifier)));
770 register_transport(transport);
771 future.wait();
772 }
773}
774
Josh Gaoc51726c2018-10-11 16:33:05 -0700775void usb_init() {
Josh Gao8038e352019-03-18 16:33:18 -0700776 bool use_nonblocking = android::base::GetBoolProperty(
777 "persist.adb.nonblocking_ffs",
778 android::base::GetBoolProperty("ro.adb.nonblocking_ffs", true));
779
Josh Gao02e94a42019-02-28 07:26:20 +0000780 if (use_nonblocking) {
Josh Gao0d780392019-02-26 22:10:33 +0000781 std::thread(usb_ffs_open_thread).detach();
Josh Gao02e94a42019-02-28 07:26:20 +0000782 } else {
783 usb_init_legacy();
Josh Gaoc51726c2018-10-11 16:33:05 -0700784 }
785}