blob: c7f8895b857596cd5140b547eadd7baf1caf2fd1 [file] [log] [blame]
Josh Gaoc51726c2018-10-11 16:33:05 -07001/*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define TRACE_TAG USB
18
19#include "sysdeps.h"
20
21#include <errno.h>
22#include <stdio.h>
23#include <stdlib.h>
24#include <string.h>
25#include <sys/ioctl.h>
26#include <sys/types.h>
27#include <unistd.h>
28
29#include <linux/usb/functionfs.h>
30#include <sys/eventfd.h>
31
Josh Gao86b33be2019-02-26 17:53:52 -080032#include <algorithm>
Josh Gaoc51726c2018-10-11 16:33:05 -070033#include <array>
34#include <future>
35#include <memory>
36#include <mutex>
37#include <optional>
38#include <vector>
39
40#include <asyncio/AsyncIO.h>
41
42#include <android-base/logging.h>
43#include <android-base/macros.h>
44#include <android-base/properties.h>
45#include <android-base/thread_annotations.h>
46
47#include <adbd/usb.h>
48
49#include "adb_unique_fd.h"
50#include "adb_utils.h"
51#include "sysdeps/chrono.h"
52#include "transport.h"
53#include "types.h"
54
55using android::base::StringPrintf;
56
Josh Gaoc0b831b2019-02-13 15:27:28 -080057// We can't find out whether we have support for AIO on ffs endpoints until we submit a read.
58static std::optional<bool> gFfsAioSupported;
59
Josh Gao08ccc732019-04-16 11:20:04 -070060// Not all USB controllers support operations larger than 16k, so don't go above that.
Josh Gaod0feaf92019-04-24 14:28:25 -070061// Also, each submitted operation does an allocation in the kernel of that size, so we want to
62// minimize our queue depth while still maintaining a deep enough queue to keep the USB stack fed.
63static constexpr size_t kUsbReadQueueDepth = 8;
Josh Gao08ccc732019-04-16 11:20:04 -070064static constexpr size_t kUsbReadSize = 4 * PAGE_SIZE;
Josh Gaoc51726c2018-10-11 16:33:05 -070065
Josh Gaod0feaf92019-04-24 14:28:25 -070066static constexpr size_t kUsbWriteQueueDepth = 8;
Josh Gao08ccc732019-04-16 11:20:04 -070067static constexpr size_t kUsbWriteSize = 4 * PAGE_SIZE;
Josh Gaoc51726c2018-10-11 16:33:05 -070068
Dan Albert782036b2019-06-24 14:35:35 -070069static const char* to_string(enum usb_functionfs_event_type type) {
70 switch (type) {
71 case FUNCTIONFS_BIND:
72 return "FUNCTIONFS_BIND";
73 case FUNCTIONFS_UNBIND:
74 return "FUNCTIONFS_UNBIND";
75 case FUNCTIONFS_ENABLE:
76 return "FUNCTIONFS_ENABLE";
77 case FUNCTIONFS_DISABLE:
78 return "FUNCTIONFS_DISABLE";
79 case FUNCTIONFS_SETUP:
80 return "FUNCTIONFS_SETUP";
81 case FUNCTIONFS_SUSPEND:
82 return "FUNCTIONFS_SUSPEND";
83 case FUNCTIONFS_RESUME:
84 return "FUNCTIONFS_RESUME";
85 }
86}
87
Josh Gaoc51726c2018-10-11 16:33:05 -070088enum class TransferDirection : uint64_t {
89 READ = 0,
90 WRITE = 1,
91};
92
93struct TransferId {
94 TransferDirection direction : 1;
95 uint64_t id : 63;
96
97 TransferId() : TransferId(TransferDirection::READ, 0) {}
98
99 private:
100 TransferId(TransferDirection direction, uint64_t id) : direction(direction), id(id) {}
101
102 public:
103 explicit operator uint64_t() const {
104 uint64_t result;
105 static_assert(sizeof(*this) == sizeof(result));
106 memcpy(&result, this, sizeof(*this));
107 return result;
108 }
109
110 static TransferId read(uint64_t id) { return TransferId(TransferDirection::READ, id); }
111 static TransferId write(uint64_t id) { return TransferId(TransferDirection::WRITE, id); }
112
113 static TransferId from_value(uint64_t value) {
114 TransferId result;
115 memcpy(&result, &value, sizeof(value));
116 return result;
117 }
118};
119
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700120template <class Payload>
Josh Gaoc51726c2018-10-11 16:33:05 -0700121struct IoBlock {
Josh Gaob0195742019-03-18 14:11:28 -0700122 bool pending = false;
Evgenii Stepanov9da358d2019-05-15 18:45:01 -0700123 struct iocb control = {};
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700124 Payload payload;
Josh Gaoc51726c2018-10-11 16:33:05 -0700125
126 TransferId id() const { return TransferId::from_value(control.aio_data); }
127};
128
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700129using IoReadBlock = IoBlock<Block>;
130using IoWriteBlock = IoBlock<std::shared_ptr<Block>>;
131
Josh Gaoc51726c2018-10-11 16:33:05 -0700132struct ScopedAioContext {
133 ScopedAioContext() = default;
134 ~ScopedAioContext() { reset(); }
135
136 ScopedAioContext(ScopedAioContext&& move) { reset(move.release()); }
137 ScopedAioContext(const ScopedAioContext& copy) = delete;
138
139 ScopedAioContext& operator=(ScopedAioContext&& move) {
140 reset(move.release());
141 return *this;
142 }
143 ScopedAioContext& operator=(const ScopedAioContext& copy) = delete;
144
145 static ScopedAioContext Create(size_t max_events) {
146 aio_context_t ctx = 0;
147 if (io_setup(max_events, &ctx) != 0) {
148 PLOG(FATAL) << "failed to create aio_context_t";
149 }
150 ScopedAioContext result;
151 result.reset(ctx);
152 return result;
153 }
154
155 aio_context_t release() {
156 aio_context_t result = context_;
157 context_ = 0;
158 return result;
159 }
160
161 void reset(aio_context_t new_context = 0) {
162 if (context_ != 0) {
163 io_destroy(context_);
164 }
165
166 context_ = new_context;
167 }
168
169 aio_context_t get() { return context_; }
170
171 private:
172 aio_context_t context_ = 0;
173};
174
175struct UsbFfsConnection : public Connection {
Dan Albert782036b2019-06-24 14:35:35 -0700176 UsbFfsConnection(unique_fd control, unique_fd read, unique_fd write,
Josh Gaoc51726c2018-10-11 16:33:05 -0700177 std::promise<void> destruction_notifier)
Josh Gao19dc2962019-03-26 18:47:45 -0700178 : worker_started_(false),
179 stopped_(false),
Josh Gaoc51726c2018-10-11 16:33:05 -0700180 destruction_notifier_(std::move(destruction_notifier)),
Dan Albert782036b2019-06-24 14:35:35 -0700181 control_fd_(std::move(control)),
Josh Gaoc51726c2018-10-11 16:33:05 -0700182 read_fd_(std::move(read)),
183 write_fd_(std::move(write)) {
184 LOG(INFO) << "UsbFfsConnection constructed";
Josh Gaoc0b831b2019-02-13 15:27:28 -0800185 worker_event_fd_.reset(eventfd(0, EFD_CLOEXEC));
186 if (worker_event_fd_ == -1) {
187 PLOG(FATAL) << "failed to create eventfd";
188 }
189
Dan Albert782036b2019-06-24 14:35:35 -0700190 monitor_event_fd_.reset(eventfd(0, EFD_CLOEXEC));
191 if (monitor_event_fd_ == -1) {
192 PLOG(FATAL) << "failed to create eventfd";
193 }
194
Josh Gaoc51726c2018-10-11 16:33:05 -0700195 aio_context_ = ScopedAioContext::Create(kUsbReadQueueDepth + kUsbWriteQueueDepth);
196 }
197
198 ~UsbFfsConnection() {
199 LOG(INFO) << "UsbFfsConnection being destroyed";
200 Stop();
201 monitor_thread_.join();
Josh Gaoc0b831b2019-02-13 15:27:28 -0800202
203 // We need to explicitly close our file descriptors before we notify our destruction,
204 // because the thread listening on the future will immediately try to reopen the endpoint.
Josh Gao19dc2962019-03-26 18:47:45 -0700205 aio_context_.reset();
Dan Albert782036b2019-06-24 14:35:35 -0700206 control_fd_.reset();
Josh Gaoc0b831b2019-02-13 15:27:28 -0800207 read_fd_.reset();
208 write_fd_.reset();
209
Josh Gaoc51726c2018-10-11 16:33:05 -0700210 destruction_notifier_.set_value();
211 }
212
213 virtual bool Write(std::unique_ptr<apacket> packet) override final {
214 LOG(DEBUG) << "USB write: " << dump_header(&packet->msg);
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700215 auto header = std::make_shared<Block>(sizeof(packet->msg));
216 memcpy(header->data(), &packet->msg, sizeof(packet->msg));
Josh Gaoc51726c2018-10-11 16:33:05 -0700217
218 std::lock_guard<std::mutex> lock(write_mutex_);
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700219 write_requests_.push_back(
220 CreateWriteBlock(std::move(header), 0, sizeof(packet->msg), next_write_id_++));
Josh Gaoc51726c2018-10-11 16:33:05 -0700221 if (!packet->payload.empty()) {
Josh Gao86b33be2019-02-26 17:53:52 -0800222 // The kernel attempts to allocate a contiguous block of memory for each write,
223 // which can fail if the write is large and the kernel heap is fragmented.
224 // Split large writes into smaller chunks to avoid this.
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700225 auto payload = std::make_shared<Block>(std::move(packet->payload));
Josh Gao86b33be2019-02-26 17:53:52 -0800226 size_t offset = 0;
227 size_t len = payload->size();
228
229 while (len > 0) {
230 size_t write_size = std::min(kUsbWriteSize, len);
231 write_requests_.push_back(
232 CreateWriteBlock(payload, offset, write_size, next_write_id_++));
233 len -= write_size;
234 offset += write_size;
235 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700236 }
237 SubmitWrites();
238 return true;
239 }
240
241 virtual void Start() override final { StartMonitor(); }
242
243 virtual void Stop() override final {
244 if (stopped_.exchange(true)) {
245 return;
246 }
247 stopped_ = true;
248 uint64_t notify = 1;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800249 ssize_t rc = adb_write(worker_event_fd_.get(), &notify, sizeof(notify));
Josh Gaoc51726c2018-10-11 16:33:05 -0700250 if (rc < 0) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800251 PLOG(FATAL) << "failed to notify worker eventfd to stop UsbFfsConnection";
Josh Gaoc51726c2018-10-11 16:33:05 -0700252 }
253 CHECK_EQ(static_cast<size_t>(rc), sizeof(notify));
Dan Albert782036b2019-06-24 14:35:35 -0700254
255 rc = adb_write(monitor_event_fd_.get(), &notify, sizeof(notify));
256 if (rc < 0) {
257 PLOG(FATAL) << "failed to notify monitor eventfd to stop UsbFfsConnection";
258 }
259
260 CHECK_EQ(static_cast<size_t>(rc), sizeof(notify));
Josh Gaoc51726c2018-10-11 16:33:05 -0700261 }
262
Joshua Duong5cf78682020-01-21 13:19:42 -0800263 virtual bool DoTlsHandshake(RSA* key, std::string* auth_key) override final {
264 // TODO: support TLS for usb connections.
265 LOG(FATAL) << "Not supported yet.";
266 return false;
267 }
268
Josh Gaoc51726c2018-10-11 16:33:05 -0700269 private:
270 void StartMonitor() {
271 // This is a bit of a mess.
272 // It's possible for io_submit to end up blocking, if we call it as the endpoint
273 // becomes disabled. Work around this by having a monitor thread to listen for functionfs
274 // lifecycle events. If we notice an error condition (either we've become disabled, or we
275 // were never enabled in the first place), we send interruption signals to the worker thread
276 // until it dies, and then report failure to the transport via HandleError, which will
277 // eventually result in the transport being destroyed, which will result in UsbFfsConnection
278 // being destroyed, which unblocks the open thread and restarts this entire process.
Josh Gaoc51726c2018-10-11 16:33:05 -0700279 static std::once_flag handler_once;
280 std::call_once(handler_once, []() { signal(kInterruptionSignal, [](int) {}); });
281
282 monitor_thread_ = std::thread([this]() {
283 adb_thread_setname("UsbFfs-monitor");
284
Dan Albert782036b2019-06-24 14:35:35 -0700285 bool bound = false;
Josh Gao6933d542019-03-26 13:21:42 -0700286 bool enabled = false;
Josh Gaoc51726c2018-10-11 16:33:05 -0700287 bool running = true;
288 while (running) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800289 adb_pollfd pfd[2] = {
Dan Albert782036b2019-06-24 14:35:35 -0700290 { .fd = control_fd_.get(), .events = POLLIN, .revents = 0 },
291 { .fd = monitor_event_fd_.get(), .events = POLLIN, .revents = 0 },
Josh Gaoc0b831b2019-02-13 15:27:28 -0800292 };
Josh Gao19dc2962019-03-26 18:47:45 -0700293
Dan Albert782036b2019-06-24 14:35:35 -0700294 // If we don't see our first bind within a second, try again.
295 int timeout_ms = bound ? -1 : 1000;
296
297 int rc = TEMP_FAILURE_RETRY(adb_poll(pfd, 2, timeout_ms));
Josh Gaoc0b831b2019-02-13 15:27:28 -0800298 if (rc == -1) {
299 PLOG(FATAL) << "poll on USB control fd failed";
Dan Albert782036b2019-06-24 14:35:35 -0700300 } else if (rc == 0) {
301 LOG(WARNING) << "timed out while waiting for FUNCTIONFS_BIND, trying again";
302 break;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800303 }
304
305 if (pfd[1].revents) {
Dan Albert782036b2019-06-24 14:35:35 -0700306 // We were told to die.
307 break;
Josh Gaoc51726c2018-10-11 16:33:05 -0700308 }
309
310 struct usb_functionfs_event event;
Dan Albert782036b2019-06-24 14:35:35 -0700311 rc = TEMP_FAILURE_RETRY(adb_read(control_fd_.get(), &event, sizeof(event)));
Josh Gao1f7ae9d2019-05-10 11:37:34 -0700312 if (rc == -1) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700313 PLOG(FATAL) << "failed to read functionfs event";
Josh Gao1f7ae9d2019-05-10 11:37:34 -0700314 } else if (rc == 0) {
315 LOG(WARNING) << "hit EOF on functionfs control fd";
316 break;
317 } else if (rc != sizeof(event)) {
318 LOG(FATAL) << "read functionfs event of unexpected size, expected "
319 << sizeof(event) << ", got " << rc;
Josh Gaoc51726c2018-10-11 16:33:05 -0700320 }
321
322 LOG(INFO) << "USB event: "
Dan Albert782036b2019-06-24 14:35:35 -0700323 << to_string(static_cast<usb_functionfs_event_type>(event.type));
Josh Gaoc51726c2018-10-11 16:33:05 -0700324
325 switch (event.type) {
326 case FUNCTIONFS_BIND:
Dan Albert782036b2019-06-24 14:35:35 -0700327 if (bound) {
328 LOG(WARNING) << "received FUNCTIONFS_BIND while already bound?";
329 running = false;
330 break;
331 }
332
333 if (enabled) {
334 LOG(WARNING) << "received FUNCTIONFS_BIND while already enabled?";
335 running = false;
336 break;
337 }
338
339 bound = true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700340 break;
341
342 case FUNCTIONFS_ENABLE:
Dan Albert782036b2019-06-24 14:35:35 -0700343 if (!bound) {
344 LOG(WARNING) << "received FUNCTIONFS_ENABLE while not bound?";
345 running = false;
346 break;
347 }
348
Josh Gao87afd522019-03-28 11:05:53 -0700349 if (enabled) {
350 LOG(WARNING) << "received FUNCTIONFS_ENABLE while already enabled?";
351 running = false;
Josh Gao94fb36b2019-05-01 16:53:53 -0700352 break;
Josh Gao87afd522019-03-28 11:05:53 -0700353 }
354
355 enabled = true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700356 StartWorker();
357 break;
358
359 case FUNCTIONFS_DISABLE:
Dan Albert782036b2019-06-24 14:35:35 -0700360 if (!bound) {
361 LOG(WARNING) << "received FUNCTIONFS_DISABLE while not bound?";
362 }
363
Josh Gao87afd522019-03-28 11:05:53 -0700364 if (!enabled) {
365 LOG(WARNING) << "received FUNCTIONFS_DISABLE while not enabled?";
366 }
367
368 enabled = false;
Josh Gao6933d542019-03-26 13:21:42 -0700369 running = false;
370 break;
371
372 case FUNCTIONFS_UNBIND:
Josh Gao87afd522019-03-28 11:05:53 -0700373 if (enabled) {
374 LOG(WARNING) << "received FUNCTIONFS_UNBIND while still enabled?";
375 }
Josh Gao6933d542019-03-26 13:21:42 -0700376
Dan Albert782036b2019-06-24 14:35:35 -0700377 if (!bound) {
378 LOG(WARNING) << "received FUNCTIONFS_UNBIND when not bound?";
379 }
380
381 bound = false;
Josh Gaoc51726c2018-10-11 16:33:05 -0700382 running = false;
383 break;
Josh Gao12807c72019-05-15 18:03:29 -0700384
385 case FUNCTIONFS_SETUP: {
Dan Albert782036b2019-06-24 14:35:35 -0700386 LOG(INFO) << "received FUNCTIONFS_SETUP control transfer: bRequestType = "
387 << static_cast<int>(event.u.setup.bRequestType)
388 << ", bRequest = " << static_cast<int>(event.u.setup.bRequest)
389 << ", wValue = " << static_cast<int>(event.u.setup.wValue)
390 << ", wIndex = " << static_cast<int>(event.u.setup.wIndex)
391 << ", wLength = " << static_cast<int>(event.u.setup.wLength);
392
393 if ((event.u.setup.bRequestType & USB_DIR_IN)) {
394 LOG(INFO) << "acking device-to-host control transfer";
395 ssize_t rc = adb_write(control_fd_.get(), "", 0);
396 if (rc != 0) {
397 PLOG(ERROR) << "failed to write empty packet to host";
398 break;
399 }
400 } else {
401 std::string buf;
402 buf.resize(event.u.setup.wLength + 1);
403
404 ssize_t rc = adb_read(control_fd_.get(), buf.data(), buf.size());
405 if (rc != event.u.setup.wLength) {
406 LOG(ERROR)
407 << "read " << rc
408 << " bytes when trying to read control request, expected "
409 << event.u.setup.wLength;
410 }
411
412 LOG(INFO) << "control request contents: " << buf;
413 break;
414 }
Josh Gao12807c72019-05-15 18:03:29 -0700415 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700416 }
417 }
418
Josh Gaoe778b3a2019-02-28 13:29:32 -0800419 StopWorker();
Josh Gao19dc2962019-03-26 18:47:45 -0700420 HandleError("monitor thread finished");
Josh Gaoc51726c2018-10-11 16:33:05 -0700421 });
422 }
423
424 void StartWorker() {
Josh Gao19dc2962019-03-26 18:47:45 -0700425 CHECK(!worker_started_);
426 worker_started_ = true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700427 worker_thread_ = std::thread([this]() {
428 adb_thread_setname("UsbFfs-worker");
429 for (size_t i = 0; i < kUsbReadQueueDepth; ++i) {
430 read_requests_[i] = CreateReadBlock(next_read_id_++);
Josh Gaoc0b831b2019-02-13 15:27:28 -0800431 if (!SubmitRead(&read_requests_[i])) {
432 return;
433 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700434 }
435
436 while (!stopped_) {
437 uint64_t dummy;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800438 ssize_t rc = adb_read(worker_event_fd_.get(), &dummy, sizeof(dummy));
Josh Gaoc51726c2018-10-11 16:33:05 -0700439 if (rc == -1) {
440 PLOG(FATAL) << "failed to read from eventfd";
441 } else if (rc == 0) {
442 LOG(FATAL) << "hit EOF on eventfd";
443 }
444
Josh Gao6933d542019-03-26 13:21:42 -0700445 ReadEvents();
Josh Gaoc51726c2018-10-11 16:33:05 -0700446 }
447 });
448 }
449
Josh Gaoe778b3a2019-02-28 13:29:32 -0800450 void StopWorker() {
Josh Gao19dc2962019-03-26 18:47:45 -0700451 if (!worker_started_) {
452 return;
453 }
454
Josh Gaoe778b3a2019-02-28 13:29:32 -0800455 pthread_t worker_thread_handle = worker_thread_.native_handle();
456 while (true) {
457 int rc = pthread_kill(worker_thread_handle, kInterruptionSignal);
458 if (rc != 0) {
459 LOG(ERROR) << "failed to send interruption signal to worker: " << strerror(rc);
460 break;
461 }
462
463 std::this_thread::sleep_for(100ms);
464
465 rc = pthread_kill(worker_thread_handle, 0);
466 if (rc == 0) {
467 continue;
468 } else if (rc == ESRCH) {
469 break;
470 } else {
471 LOG(ERROR) << "failed to send interruption signal to worker: " << strerror(rc);
472 }
473 }
474
475 worker_thread_.join();
476 }
477
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700478 void PrepareReadBlock(IoReadBlock* block, uint64_t id) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700479 block->pending = false;
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700480 if (block->payload.capacity() >= kUsbReadSize) {
481 block->payload.resize(kUsbReadSize);
482 } else {
483 block->payload = Block(kUsbReadSize);
484 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700485 block->control.aio_data = static_cast<uint64_t>(TransferId::read(id));
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700486 block->control.aio_buf = reinterpret_cast<uintptr_t>(block->payload.data());
487 block->control.aio_nbytes = block->payload.size();
Josh Gaoc51726c2018-10-11 16:33:05 -0700488 }
489
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700490 IoReadBlock CreateReadBlock(uint64_t id) {
491 IoReadBlock block;
Josh Gaoc51726c2018-10-11 16:33:05 -0700492 PrepareReadBlock(&block, id);
493 block.control.aio_rw_flags = 0;
494 block.control.aio_lio_opcode = IOCB_CMD_PREAD;
495 block.control.aio_reqprio = 0;
496 block.control.aio_fildes = read_fd_.get();
497 block.control.aio_offset = 0;
498 block.control.aio_flags = IOCB_FLAG_RESFD;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800499 block.control.aio_resfd = worker_event_fd_.get();
Josh Gaoc51726c2018-10-11 16:33:05 -0700500 return block;
501 }
502
Josh Gao6933d542019-03-26 13:21:42 -0700503 void ReadEvents() {
Josh Gaoc51726c2018-10-11 16:33:05 -0700504 static constexpr size_t kMaxEvents = kUsbReadQueueDepth + kUsbWriteQueueDepth;
505 struct io_event events[kMaxEvents];
506 struct timespec timeout = {.tv_sec = 0, .tv_nsec = 0};
507 int rc = io_getevents(aio_context_.get(), 0, kMaxEvents, events, &timeout);
508 if (rc == -1) {
509 HandleError(StringPrintf("io_getevents failed while reading: %s", strerror(errno)));
510 return;
511 }
512
513 for (int event_idx = 0; event_idx < rc; ++event_idx) {
514 auto& event = events[event_idx];
515 TransferId id = TransferId::from_value(event.data);
516
517 if (event.res < 0) {
518 std::string error =
519 StringPrintf("%s %" PRIu64 " failed with error %s",
520 id.direction == TransferDirection::READ ? "read" : "write",
521 id.id, strerror(-event.res));
522 HandleError(error);
523 return;
524 }
525
526 if (id.direction == TransferDirection::READ) {
Dan Albert2547f742019-06-24 14:35:52 -0700527 HandleRead(id, event.res);
Josh Gaoc51726c2018-10-11 16:33:05 -0700528 } else {
529 HandleWrite(id);
530 }
531 }
532 }
533
Dan Albert2547f742019-06-24 14:35:52 -0700534 void HandleRead(TransferId id, int64_t size) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700535 uint64_t read_idx = id.id % kUsbReadQueueDepth;
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700536 IoReadBlock* block = &read_requests_[read_idx];
Josh Gaoc51726c2018-10-11 16:33:05 -0700537 block->pending = false;
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700538 block->payload.resize(size);
Josh Gaoc51726c2018-10-11 16:33:05 -0700539
540 // Notification for completed reads can be received out of order.
541 if (block->id().id != needed_read_id_) {
542 LOG(VERBOSE) << "read " << block->id().id << " completed while waiting for "
543 << needed_read_id_;
Dan Albert2547f742019-06-24 14:35:52 -0700544 return;
Josh Gaoc51726c2018-10-11 16:33:05 -0700545 }
546
547 for (uint64_t id = needed_read_id_;; ++id) {
548 size_t read_idx = id % kUsbReadQueueDepth;
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700549 IoReadBlock* current_block = &read_requests_[read_idx];
Josh Gaoc51726c2018-10-11 16:33:05 -0700550 if (current_block->pending) {
551 break;
552 }
Dan Albert2547f742019-06-24 14:35:52 -0700553 ProcessRead(current_block);
Josh Gaoc51726c2018-10-11 16:33:05 -0700554 ++needed_read_id_;
555 }
556 }
557
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700558 void ProcessRead(IoReadBlock* block) {
559 if (!block->payload.empty()) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700560 if (!incoming_header_.has_value()) {
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700561 CHECK_EQ(sizeof(amessage), block->payload.size());
562 amessage& msg = incoming_header_.emplace();
563 memcpy(&msg, block->payload.data(), sizeof(msg));
Josh Gaoc51726c2018-10-11 16:33:05 -0700564 LOG(DEBUG) << "USB read:" << dump_header(&msg);
565 incoming_header_ = msg;
566 } else {
567 size_t bytes_left = incoming_header_->data_length - incoming_payload_.size();
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700568 Block payload = std::move(block->payload);
Dan Albert2547f742019-06-24 14:35:52 -0700569 CHECK_LE(payload.size(), bytes_left);
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700570 incoming_payload_.append(std::move(payload));
Josh Gaoc51726c2018-10-11 16:33:05 -0700571 }
572
573 if (incoming_header_->data_length == incoming_payload_.size()) {
574 auto packet = std::make_unique<apacket>();
575 packet->msg = *incoming_header_;
576
577 // TODO: Make apacket contain an IOVector so we don't have to coalesce.
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700578 packet->payload = std::move(incoming_payload_).coalesce();
Josh Gaoc51726c2018-10-11 16:33:05 -0700579 read_callback_(this, std::move(packet));
580
581 incoming_header_.reset();
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700582 // reuse the capacity of the incoming payload while we can.
583 auto free_block = incoming_payload_.clear();
584 if (block->payload.capacity() == 0) {
585 block->payload = std::move(free_block);
586 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700587 }
588 }
589
590 PrepareReadBlock(block, block->id().id + kUsbReadQueueDepth);
591 SubmitRead(block);
592 }
593
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700594 bool SubmitRead(IoReadBlock* block) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700595 block->pending = true;
596 struct iocb* iocb = &block->control;
597 if (io_submit(aio_context_.get(), 1, &iocb) != 1) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800598 if (errno == EINVAL && !gFfsAioSupported.has_value()) {
599 HandleError("failed to submit first read, AIO on FFS not supported");
600 gFfsAioSupported = false;
601 return false;
602 }
603
Josh Gaoc51726c2018-10-11 16:33:05 -0700604 HandleError(StringPrintf("failed to submit read: %s", strerror(errno)));
Josh Gaoc0b831b2019-02-13 15:27:28 -0800605 return false;
Josh Gaoc51726c2018-10-11 16:33:05 -0700606 }
Josh Gaoc0b831b2019-02-13 15:27:28 -0800607
608 gFfsAioSupported = true;
609 return true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700610 }
611
612 void HandleWrite(TransferId id) {
613 std::lock_guard<std::mutex> lock(write_mutex_);
614 auto it =
615 std::find_if(write_requests_.begin(), write_requests_.end(), [id](const auto& req) {
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700616 return static_cast<uint64_t>(req.id()) == static_cast<uint64_t>(id);
Josh Gaoc51726c2018-10-11 16:33:05 -0700617 });
618 CHECK(it != write_requests_.end());
619
620 write_requests_.erase(it);
621 size_t outstanding_writes = --writes_submitted_;
622 LOG(DEBUG) << "USB write: reaped, down to " << outstanding_writes;
623
624 SubmitWrites();
625 }
626
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700627 IoWriteBlock CreateWriteBlock(std::shared_ptr<Block> payload, size_t offset, size_t len,
628 uint64_t id) {
629 auto block = IoWriteBlock();
630 block.payload = std::move(payload);
631 block.control.aio_data = static_cast<uint64_t>(TransferId::write(id));
632 block.control.aio_rw_flags = 0;
633 block.control.aio_lio_opcode = IOCB_CMD_PWRITE;
634 block.control.aio_reqprio = 0;
635 block.control.aio_fildes = write_fd_.get();
636 block.control.aio_buf = reinterpret_cast<uintptr_t>(block.payload->data() + offset);
637 block.control.aio_nbytes = len;
638 block.control.aio_offset = 0;
639 block.control.aio_flags = IOCB_FLAG_RESFD;
640 block.control.aio_resfd = worker_event_fd_.get();
Josh Gaoc51726c2018-10-11 16:33:05 -0700641 return block;
642 }
643
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700644 IoWriteBlock CreateWriteBlock(Block&& payload, uint64_t id) {
645 size_t len = payload.size();
646 return CreateWriteBlock(std::make_shared<Block>(std::move(payload)), 0, len, id);
Josh Gao86b33be2019-02-26 17:53:52 -0800647 }
648
Josh Gaoc51726c2018-10-11 16:33:05 -0700649 void SubmitWrites() REQUIRES(write_mutex_) {
650 if (writes_submitted_ == kUsbWriteQueueDepth) {
651 return;
652 }
653
654 ssize_t writes_to_submit = std::min(kUsbWriteQueueDepth - writes_submitted_,
655 write_requests_.size() - writes_submitted_);
656 CHECK_GE(writes_to_submit, 0);
657 if (writes_to_submit == 0) {
658 return;
659 }
660
661 struct iocb* iocbs[kUsbWriteQueueDepth];
662 for (int i = 0; i < writes_to_submit; ++i) {
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700663 CHECK(!write_requests_[writes_submitted_ + i].pending);
664 write_requests_[writes_submitted_ + i].pending = true;
665 iocbs[i] = &write_requests_[writes_submitted_ + i].control;
Josh Gaoc51726c2018-10-11 16:33:05 -0700666 LOG(VERBOSE) << "submitting write_request " << static_cast<void*>(iocbs[i]);
667 }
668
Josh Gao63b52ec2019-03-26 13:06:38 -0700669 writes_submitted_ += writes_to_submit;
670
Josh Gaoc51726c2018-10-11 16:33:05 -0700671 int rc = io_submit(aio_context_.get(), writes_to_submit, iocbs);
672 if (rc == -1) {
673 HandleError(StringPrintf("failed to submit write requests: %s", strerror(errno)));
674 return;
675 } else if (rc != writes_to_submit) {
676 LOG(FATAL) << "failed to submit all writes: wanted to submit " << writes_to_submit
677 << ", actually submitted " << rc;
678 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700679 }
680
681 void HandleError(const std::string& error) {
682 std::call_once(error_flag_, [&]() {
683 error_callback_(this, error);
684 if (!stopped_) {
685 Stop();
686 }
687 });
688 }
689
690 std::thread monitor_thread_;
Josh Gao19dc2962019-03-26 18:47:45 -0700691
692 bool worker_started_;
Josh Gaoc51726c2018-10-11 16:33:05 -0700693 std::thread worker_thread_;
694
695 std::atomic<bool> stopped_;
696 std::promise<void> destruction_notifier_;
697 std::once_flag error_flag_;
698
Josh Gaoc0b831b2019-02-13 15:27:28 -0800699 unique_fd worker_event_fd_;
Dan Albert782036b2019-06-24 14:35:35 -0700700 unique_fd monitor_event_fd_;
Josh Gaoc51726c2018-10-11 16:33:05 -0700701
702 ScopedAioContext aio_context_;
Dan Albert782036b2019-06-24 14:35:35 -0700703 unique_fd control_fd_;
Josh Gaoc51726c2018-10-11 16:33:05 -0700704 unique_fd read_fd_;
705 unique_fd write_fd_;
706
707 std::optional<amessage> incoming_header_;
708 IOVector incoming_payload_;
709
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700710 std::array<IoReadBlock, kUsbReadQueueDepth> read_requests_;
Josh Gaoc51726c2018-10-11 16:33:05 -0700711 IOVector read_data_;
712
713 // ID of the next request that we're going to send out.
714 size_t next_read_id_ = 0;
715
716 // ID of the next packet we're waiting for.
717 size_t needed_read_id_ = 0;
718
719 std::mutex write_mutex_;
Yurii Zubrytskyi5dda7f62019-07-12 14:11:54 -0700720 std::deque<IoWriteBlock> write_requests_ GUARDED_BY(write_mutex_);
Josh Gaoc51726c2018-10-11 16:33:05 -0700721 size_t next_write_id_ GUARDED_BY(write_mutex_) = 0;
722 size_t writes_submitted_ GUARDED_BY(write_mutex_) = 0;
Josh Gaoe778b3a2019-02-28 13:29:32 -0800723
724 static constexpr int kInterruptionSignal = SIGUSR1;
Josh Gaoc51726c2018-10-11 16:33:05 -0700725};
726
Josh Gaoc0b831b2019-02-13 15:27:28 -0800727void usb_init_legacy();
728
Josh Gaoc51726c2018-10-11 16:33:05 -0700729static void usb_ffs_open_thread() {
730 adb_thread_setname("usb ffs open");
731
732 while (true) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800733 if (gFfsAioSupported.has_value() && !gFfsAioSupported.value()) {
734 LOG(INFO) << "failed to use nonblocking ffs, falling back to legacy";
735 return usb_init_legacy();
736 }
737
Dan Albert782036b2019-06-24 14:35:35 -0700738 unique_fd control;
739 unique_fd bulk_out;
740 unique_fd bulk_in;
Josh Gaoc51726c2018-10-11 16:33:05 -0700741 if (!open_functionfs(&control, &bulk_out, &bulk_in)) {
742 std::this_thread::sleep_for(1s);
743 continue;
744 }
745
746 atransport* transport = new atransport();
747 transport->serial = "UsbFfs";
748 std::promise<void> destruction_notifier;
749 std::future<void> future = destruction_notifier.get_future();
750 transport->SetConnection(std::make_unique<UsbFfsConnection>(
Dan Albert782036b2019-06-24 14:35:35 -0700751 std::move(control), std::move(bulk_out), std::move(bulk_in),
Josh Gaoc51726c2018-10-11 16:33:05 -0700752 std::move(destruction_notifier)));
753 register_transport(transport);
754 future.wait();
755 }
756}
757
Josh Gaoc51726c2018-10-11 16:33:05 -0700758void usb_init() {
Josh Gao8038e352019-03-18 16:33:18 -0700759 bool use_nonblocking = android::base::GetBoolProperty(
760 "persist.adb.nonblocking_ffs",
761 android::base::GetBoolProperty("ro.adb.nonblocking_ffs", true));
762
Josh Gao02e94a42019-02-28 07:26:20 +0000763 if (use_nonblocking) {
Josh Gao0d780392019-02-26 22:10:33 +0000764 std::thread(usb_ffs_open_thread).detach();
Josh Gao02e94a42019-02-28 07:26:20 +0000765 } else {
766 usb_init_legacy();
Josh Gaoc51726c2018-10-11 16:33:05 -0700767 }
768}