blob: e6a19a52154bca6b6ae92e9c5f97bc5d4ad63a23 [file] [log] [blame]
Josh Gaoc51726c2018-10-11 16:33:05 -07001/*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define TRACE_TAG USB
18
19#include "sysdeps.h"
20
21#include <errno.h>
22#include <stdio.h>
23#include <stdlib.h>
24#include <string.h>
25#include <sys/ioctl.h>
26#include <sys/types.h>
27#include <unistd.h>
28
29#include <linux/usb/functionfs.h>
30#include <sys/eventfd.h>
31
Josh Gao86b33be2019-02-26 17:53:52 -080032#include <algorithm>
Josh Gaoc51726c2018-10-11 16:33:05 -070033#include <array>
34#include <future>
35#include <memory>
36#include <mutex>
37#include <optional>
38#include <vector>
39
40#include <asyncio/AsyncIO.h>
41
42#include <android-base/logging.h>
43#include <android-base/macros.h>
44#include <android-base/properties.h>
45#include <android-base/thread_annotations.h>
46
47#include <adbd/usb.h>
48
49#include "adb_unique_fd.h"
50#include "adb_utils.h"
51#include "sysdeps/chrono.h"
52#include "transport.h"
53#include "types.h"
54
55using android::base::StringPrintf;
56
Josh Gaoc0b831b2019-02-13 15:27:28 -080057// We can't find out whether we have support for AIO on ffs endpoints until we submit a read.
58static std::optional<bool> gFfsAioSupported;
59
Josh Gao08ccc732019-04-16 11:20:04 -070060// Not all USB controllers support operations larger than 16k, so don't go above that.
Josh Gao5841a962019-02-28 15:44:05 -080061static constexpr size_t kUsbReadQueueDepth = 32;
Josh Gao08ccc732019-04-16 11:20:04 -070062static constexpr size_t kUsbReadSize = 4 * PAGE_SIZE;
Josh Gaoc51726c2018-10-11 16:33:05 -070063
Josh Gao5841a962019-02-28 15:44:05 -080064static constexpr size_t kUsbWriteQueueDepth = 32;
Josh Gao08ccc732019-04-16 11:20:04 -070065static constexpr size_t kUsbWriteSize = 4 * PAGE_SIZE;
Josh Gaoc51726c2018-10-11 16:33:05 -070066
67static const char* to_string(enum usb_functionfs_event_type type) {
68 switch (type) {
69 case FUNCTIONFS_BIND:
70 return "FUNCTIONFS_BIND";
71 case FUNCTIONFS_UNBIND:
72 return "FUNCTIONFS_UNBIND";
73 case FUNCTIONFS_ENABLE:
74 return "FUNCTIONFS_ENABLE";
75 case FUNCTIONFS_DISABLE:
76 return "FUNCTIONFS_DISABLE";
77 case FUNCTIONFS_SETUP:
78 return "FUNCTIONFS_SETUP";
79 case FUNCTIONFS_SUSPEND:
80 return "FUNCTIONFS_SUSPEND";
81 case FUNCTIONFS_RESUME:
82 return "FUNCTIONFS_RESUME";
83 }
84}
85
86enum class TransferDirection : uint64_t {
87 READ = 0,
88 WRITE = 1,
89};
90
91struct TransferId {
92 TransferDirection direction : 1;
93 uint64_t id : 63;
94
95 TransferId() : TransferId(TransferDirection::READ, 0) {}
96
97 private:
98 TransferId(TransferDirection direction, uint64_t id) : direction(direction), id(id) {}
99
100 public:
101 explicit operator uint64_t() const {
102 uint64_t result;
103 static_assert(sizeof(*this) == sizeof(result));
104 memcpy(&result, this, sizeof(*this));
105 return result;
106 }
107
108 static TransferId read(uint64_t id) { return TransferId(TransferDirection::READ, id); }
109 static TransferId write(uint64_t id) { return TransferId(TransferDirection::WRITE, id); }
110
111 static TransferId from_value(uint64_t value) {
112 TransferId result;
113 memcpy(&result, &value, sizeof(value));
114 return result;
115 }
116};
117
118struct IoBlock {
Josh Gaob0195742019-03-18 14:11:28 -0700119 bool pending = false;
Josh Gaoc51726c2018-10-11 16:33:05 -0700120 struct iocb control;
Josh Gao86b33be2019-02-26 17:53:52 -0800121 std::shared_ptr<Block> payload;
Josh Gaoc51726c2018-10-11 16:33:05 -0700122
123 TransferId id() const { return TransferId::from_value(control.aio_data); }
124};
125
126struct ScopedAioContext {
127 ScopedAioContext() = default;
128 ~ScopedAioContext() { reset(); }
129
130 ScopedAioContext(ScopedAioContext&& move) { reset(move.release()); }
131 ScopedAioContext(const ScopedAioContext& copy) = delete;
132
133 ScopedAioContext& operator=(ScopedAioContext&& move) {
134 reset(move.release());
135 return *this;
136 }
137 ScopedAioContext& operator=(const ScopedAioContext& copy) = delete;
138
139 static ScopedAioContext Create(size_t max_events) {
140 aio_context_t ctx = 0;
141 if (io_setup(max_events, &ctx) != 0) {
142 PLOG(FATAL) << "failed to create aio_context_t";
143 }
144 ScopedAioContext result;
145 result.reset(ctx);
146 return result;
147 }
148
149 aio_context_t release() {
150 aio_context_t result = context_;
151 context_ = 0;
152 return result;
153 }
154
155 void reset(aio_context_t new_context = 0) {
156 if (context_ != 0) {
157 io_destroy(context_);
158 }
159
160 context_ = new_context;
161 }
162
163 aio_context_t get() { return context_; }
164
165 private:
166 aio_context_t context_ = 0;
167};
168
169struct UsbFfsConnection : public Connection {
170 UsbFfsConnection(unique_fd control, unique_fd read, unique_fd write,
171 std::promise<void> destruction_notifier)
Josh Gao19dc2962019-03-26 18:47:45 -0700172 : worker_started_(false),
173 stopped_(false),
Josh Gaoc51726c2018-10-11 16:33:05 -0700174 destruction_notifier_(std::move(destruction_notifier)),
175 control_fd_(std::move(control)),
176 read_fd_(std::move(read)),
177 write_fd_(std::move(write)) {
178 LOG(INFO) << "UsbFfsConnection constructed";
Josh Gaoc0b831b2019-02-13 15:27:28 -0800179 worker_event_fd_.reset(eventfd(0, EFD_CLOEXEC));
180 if (worker_event_fd_ == -1) {
181 PLOG(FATAL) << "failed to create eventfd";
182 }
183
184 monitor_event_fd_.reset(eventfd(0, EFD_CLOEXEC));
185 if (monitor_event_fd_ == -1) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700186 PLOG(FATAL) << "failed to create eventfd";
187 }
188
189 aio_context_ = ScopedAioContext::Create(kUsbReadQueueDepth + kUsbWriteQueueDepth);
190 }
191
192 ~UsbFfsConnection() {
193 LOG(INFO) << "UsbFfsConnection being destroyed";
194 Stop();
195 monitor_thread_.join();
Josh Gaoc0b831b2019-02-13 15:27:28 -0800196
197 // We need to explicitly close our file descriptors before we notify our destruction,
198 // because the thread listening on the future will immediately try to reopen the endpoint.
Josh Gao19dc2962019-03-26 18:47:45 -0700199 aio_context_.reset();
Josh Gaoc0b831b2019-02-13 15:27:28 -0800200 control_fd_.reset();
201 read_fd_.reset();
202 write_fd_.reset();
203
Josh Gaoc51726c2018-10-11 16:33:05 -0700204 destruction_notifier_.set_value();
205 }
206
207 virtual bool Write(std::unique_ptr<apacket> packet) override final {
208 LOG(DEBUG) << "USB write: " << dump_header(&packet->msg);
209 Block header(sizeof(packet->msg));
210 memcpy(header.data(), &packet->msg, sizeof(packet->msg));
211
212 std::lock_guard<std::mutex> lock(write_mutex_);
213 write_requests_.push_back(CreateWriteBlock(std::move(header), next_write_id_++));
214 if (!packet->payload.empty()) {
Josh Gao86b33be2019-02-26 17:53:52 -0800215 // The kernel attempts to allocate a contiguous block of memory for each write,
216 // which can fail if the write is large and the kernel heap is fragmented.
217 // Split large writes into smaller chunks to avoid this.
218 std::shared_ptr<Block> payload = std::make_shared<Block>(std::move(packet->payload));
219 size_t offset = 0;
220 size_t len = payload->size();
221
222 while (len > 0) {
223 size_t write_size = std::min(kUsbWriteSize, len);
224 write_requests_.push_back(
225 CreateWriteBlock(payload, offset, write_size, next_write_id_++));
226 len -= write_size;
227 offset += write_size;
228 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700229 }
230 SubmitWrites();
231 return true;
232 }
233
234 virtual void Start() override final { StartMonitor(); }
235
236 virtual void Stop() override final {
237 if (stopped_.exchange(true)) {
238 return;
239 }
240 stopped_ = true;
241 uint64_t notify = 1;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800242 ssize_t rc = adb_write(worker_event_fd_.get(), &notify, sizeof(notify));
Josh Gaoc51726c2018-10-11 16:33:05 -0700243 if (rc < 0) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800244 PLOG(FATAL) << "failed to notify worker eventfd to stop UsbFfsConnection";
Josh Gaoc51726c2018-10-11 16:33:05 -0700245 }
246 CHECK_EQ(static_cast<size_t>(rc), sizeof(notify));
Josh Gaoc0b831b2019-02-13 15:27:28 -0800247
248 rc = adb_write(monitor_event_fd_.get(), &notify, sizeof(notify));
249 if (rc < 0) {
250 PLOG(FATAL) << "failed to notify monitor eventfd to stop UsbFfsConnection";
251 }
252
253 CHECK_EQ(static_cast<size_t>(rc), sizeof(notify));
Josh Gaoc51726c2018-10-11 16:33:05 -0700254 }
255
256 private:
257 void StartMonitor() {
258 // This is a bit of a mess.
259 // It's possible for io_submit to end up blocking, if we call it as the endpoint
260 // becomes disabled. Work around this by having a monitor thread to listen for functionfs
261 // lifecycle events. If we notice an error condition (either we've become disabled, or we
262 // were never enabled in the first place), we send interruption signals to the worker thread
263 // until it dies, and then report failure to the transport via HandleError, which will
264 // eventually result in the transport being destroyed, which will result in UsbFfsConnection
265 // being destroyed, which unblocks the open thread and restarts this entire process.
Josh Gaoc51726c2018-10-11 16:33:05 -0700266 static std::once_flag handler_once;
267 std::call_once(handler_once, []() { signal(kInterruptionSignal, [](int) {}); });
268
269 monitor_thread_ = std::thread([this]() {
270 adb_thread_setname("UsbFfs-monitor");
271
272 bool bound = false;
Josh Gao6933d542019-03-26 13:21:42 -0700273 bool enabled = false;
Josh Gaoc51726c2018-10-11 16:33:05 -0700274 bool running = true;
275 while (running) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800276 adb_pollfd pfd[2] = {
277 { .fd = control_fd_.get(), .events = POLLIN, .revents = 0 },
278 { .fd = monitor_event_fd_.get(), .events = POLLIN, .revents = 0 },
279 };
Josh Gao19dc2962019-03-26 18:47:45 -0700280
281 // If we don't see our first bind within a second, try again.
282 int timeout_ms = bound ? -1 : 1000;
283
284 int rc = TEMP_FAILURE_RETRY(adb_poll(pfd, 2, timeout_ms));
Josh Gaoc0b831b2019-02-13 15:27:28 -0800285 if (rc == -1) {
286 PLOG(FATAL) << "poll on USB control fd failed";
287 } else if (rc == 0) {
Josh Gao19dc2962019-03-26 18:47:45 -0700288 LOG(WARNING) << "timed out while waiting for FUNCTIONFS_BIND, trying again";
289 break;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800290 }
291
292 if (pfd[1].revents) {
293 // We were told to die.
294 break;
Josh Gaoc51726c2018-10-11 16:33:05 -0700295 }
296
297 struct usb_functionfs_event event;
298 if (TEMP_FAILURE_RETRY(adb_read(control_fd_.get(), &event, sizeof(event))) !=
299 sizeof(event)) {
300 PLOG(FATAL) << "failed to read functionfs event";
301 }
302
303 LOG(INFO) << "USB event: "
304 << to_string(static_cast<usb_functionfs_event_type>(event.type));
305
306 switch (event.type) {
307 case FUNCTIONFS_BIND:
Josh Gao87afd522019-03-28 11:05:53 -0700308 if (bound) {
309 LOG(WARNING) << "received FUNCTIONFS_BIND while already bound?";
310 running = false;
Josh Gao94fb36b2019-05-01 16:53:53 -0700311 break;
Josh Gao87afd522019-03-28 11:05:53 -0700312 }
Josh Gao6933d542019-03-26 13:21:42 -0700313
Josh Gao87afd522019-03-28 11:05:53 -0700314 if (enabled) {
315 LOG(WARNING) << "received FUNCTIONFS_BIND while already enabled?";
316 running = false;
Josh Gao94fb36b2019-05-01 16:53:53 -0700317 break;
Josh Gao87afd522019-03-28 11:05:53 -0700318 }
319
320 bound = true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700321 break;
322
323 case FUNCTIONFS_ENABLE:
Josh Gao87afd522019-03-28 11:05:53 -0700324 if (!bound) {
325 LOG(WARNING) << "received FUNCTIONFS_ENABLE while not bound?";
326 running = false;
Josh Gao94fb36b2019-05-01 16:53:53 -0700327 break;
Josh Gao87afd522019-03-28 11:05:53 -0700328 }
Josh Gao6933d542019-03-26 13:21:42 -0700329
Josh Gao87afd522019-03-28 11:05:53 -0700330 if (enabled) {
331 LOG(WARNING) << "received FUNCTIONFS_ENABLE while already enabled?";
332 running = false;
Josh Gao94fb36b2019-05-01 16:53:53 -0700333 break;
Josh Gao87afd522019-03-28 11:05:53 -0700334 }
335
336 enabled = true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700337 StartWorker();
338 break;
339
340 case FUNCTIONFS_DISABLE:
Josh Gao87afd522019-03-28 11:05:53 -0700341 if (!bound) {
342 LOG(WARNING) << "received FUNCTIONFS_DISABLE while not bound?";
343 }
Josh Gao6933d542019-03-26 13:21:42 -0700344
Josh Gao87afd522019-03-28 11:05:53 -0700345 if (!enabled) {
346 LOG(WARNING) << "received FUNCTIONFS_DISABLE while not enabled?";
347 }
348
349 enabled = false;
Josh Gao6933d542019-03-26 13:21:42 -0700350 running = false;
351 break;
352
353 case FUNCTIONFS_UNBIND:
Josh Gao87afd522019-03-28 11:05:53 -0700354 if (enabled) {
355 LOG(WARNING) << "received FUNCTIONFS_UNBIND while still enabled?";
356 }
Josh Gao6933d542019-03-26 13:21:42 -0700357
Josh Gao87afd522019-03-28 11:05:53 -0700358 if (!bound) {
359 LOG(WARNING) << "received FUNCTIONFS_UNBIND when not bound?";
360 }
361
362 bound = false;
Josh Gaoc51726c2018-10-11 16:33:05 -0700363 running = false;
364 break;
365 }
366 }
367
Josh Gaoe778b3a2019-02-28 13:29:32 -0800368 StopWorker();
Josh Gao19dc2962019-03-26 18:47:45 -0700369 HandleError("monitor thread finished");
Josh Gaoc51726c2018-10-11 16:33:05 -0700370 });
371 }
372
373 void StartWorker() {
Josh Gao19dc2962019-03-26 18:47:45 -0700374 CHECK(!worker_started_);
375 worker_started_ = true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700376 worker_thread_ = std::thread([this]() {
377 adb_thread_setname("UsbFfs-worker");
378 for (size_t i = 0; i < kUsbReadQueueDepth; ++i) {
379 read_requests_[i] = CreateReadBlock(next_read_id_++);
Josh Gaoc0b831b2019-02-13 15:27:28 -0800380 if (!SubmitRead(&read_requests_[i])) {
381 return;
382 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700383 }
384
385 while (!stopped_) {
386 uint64_t dummy;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800387 ssize_t rc = adb_read(worker_event_fd_.get(), &dummy, sizeof(dummy));
Josh Gaoc51726c2018-10-11 16:33:05 -0700388 if (rc == -1) {
389 PLOG(FATAL) << "failed to read from eventfd";
390 } else if (rc == 0) {
391 LOG(FATAL) << "hit EOF on eventfd";
392 }
393
Josh Gao6933d542019-03-26 13:21:42 -0700394 ReadEvents();
Josh Gaoc51726c2018-10-11 16:33:05 -0700395 }
396 });
397 }
398
Josh Gaoe778b3a2019-02-28 13:29:32 -0800399 void StopWorker() {
Josh Gao19dc2962019-03-26 18:47:45 -0700400 if (!worker_started_) {
401 return;
402 }
403
Josh Gaoe778b3a2019-02-28 13:29:32 -0800404 pthread_t worker_thread_handle = worker_thread_.native_handle();
405 while (true) {
406 int rc = pthread_kill(worker_thread_handle, kInterruptionSignal);
407 if (rc != 0) {
408 LOG(ERROR) << "failed to send interruption signal to worker: " << strerror(rc);
409 break;
410 }
411
412 std::this_thread::sleep_for(100ms);
413
414 rc = pthread_kill(worker_thread_handle, 0);
415 if (rc == 0) {
416 continue;
417 } else if (rc == ESRCH) {
418 break;
419 } else {
420 LOG(ERROR) << "failed to send interruption signal to worker: " << strerror(rc);
421 }
422 }
423
424 worker_thread_.join();
425 }
426
Josh Gaoc51726c2018-10-11 16:33:05 -0700427 void PrepareReadBlock(IoBlock* block, uint64_t id) {
428 block->pending = false;
Josh Gao86b33be2019-02-26 17:53:52 -0800429 block->payload = std::make_shared<Block>(kUsbReadSize);
Josh Gaoc51726c2018-10-11 16:33:05 -0700430 block->control.aio_data = static_cast<uint64_t>(TransferId::read(id));
Josh Gao86b33be2019-02-26 17:53:52 -0800431 block->control.aio_buf = reinterpret_cast<uintptr_t>(block->payload->data());
432 block->control.aio_nbytes = block->payload->size();
Josh Gaoc51726c2018-10-11 16:33:05 -0700433 }
434
435 IoBlock CreateReadBlock(uint64_t id) {
436 IoBlock block;
437 PrepareReadBlock(&block, id);
438 block.control.aio_rw_flags = 0;
439 block.control.aio_lio_opcode = IOCB_CMD_PREAD;
440 block.control.aio_reqprio = 0;
441 block.control.aio_fildes = read_fd_.get();
442 block.control.aio_offset = 0;
443 block.control.aio_flags = IOCB_FLAG_RESFD;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800444 block.control.aio_resfd = worker_event_fd_.get();
Josh Gaoc51726c2018-10-11 16:33:05 -0700445 return block;
446 }
447
Josh Gao6933d542019-03-26 13:21:42 -0700448 void ReadEvents() {
Josh Gaoc51726c2018-10-11 16:33:05 -0700449 static constexpr size_t kMaxEvents = kUsbReadQueueDepth + kUsbWriteQueueDepth;
450 struct io_event events[kMaxEvents];
451 struct timespec timeout = {.tv_sec = 0, .tv_nsec = 0};
452 int rc = io_getevents(aio_context_.get(), 0, kMaxEvents, events, &timeout);
453 if (rc == -1) {
454 HandleError(StringPrintf("io_getevents failed while reading: %s", strerror(errno)));
455 return;
456 }
457
458 for (int event_idx = 0; event_idx < rc; ++event_idx) {
459 auto& event = events[event_idx];
460 TransferId id = TransferId::from_value(event.data);
461
462 if (event.res < 0) {
463 std::string error =
464 StringPrintf("%s %" PRIu64 " failed with error %s",
465 id.direction == TransferDirection::READ ? "read" : "write",
466 id.id, strerror(-event.res));
467 HandleError(error);
468 return;
469 }
470
471 if (id.direction == TransferDirection::READ) {
472 HandleRead(id, event.res);
473 } else {
474 HandleWrite(id);
475 }
476 }
477 }
478
479 void HandleRead(TransferId id, int64_t size) {
480 uint64_t read_idx = id.id % kUsbReadQueueDepth;
481 IoBlock* block = &read_requests_[read_idx];
482 block->pending = false;
Josh Gao86b33be2019-02-26 17:53:52 -0800483 block->payload->resize(size);
Josh Gaoc51726c2018-10-11 16:33:05 -0700484
485 // Notification for completed reads can be received out of order.
486 if (block->id().id != needed_read_id_) {
487 LOG(VERBOSE) << "read " << block->id().id << " completed while waiting for "
488 << needed_read_id_;
489 return;
490 }
491
492 for (uint64_t id = needed_read_id_;; ++id) {
493 size_t read_idx = id % kUsbReadQueueDepth;
494 IoBlock* current_block = &read_requests_[read_idx];
495 if (current_block->pending) {
496 break;
497 }
498 ProcessRead(current_block);
499 ++needed_read_id_;
500 }
501 }
502
503 void ProcessRead(IoBlock* block) {
Josh Gao86b33be2019-02-26 17:53:52 -0800504 if (!block->payload->empty()) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700505 if (!incoming_header_.has_value()) {
Josh Gao86b33be2019-02-26 17:53:52 -0800506 CHECK_EQ(sizeof(amessage), block->payload->size());
Josh Gaoc51726c2018-10-11 16:33:05 -0700507 amessage msg;
Josh Gao86b33be2019-02-26 17:53:52 -0800508 memcpy(&msg, block->payload->data(), sizeof(amessage));
Josh Gaoc51726c2018-10-11 16:33:05 -0700509 LOG(DEBUG) << "USB read:" << dump_header(&msg);
510 incoming_header_ = msg;
511 } else {
512 size_t bytes_left = incoming_header_->data_length - incoming_payload_.size();
Josh Gao86b33be2019-02-26 17:53:52 -0800513 Block payload = std::move(*block->payload);
Josh Gaoc51726c2018-10-11 16:33:05 -0700514 CHECK_LE(payload.size(), bytes_left);
515 incoming_payload_.append(std::make_unique<Block>(std::move(payload)));
516 }
517
518 if (incoming_header_->data_length == incoming_payload_.size()) {
519 auto packet = std::make_unique<apacket>();
520 packet->msg = *incoming_header_;
521
522 // TODO: Make apacket contain an IOVector so we don't have to coalesce.
523 packet->payload = incoming_payload_.coalesce();
524 read_callback_(this, std::move(packet));
525
526 incoming_header_.reset();
527 incoming_payload_.clear();
528 }
529 }
530
531 PrepareReadBlock(block, block->id().id + kUsbReadQueueDepth);
532 SubmitRead(block);
533 }
534
Josh Gaoc0b831b2019-02-13 15:27:28 -0800535 bool SubmitRead(IoBlock* block) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700536 block->pending = true;
537 struct iocb* iocb = &block->control;
538 if (io_submit(aio_context_.get(), 1, &iocb) != 1) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800539 if (errno == EINVAL && !gFfsAioSupported.has_value()) {
540 HandleError("failed to submit first read, AIO on FFS not supported");
541 gFfsAioSupported = false;
542 return false;
543 }
544
Josh Gaoc51726c2018-10-11 16:33:05 -0700545 HandleError(StringPrintf("failed to submit read: %s", strerror(errno)));
Josh Gaoc0b831b2019-02-13 15:27:28 -0800546 return false;
Josh Gaoc51726c2018-10-11 16:33:05 -0700547 }
Josh Gaoc0b831b2019-02-13 15:27:28 -0800548
549 gFfsAioSupported = true;
550 return true;
Josh Gaoc51726c2018-10-11 16:33:05 -0700551 }
552
553 void HandleWrite(TransferId id) {
554 std::lock_guard<std::mutex> lock(write_mutex_);
555 auto it =
556 std::find_if(write_requests_.begin(), write_requests_.end(), [id](const auto& req) {
557 return static_cast<uint64_t>(req->id()) == static_cast<uint64_t>(id);
558 });
559 CHECK(it != write_requests_.end());
560
561 write_requests_.erase(it);
562 size_t outstanding_writes = --writes_submitted_;
563 LOG(DEBUG) << "USB write: reaped, down to " << outstanding_writes;
564
565 SubmitWrites();
566 }
567
Josh Gao86b33be2019-02-26 17:53:52 -0800568 std::unique_ptr<IoBlock> CreateWriteBlock(std::shared_ptr<Block> payload, size_t offset,
569 size_t len, uint64_t id) {
Josh Gaoc51726c2018-10-11 16:33:05 -0700570 auto block = std::make_unique<IoBlock>();
571 block->payload = std::move(payload);
572 block->control.aio_data = static_cast<uint64_t>(TransferId::write(id));
573 block->control.aio_rw_flags = 0;
574 block->control.aio_lio_opcode = IOCB_CMD_PWRITE;
575 block->control.aio_reqprio = 0;
576 block->control.aio_fildes = write_fd_.get();
Josh Gao86b33be2019-02-26 17:53:52 -0800577 block->control.aio_buf = reinterpret_cast<uintptr_t>(block->payload->data() + offset);
578 block->control.aio_nbytes = len;
Josh Gaoc51726c2018-10-11 16:33:05 -0700579 block->control.aio_offset = 0;
580 block->control.aio_flags = IOCB_FLAG_RESFD;
Josh Gaoc0b831b2019-02-13 15:27:28 -0800581 block->control.aio_resfd = worker_event_fd_.get();
Josh Gaoc51726c2018-10-11 16:33:05 -0700582 return block;
583 }
584
Josh Gao86b33be2019-02-26 17:53:52 -0800585 std::unique_ptr<IoBlock> CreateWriteBlock(Block payload, uint64_t id) {
586 std::shared_ptr<Block> block = std::make_shared<Block>(std::move(payload));
587 size_t len = block->size();
588 return CreateWriteBlock(std::move(block), 0, len, id);
589 }
590
Josh Gaoc51726c2018-10-11 16:33:05 -0700591 void SubmitWrites() REQUIRES(write_mutex_) {
592 if (writes_submitted_ == kUsbWriteQueueDepth) {
593 return;
594 }
595
596 ssize_t writes_to_submit = std::min(kUsbWriteQueueDepth - writes_submitted_,
597 write_requests_.size() - writes_submitted_);
598 CHECK_GE(writes_to_submit, 0);
599 if (writes_to_submit == 0) {
600 return;
601 }
602
603 struct iocb* iocbs[kUsbWriteQueueDepth];
604 for (int i = 0; i < writes_to_submit; ++i) {
605 CHECK(!write_requests_[writes_submitted_ + i]->pending);
606 write_requests_[writes_submitted_ + i]->pending = true;
607 iocbs[i] = &write_requests_[writes_submitted_ + i]->control;
608 LOG(VERBOSE) << "submitting write_request " << static_cast<void*>(iocbs[i]);
609 }
610
Josh Gao63b52ec2019-03-26 13:06:38 -0700611 writes_submitted_ += writes_to_submit;
612
Josh Gaoc51726c2018-10-11 16:33:05 -0700613 int rc = io_submit(aio_context_.get(), writes_to_submit, iocbs);
614 if (rc == -1) {
615 HandleError(StringPrintf("failed to submit write requests: %s", strerror(errno)));
616 return;
617 } else if (rc != writes_to_submit) {
618 LOG(FATAL) << "failed to submit all writes: wanted to submit " << writes_to_submit
619 << ", actually submitted " << rc;
620 }
Josh Gaoc51726c2018-10-11 16:33:05 -0700621 }
622
623 void HandleError(const std::string& error) {
624 std::call_once(error_flag_, [&]() {
625 error_callback_(this, error);
626 if (!stopped_) {
627 Stop();
628 }
629 });
630 }
631
632 std::thread monitor_thread_;
Josh Gao19dc2962019-03-26 18:47:45 -0700633
634 bool worker_started_;
Josh Gaoc51726c2018-10-11 16:33:05 -0700635 std::thread worker_thread_;
636
637 std::atomic<bool> stopped_;
638 std::promise<void> destruction_notifier_;
639 std::once_flag error_flag_;
640
Josh Gaoc0b831b2019-02-13 15:27:28 -0800641 unique_fd worker_event_fd_;
642 unique_fd monitor_event_fd_;
Josh Gaoc51726c2018-10-11 16:33:05 -0700643
644 ScopedAioContext aio_context_;
645 unique_fd control_fd_;
646 unique_fd read_fd_;
647 unique_fd write_fd_;
648
649 std::optional<amessage> incoming_header_;
650 IOVector incoming_payload_;
651
652 std::array<IoBlock, kUsbReadQueueDepth> read_requests_;
653 IOVector read_data_;
654
655 // ID of the next request that we're going to send out.
656 size_t next_read_id_ = 0;
657
658 // ID of the next packet we're waiting for.
659 size_t needed_read_id_ = 0;
660
661 std::mutex write_mutex_;
662 std::deque<std::unique_ptr<IoBlock>> write_requests_ GUARDED_BY(write_mutex_);
663 size_t next_write_id_ GUARDED_BY(write_mutex_) = 0;
664 size_t writes_submitted_ GUARDED_BY(write_mutex_) = 0;
Josh Gaoe778b3a2019-02-28 13:29:32 -0800665
666 static constexpr int kInterruptionSignal = SIGUSR1;
Josh Gaoc51726c2018-10-11 16:33:05 -0700667};
668
Josh Gaoc0b831b2019-02-13 15:27:28 -0800669void usb_init_legacy();
670
Josh Gaoc51726c2018-10-11 16:33:05 -0700671static void usb_ffs_open_thread() {
672 adb_thread_setname("usb ffs open");
673
674 while (true) {
Josh Gaoc0b831b2019-02-13 15:27:28 -0800675 if (gFfsAioSupported.has_value() && !gFfsAioSupported.value()) {
676 LOG(INFO) << "failed to use nonblocking ffs, falling back to legacy";
677 return usb_init_legacy();
678 }
679
Josh Gaoc51726c2018-10-11 16:33:05 -0700680 unique_fd control;
681 unique_fd bulk_out;
682 unique_fd bulk_in;
683 if (!open_functionfs(&control, &bulk_out, &bulk_in)) {
684 std::this_thread::sleep_for(1s);
685 continue;
686 }
687
688 atransport* transport = new atransport();
689 transport->serial = "UsbFfs";
690 std::promise<void> destruction_notifier;
691 std::future<void> future = destruction_notifier.get_future();
692 transport->SetConnection(std::make_unique<UsbFfsConnection>(
693 std::move(control), std::move(bulk_out), std::move(bulk_in),
694 std::move(destruction_notifier)));
695 register_transport(transport);
696 future.wait();
697 }
698}
699
Josh Gaoc51726c2018-10-11 16:33:05 -0700700void usb_init() {
Josh Gao8038e352019-03-18 16:33:18 -0700701 bool use_nonblocking = android::base::GetBoolProperty(
702 "persist.adb.nonblocking_ffs",
703 android::base::GetBoolProperty("ro.adb.nonblocking_ffs", true));
704
Josh Gao02e94a42019-02-28 07:26:20 +0000705 if (use_nonblocking) {
Josh Gao0d780392019-02-26 22:10:33 +0000706 std::thread(usb_ffs_open_thread).detach();
Josh Gao02e94a42019-02-28 07:26:20 +0000707 } else {
708 usb_init_legacy();
Josh Gaoc51726c2018-10-11 16:33:05 -0700709 }
710}