blob: 7a17f257c2ab4d3f31e264045e0e4b5aaf9ce5d2 [file] [log] [blame]
Michael Butlerf6b2d1a2020-12-19 14:44:35 -08001/*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "ExecutionBurstController"
18
19#include "ExecutionBurstController.h"
Michael Butler76e491f2020-12-19 01:55:32 -080020#include "ExecutionBurstUtils.h"
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080021
22#include <android-base/logging.h>
Michael Butler76e491f2020-12-19 01:55:32 -080023#include <android-base/thread_annotations.h>
24#include <nnapi/IBurst.h>
25#include <nnapi/IPreparedModel.h>
26#include <nnapi/Result.h>
27#include <nnapi/TypeUtils.h>
28#include <nnapi/Types.h>
29#include <nnapi/Validation.h>
30#include <nnapi/hal/1.0/Conversions.h>
31#include <nnapi/hal/HandleError.h>
32#include <nnapi/hal/ProtectCallback.h>
33#include <nnapi/hal/TransferValue.h>
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080034
35#include <algorithm>
36#include <cstring>
37#include <limits>
38#include <memory>
39#include <string>
Michael Butler76e491f2020-12-19 01:55:32 -080040#include <thread>
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080041#include <tuple>
42#include <utility>
43#include <vector>
44
Michael Butler76e491f2020-12-19 01:55:32 -080045#include "Callbacks.h"
46#include "Conversions.h"
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080047#include "Tracing.h"
48#include "Utils.h"
49
Michael Butler76e491f2020-12-19 01:55:32 -080050namespace android::hardware::neuralnetworks::V1_2::utils {
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080051namespace {
52
Michael Butler76e491f2020-12-19 01:55:32 -080053nn::GeneralResult<sp<IBurstContext>> executionBurstResultCallback(
54 V1_0::ErrorStatus status, const sp<IBurstContext>& burstContext) {
55 HANDLE_HAL_STATUS(status) << "IPreparedModel::configureExecutionBurst failed with status "
56 << toString(status);
57 if (burstContext == nullptr) {
58 return NN_ERROR(nn::ErrorStatus::GENERAL_FAILURE)
59 << "IPreparedModel::configureExecutionBurst returned nullptr for burst";
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080060 }
Michael Butler76e491f2020-12-19 01:55:32 -080061 return burstContext;
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080062}
63
Michael Butler76e491f2020-12-19 01:55:32 -080064nn::GeneralResult<hidl_vec<hidl_memory>> getMemoriesHelper(
65 const hidl_vec<int32_t>& slots,
66 const std::shared_ptr<ExecutionBurstController::MemoryCache>& memoryCache) {
67 hidl_vec<hidl_memory> memories(slots.size());
68 for (size_t i = 0; i < slots.size(); ++i) {
69 const int32_t slot = slots[i];
70 const auto memory = NN_TRY(memoryCache->getMemory(slot));
71 memories[i] = NN_TRY(V1_0::utils::unvalidatedConvert(memory));
72 if (!memories[i].valid()) {
73 return NN_ERROR() << "memory at slot " << slot << " is invalid";
74 }
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080075 }
Michael Butler76e491f2020-12-19 01:55:32 -080076 return memories;
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080077}
78
Michael Butler76e491f2020-12-19 01:55:32 -080079} // namespace
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080080
Michael Butler76e491f2020-12-19 01:55:32 -080081// MemoryCache methods
82
83ExecutionBurstController::MemoryCache::MemoryCache() {
84 constexpr size_t kPreallocatedCount = 1024;
85 std::vector<int32_t> freeSlotsSpace;
86 freeSlotsSpace.reserve(kPreallocatedCount);
87 mFreeSlots = std::stack<int32_t, std::vector<int32_t>>(std::move(freeSlotsSpace));
88 mMemoryCache.reserve(kPreallocatedCount);
89 mCacheCleaner.reserve(kPreallocatedCount);
Michael Butlerf6b2d1a2020-12-19 14:44:35 -080090}
91
Michael Butler76e491f2020-12-19 01:55:32 -080092void ExecutionBurstController::MemoryCache::setBurstContext(sp<IBurstContext> burstContext) {
93 std::lock_guard guard(mMutex);
94 mBurstContext = std::move(burstContext);
95}
96
97std::pair<int32_t, ExecutionBurstController::MemoryCache::SharedCleanup>
98ExecutionBurstController::MemoryCache::cacheMemory(const nn::SharedMemory& memory) {
99 std::unique_lock lock(mMutex);
100 base::ScopedLockAssertion lockAssert(mMutex);
101
102 // Use existing cache entry if (1) the Memory object is in the cache and (2) the cache entry is
103 // not currently being freed.
104 auto iter = mMemoryIdToSlot.find(memory);
105 while (iter != mMemoryIdToSlot.end()) {
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800106 const int32_t slot = iter->second;
Michael Butler76e491f2020-12-19 01:55:32 -0800107 if (auto cleaner = mCacheCleaner.at(slot).lock()) {
108 return std::make_pair(slot, std::move(cleaner));
109 }
110
111 // If the code reaches this point, the Memory object was in the cache, but is currently
112 // being destroyed. This code waits until the cache entry has been freed, then loops to
113 // ensure the cache entry has been freed or has been made present by another thread.
114 mCond.wait(lock);
115 iter = mMemoryIdToSlot.find(memory);
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800116 }
Michael Butler76e491f2020-12-19 01:55:32 -0800117
118 // Allocate a new cache entry.
119 const int32_t slot = allocateSlotLocked();
120 mMemoryIdToSlot[memory] = slot;
121 mMemoryCache[slot] = memory;
122
123 // Create reference-counted self-cleaning cache object.
124 auto self = weak_from_this();
125 Task cleanup = [memory, memoryCache = std::move(self)] {
126 if (const auto lock = memoryCache.lock()) {
127 lock->freeMemory(memory);
128 }
129 };
130 auto cleaner = std::make_shared<const Cleanup>(std::move(cleanup));
131 mCacheCleaner[slot] = cleaner;
132
133 return std::make_pair(slot, std::move(cleaner));
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800134}
135
Michael Butler76e491f2020-12-19 01:55:32 -0800136nn::GeneralResult<nn::SharedMemory> ExecutionBurstController::MemoryCache::getMemory(int32_t slot) {
137 std::lock_guard guard(mMutex);
138 if (slot < 0 || static_cast<size_t>(slot) >= mMemoryCache.size()) {
139 return NN_ERROR() << "Invalid slot: " << slot << " vs " << mMemoryCache.size();
140 }
141 return mMemoryCache[slot];
142}
143
144void ExecutionBurstController::MemoryCache::freeMemory(const nn::SharedMemory& memory) {
145 {
146 std::lock_guard guard(mMutex);
147 const int32_t slot = mMemoryIdToSlot.at(memory);
148 if (mBurstContext) {
149 mBurstContext->freeMemory(slot);
150 }
151 mMemoryIdToSlot.erase(memory);
152 mMemoryCache[slot] = {};
153 mCacheCleaner[slot].reset();
154 mFreeSlots.push(slot);
155 }
156 mCond.notify_all();
157}
158
159int32_t ExecutionBurstController::MemoryCache::allocateSlotLocked() {
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800160 constexpr size_t kMaxNumberOfSlots = std::numeric_limits<int32_t>::max();
161
Michael Butler76e491f2020-12-19 01:55:32 -0800162 // If there is a free slot, use it.
163 if (!mFreeSlots.empty()) {
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800164 const int32_t slot = mFreeSlots.top();
165 mFreeSlots.pop();
166 return slot;
167 }
168
Michael Butler76e491f2020-12-19 01:55:32 -0800169 // Use a slot for the first time.
170 CHECK_LT(mMemoryCache.size(), kMaxNumberOfSlots) << "Exceeded maximum number of slots!";
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800171 const int32_t slot = static_cast<int32_t>(mMemoryCache.size());
172 mMemoryCache.emplace_back();
Michael Butler76e491f2020-12-19 01:55:32 -0800173 mCacheCleaner.emplace_back();
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800174
175 return slot;
176}
177
Michael Butler76e491f2020-12-19 01:55:32 -0800178// ExecutionBurstCallback methods
179
180ExecutionBurstController::ExecutionBurstCallback::ExecutionBurstCallback(
181 const std::shared_ptr<MemoryCache>& memoryCache)
182 : kMemoryCache(memoryCache) {
183 CHECK(memoryCache != nullptr);
184}
185
186Return<void> ExecutionBurstController::ExecutionBurstCallback::getMemories(
187 const hidl_vec<int32_t>& slots, getMemories_cb cb) {
188 const auto memoryCache = kMemoryCache.lock();
189 if (memoryCache == nullptr) {
190 LOG(ERROR) << "ExecutionBurstController::ExecutionBurstCallback::getMemories called after "
191 "the MemoryCache has been freed";
192 cb(V1_0::ErrorStatus::GENERAL_FAILURE, {});
193 return Void();
194 }
195
196 const auto maybeMemories = getMemoriesHelper(slots, memoryCache);
197 if (!maybeMemories.has_value()) {
198 const auto& [message, code] = maybeMemories.error();
199 LOG(ERROR) << "ExecutionBurstController::ExecutionBurstCallback::getMemories failed with "
200 << code << ": " << message;
201 cb(V1_0::ErrorStatus::INVALID_ARGUMENT, {});
202 return Void();
203 }
204
205 cb(V1_0::ErrorStatus::NONE, maybeMemories.value());
206 return Void();
207}
208
209// ExecutionBurstController methods
210
211nn::GeneralResult<std::shared_ptr<const ExecutionBurstController>> ExecutionBurstController::create(
212 const sp<V1_2::IPreparedModel>& preparedModel, FallbackFunction fallback,
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800213 std::chrono::microseconds pollingTimeWindow) {
214 // check inputs
215 if (preparedModel == nullptr) {
Michael Butler76e491f2020-12-19 01:55:32 -0800216 return NN_ERROR() << "ExecutionBurstController::create passed a nullptr";
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800217 }
218
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800219 // create FMQ objects
Michael Butler76e491f2020-12-19 01:55:32 -0800220 auto [requestChannelSender, requestChannelDescriptor] =
221 NN_TRY(RequestChannelSender::create(kExecutionBurstChannelLength));
222 auto [resultChannelReceiver, resultChannelDescriptor] =
223 NN_TRY(ResultChannelReceiver::create(kExecutionBurstChannelLength, pollingTimeWindow));
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800224
225 // check FMQ objects
Michael Butler76e491f2020-12-19 01:55:32 -0800226 CHECK(requestChannelSender != nullptr);
227 CHECK(requestChannelDescriptor != nullptr);
228 CHECK(resultChannelReceiver != nullptr);
229 CHECK(resultChannelDescriptor != nullptr);
230
231 // create memory cache
232 auto memoryCache = std::make_shared<MemoryCache>();
233
234 // create callback object
235 auto burstCallback = sp<ExecutionBurstCallback>::make(memoryCache);
236 auto cb = hal::utils::CallbackValue(executionBurstResultCallback);
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800237
238 // configure burst
Michael Butler76e491f2020-12-19 01:55:32 -0800239 const Return<void> ret = preparedModel->configureExecutionBurst(
240 burstCallback, *requestChannelDescriptor, *resultChannelDescriptor, cb);
241 HANDLE_TRANSPORT_FAILURE(ret);
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800242
Michael Butler76e491f2020-12-19 01:55:32 -0800243 auto burstContext = NN_TRY(cb.take());
244 memoryCache->setBurstContext(burstContext);
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800245
246 // create death handler object
Michael Butler76e491f2020-12-19 01:55:32 -0800247 auto deathHandler = NN_TRY(neuralnetworks::utils::DeathHandler::create(burstContext));
248 deathHandler.protectCallbackForLifetimeOfDeathHandler(requestChannelSender.get());
249 deathHandler.protectCallbackForLifetimeOfDeathHandler(resultChannelReceiver.get());
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800250
251 // make and return controller
Michael Butler76e491f2020-12-19 01:55:32 -0800252 return std::make_shared<const ExecutionBurstController>(
253 PrivateConstructorTag{}, std::move(fallback), std::move(requestChannelSender),
254 std::move(resultChannelReceiver), std::move(burstCallback), std::move(burstContext),
255 std::move(memoryCache), std::move(deathHandler));
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800256}
257
258ExecutionBurstController::ExecutionBurstController(
Michael Butler76e491f2020-12-19 01:55:32 -0800259 PrivateConstructorTag /*tag*/, FallbackFunction fallback,
260 std::unique_ptr<RequestChannelSender> requestChannelSender,
261 std::unique_ptr<ResultChannelReceiver> resultChannelReceiver,
262 sp<ExecutionBurstCallback> callback, sp<IBurstContext> burstContext,
263 std::shared_ptr<MemoryCache> memoryCache, neuralnetworks::utils::DeathHandler deathHandler)
264 : kFallback(std::move(fallback)),
265 mRequestChannelSender(std::move(requestChannelSender)),
266 mResultChannelReceiver(std::move(resultChannelReceiver)),
267 mBurstCallback(std::move(callback)),
268 mBurstContext(std::move(burstContext)),
269 mMemoryCache(std::move(memoryCache)),
270 kDeathHandler(std::move(deathHandler)) {}
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800271
Michael Butler76e491f2020-12-19 01:55:32 -0800272ExecutionBurstController::OptionalCacheHold ExecutionBurstController::cacheMemory(
273 const nn::SharedMemory& memory) const {
274 auto [slot, hold] = mMemoryCache->cacheMemory(memory);
275 return hold;
276}
277
278nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>>
Michael Butler8414a6e2021-03-10 18:41:05 -0800279ExecutionBurstController::execute(const nn::Request& request, nn::MeasureTiming measure,
280 const nn::OptionalTimePoint& deadline,
281 const nn::OptionalDuration& loopTimeoutDuration) const {
Michael Butler76e491f2020-12-19 01:55:32 -0800282 // This is the first point when we know an execution is occurring, so begin to collect
283 // systraces. Note that the first point we can begin collecting systraces in
284 // ExecutionBurstServer is when the RequestChannelReceiver realizes there is data in the FMQ, so
285 // ExecutionBurstServer collects systraces at different points in the code.
286 NNTRACE_FULL(NNTRACE_LAYER_IPC, NNTRACE_PHASE_EXECUTION, "ExecutionBurstController::execute");
287
288 // if the request is valid but of a higher version than what's supported in burst execution,
289 // fall back to another execution path
290 if (const auto version = NN_TRY(hal::utils::makeExecutionFailure(nn::validate(request)));
291 version > nn::Version::ANDROID_Q) {
292 // fallback to another execution path if the packet could not be sent
293 if (kFallback) {
Michael Butler8414a6e2021-03-10 18:41:05 -0800294 return kFallback(request, measure, deadline, loopTimeoutDuration);
Michael Butler76e491f2020-12-19 01:55:32 -0800295 }
296 return NN_ERROR() << "Request object has features not supported by IBurst::execute";
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800297 }
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800298
Michael Butler76e491f2020-12-19 01:55:32 -0800299 // clear pools field of request, as they will be provided via slots
300 const auto requestWithoutPools =
301 nn::Request{.inputs = request.inputs, .outputs = request.outputs, .pools = {}};
302 auto hidlRequest = NN_TRY(
303 hal::utils::makeExecutionFailure(V1_0::utils::unvalidatedConvert(requestWithoutPools)));
304 const auto hidlMeasure = NN_TRY(hal::utils::makeExecutionFailure(convert(measure)));
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800305
Michael Butler76e491f2020-12-19 01:55:32 -0800306 // Ensure that at most one execution is in flight at any given time.
307 const bool alreadyInFlight = mExecutionInFlight.test_and_set();
308 if (alreadyInFlight) {
309 return NN_ERROR() << "IBurst already has an execution in flight";
310 }
311 const auto guard = base::make_scope_guard([this] { mExecutionInFlight.clear(); });
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800312
Michael Butler76e491f2020-12-19 01:55:32 -0800313 std::vector<int32_t> slots;
314 std::vector<OptionalCacheHold> holds;
315 slots.reserve(request.pools.size());
316 holds.reserve(request.pools.size());
317 for (const auto& memoryPool : request.pools) {
318 auto [slot, hold] = mMemoryCache->cacheMemory(std::get<nn::SharedMemory>(memoryPool));
319 slots.push_back(slot);
320 holds.push_back(std::move(hold));
321 }
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800322
323 // send request packet
Michael Butler76e491f2020-12-19 01:55:32 -0800324 const auto sendStatus = mRequestChannelSender->send(hidlRequest, hidlMeasure, slots);
325 if (!sendStatus.ok()) {
326 // fallback to another execution path if the packet could not be sent
327 if (kFallback) {
Michael Butler8414a6e2021-03-10 18:41:05 -0800328 return kFallback(request, measure, deadline, loopTimeoutDuration);
Michael Butler76e491f2020-12-19 01:55:32 -0800329 }
330 return NN_ERROR() << "Error sending FMQ packet: " << sendStatus.error();
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800331 }
332
333 // get result packet
Michael Butler76e491f2020-12-19 01:55:32 -0800334 const auto [status, outputShapes, timing] =
335 NN_TRY(hal::utils::makeExecutionFailure(mResultChannelReceiver->getBlocking()));
336 return executionCallback(status, outputShapes, timing);
Michael Butlerf6b2d1a2020-12-19 14:44:35 -0800337}
338
Michael Butler76e491f2020-12-19 01:55:32 -0800339} // namespace android::hardware::neuralnetworks::V1_2::utils