blob: b92f877a513bc37316278abfa9f9cfac643bf129 [file] [log] [blame]
Michael Butler4b276a72020-08-06 23:22:35 -07001/*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "PreparedModel.h"
18
19#include "Callbacks.h"
20#include "Conversions.h"
Xusong Wang5f6bedb2021-03-03 16:20:37 -080021#include "Execution.h"
Michael Butler4b276a72020-08-06 23:22:35 -070022#include "Utils.h"
23
24#include <android/hardware/neuralnetworks/1.0/types.h>
25#include <android/hardware/neuralnetworks/1.1/types.h>
26#include <android/hardware/neuralnetworks/1.2/types.h>
27#include <android/hardware/neuralnetworks/1.3/IPreparedModel.h>
28#include <android/hardware/neuralnetworks/1.3/types.h>
29#include <nnapi/IPreparedModel.h>
30#include <nnapi/Result.h>
Michael Butler6547b2a2020-11-22 19:36:30 -080031#include <nnapi/TypeUtils.h>
Michael Butler4b276a72020-08-06 23:22:35 -070032#include <nnapi/Types.h>
Michael Butler49d95e02021-10-15 18:52:52 -070033#include <nnapi/hal/1.0/HandleError.h>
Michael Butlere8645c32021-10-15 18:42:32 -070034#include <nnapi/hal/1.0/ProtectCallback.h>
Michael Butler137ee992021-11-01 16:40:31 -070035#include <nnapi/hal/1.2/Burst.h>
36#include <nnapi/hal/1.2/BurstUtils.h>
Michael Butler4b276a72020-08-06 23:22:35 -070037#include <nnapi/hal/1.2/Conversions.h>
38#include <nnapi/hal/CommonUtils.h>
Michael Butler4b276a72020-08-06 23:22:35 -070039
40#include <memory>
41#include <tuple>
42#include <utility>
43#include <vector>
44
Michael Butleraad934b2020-12-13 23:06:06 -080045// See hardware/interfaces/neuralnetworks/utils/README.md for more information on HIDL interface
46// lifetimes across processes and for protecting asynchronous calls across HIDL.
47
Michael Butler4b276a72020-08-06 23:22:35 -070048namespace android::hardware::neuralnetworks::V1_3::utils {
49namespace {
50
Michael Butler4b276a72020-08-06 23:22:35 -070051nn::GeneralResult<std::pair<nn::Timing, nn::Timing>> convertFencedExecutionCallbackResults(
Michael Butler7fd03c22020-12-06 21:50:59 -080052 ErrorStatus status, const V1_2::Timing& timingLaunched, const V1_2::Timing& timingFenced) {
Michael Butler49d95e02021-10-15 18:52:52 -070053 HANDLE_STATUS_HIDL(status) << "fenced execution callback info failed with " << toString(status);
Michael Butler6547b2a2020-11-22 19:36:30 -080054 return std::make_pair(NN_TRY(nn::convert(timingLaunched)), NN_TRY(nn::convert(timingFenced)));
Michael Butler4b276a72020-08-06 23:22:35 -070055}
56
Michael Butler7fd03c22020-12-06 21:50:59 -080057nn::GeneralResult<std::pair<nn::SyncFence, nn::ExecuteFencedInfoCallback>> fencedExecutionCallback(
58 ErrorStatus status, const hidl_handle& syncFence,
59 const sp<IFencedExecutionCallback>& callback) {
Michael Butler49d95e02021-10-15 18:52:52 -070060 HANDLE_STATUS_HIDL(status) << "fenced execution failed with " << toString(status);
Michael Butler7fd03c22020-12-06 21:50:59 -080061
Michael Butler4b276a72020-08-06 23:22:35 -070062 auto resultSyncFence = nn::SyncFence::createAsSignaled();
63 if (syncFence.getNativeHandle() != nullptr) {
Michael Butler6547b2a2020-11-22 19:36:30 -080064 auto sharedHandle = NN_TRY(nn::convert(syncFence));
Michael Butlerff9a5a52021-10-15 16:23:20 -070065 resultSyncFence = NN_TRY(nn::SyncFence::create(std::move(sharedHandle)));
Michael Butler4b276a72020-08-06 23:22:35 -070066 }
67
68 if (callback == nullptr) {
69 return NN_ERROR(nn::ErrorStatus::GENERAL_FAILURE) << "callback is null";
70 }
71
72 // Create callback which can be used to retrieve the execution error status and timings.
73 nn::ExecuteFencedInfoCallback resultCallback =
74 [callback]() -> nn::GeneralResult<std::pair<nn::Timing, nn::Timing>> {
Michael Butler7fd03c22020-12-06 21:50:59 -080075 auto cb = hal::utils::CallbackValue(convertFencedExecutionCallbackResults);
Michael Butler4b276a72020-08-06 23:22:35 -070076
77 const auto ret = callback->getExecutionInfo(cb);
Michael Butlercca3e202020-11-22 20:25:34 -080078 HANDLE_TRANSPORT_FAILURE(ret);
Michael Butler4b276a72020-08-06 23:22:35 -070079
Michael Butler7fd03c22020-12-06 21:50:59 -080080 return cb.take();
Michael Butler4b276a72020-08-06 23:22:35 -070081 };
82
83 return std::make_pair(std::move(resultSyncFence), std::move(resultCallback));
84}
85
86} // namespace
87
88nn::GeneralResult<std::shared_ptr<const PreparedModel>> PreparedModel::create(
Michael Butler7fd03c22020-12-06 21:50:59 -080089 sp<V1_3::IPreparedModel> preparedModel, bool executeSynchronously) {
Michael Butler4b276a72020-08-06 23:22:35 -070090 if (preparedModel == nullptr) {
Michael Butler7fd03c22020-12-06 21:50:59 -080091 return NN_ERROR() << "V1_3::utils::PreparedModel::create must have non-null preparedModel";
Michael Butler4b276a72020-08-06 23:22:35 -070092 }
93
94 auto deathHandler = NN_TRY(hal::utils::DeathHandler::create(preparedModel));
Michael Butler7fd03c22020-12-06 21:50:59 -080095 return std::make_shared<const PreparedModel>(PrivateConstructorTag{}, executeSynchronously,
96 std::move(preparedModel), std::move(deathHandler));
Michael Butler4b276a72020-08-06 23:22:35 -070097}
98
Michael Butler7fd03c22020-12-06 21:50:59 -080099PreparedModel::PreparedModel(PrivateConstructorTag /*tag*/, bool executeSynchronously,
100 sp<V1_3::IPreparedModel> preparedModel,
Michael Butler4b276a72020-08-06 23:22:35 -0700101 hal::utils::DeathHandler deathHandler)
Michael Butler7fd03c22020-12-06 21:50:59 -0800102 : kExecuteSynchronously(executeSynchronously),
103 kPreparedModel(std::move(preparedModel)),
104 kDeathHandler(std::move(deathHandler)) {}
Michael Butler4b276a72020-08-06 23:22:35 -0700105
106nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>>
107PreparedModel::executeSynchronously(const Request& request, V1_2::MeasureTiming measure,
108 const OptionalTimePoint& deadline,
109 const OptionalTimeoutDuration& loopTimeoutDuration) const {
Michael Butler7fd03c22020-12-06 21:50:59 -0800110 auto cb = hal::utils::CallbackValue(executionCallback);
Michael Butler4b276a72020-08-06 23:22:35 -0700111
112 const auto ret = kPreparedModel->executeSynchronously_1_3(request, measure, deadline,
113 loopTimeoutDuration, cb);
Michael Butlercca3e202020-11-22 20:25:34 -0800114 HANDLE_TRANSPORT_FAILURE(ret);
Michael Butler4b276a72020-08-06 23:22:35 -0700115
Michael Butler7fd03c22020-12-06 21:50:59 -0800116 return cb.take();
Michael Butler4b276a72020-08-06 23:22:35 -0700117}
118
119nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>>
120PreparedModel::executeAsynchronously(const Request& request, V1_2::MeasureTiming measure,
121 const OptionalTimePoint& deadline,
122 const OptionalTimeoutDuration& loopTimeoutDuration) const {
123 const auto cb = sp<ExecutionCallback>::make();
124 const auto scoped = kDeathHandler.protectCallback(cb.get());
125
126 const auto ret =
127 kPreparedModel->execute_1_3(request, measure, deadline, loopTimeoutDuration, cb);
Michael Butlercca3e202020-11-22 20:25:34 -0800128 const auto status = HANDLE_TRANSPORT_FAILURE(ret);
Michael Butler7fd03c22020-12-06 21:50:59 -0800129 if (status != ErrorStatus::OUTPUT_INSUFFICIENT_SIZE) {
Michael Butler49d95e02021-10-15 18:52:52 -0700130 HANDLE_STATUS_HIDL(status) << "execution failed with " << toString(status);
Michael Butler4b276a72020-08-06 23:22:35 -0700131 }
132
133 return cb->get();
134}
135
136nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>> PreparedModel::execute(
137 const nn::Request& request, nn::MeasureTiming measure,
Miao Wangb5c8a822021-10-26 20:03:05 +0000138 const nn::OptionalTimePoint& deadline, const nn::OptionalDuration& loopTimeoutDuration,
139 const std::vector<nn::TokenValuePair>& /*hints*/,
140 const std::vector<nn::ExtensionNameAndPrefix>& /*extensionNameToPrefix*/) const {
Michael Butler4b276a72020-08-06 23:22:35 -0700141 // Ensure that request is ready for IPC.
142 std::optional<nn::Request> maybeRequestInShared;
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800143 hal::utils::RequestRelocation relocation;
Michael Butlerff9a5a52021-10-15 16:23:20 -0700144 const nn::Request& requestInShared = NN_TRY(hal::utils::convertRequestFromPointerToShared(
145 &request, nn::kDefaultRequestMemoryAlignment, nn::kMinMemoryPadding,
146 &maybeRequestInShared, &relocation));
Michael Butler4b276a72020-08-06 23:22:35 -0700147
Michael Butlerff9a5a52021-10-15 16:23:20 -0700148 const auto hidlRequest = NN_TRY(convert(requestInShared));
149 const auto hidlMeasure = NN_TRY(convert(measure));
150 const auto hidlDeadline = NN_TRY(convert(deadline));
151 const auto hidlLoopTimeoutDuration = NN_TRY(convert(loopTimeoutDuration));
Michael Butler4b276a72020-08-06 23:22:35 -0700152
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800153 return executeInternal(hidlRequest, hidlMeasure, hidlDeadline, hidlLoopTimeoutDuration,
154 relocation);
155}
156
157nn::ExecutionResult<std::pair<std::vector<nn::OutputShape>, nn::Timing>>
158PreparedModel::executeInternal(const Request& request, V1_2::MeasureTiming measure,
159 const OptionalTimePoint& deadline,
160 const OptionalTimeoutDuration& loopTimeoutDuration,
161 const hal::utils::RequestRelocation& relocation) const {
162 if (relocation.input) {
163 relocation.input->flush();
164 }
165
Michael Butler7fd03c22020-12-06 21:50:59 -0800166 auto result = kExecuteSynchronously
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800167 ? executeSynchronously(request, measure, deadline, loopTimeoutDuration)
168 : executeAsynchronously(request, measure, deadline, loopTimeoutDuration);
Michael Butler7fd03c22020-12-06 21:50:59 -0800169 auto [outputShapes, timing] = NN_TRY(std::move(result));
Michael Butler4b276a72020-08-06 23:22:35 -0700170
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800171 if (relocation.output) {
172 relocation.output->flush();
173 }
Michael Butler7fd03c22020-12-06 21:50:59 -0800174 return std::make_pair(std::move(outputShapes), timing);
Michael Butler4b276a72020-08-06 23:22:35 -0700175}
176
177nn::GeneralResult<std::pair<nn::SyncFence, nn::ExecuteFencedInfoCallback>>
Miao Wangb5c8a822021-10-26 20:03:05 +0000178PreparedModel::executeFenced(
179 const nn::Request& request, const std::vector<nn::SyncFence>& waitFor,
180 nn::MeasureTiming measure, const nn::OptionalTimePoint& deadline,
181 const nn::OptionalDuration& loopTimeoutDuration,
182 const nn::OptionalDuration& timeoutDurationAfterFence,
183 const std::vector<nn::TokenValuePair>& /*hints*/,
184 const std::vector<nn::ExtensionNameAndPrefix>& /*extensionNameToPrefix*/) const {
Michael Butler4b276a72020-08-06 23:22:35 -0700185 // Ensure that request is ready for IPC.
186 std::optional<nn::Request> maybeRequestInShared;
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800187 hal::utils::RequestRelocation relocation;
188 const nn::Request& requestInShared = NN_TRY(hal::utils::convertRequestFromPointerToShared(
Xusong Wange3d0dad2021-05-07 14:13:22 -0700189 &request, nn::kDefaultRequestMemoryAlignment, nn::kMinMemoryPadding,
190 &maybeRequestInShared, &relocation));
Michael Butler4b276a72020-08-06 23:22:35 -0700191
192 const auto hidlRequest = NN_TRY(convert(requestInShared));
Michael Butler15965822021-10-14 23:45:11 -0700193 const auto hidlWaitFor = NN_TRY(convertSyncFences(waitFor));
Michael Butler7fd03c22020-12-06 21:50:59 -0800194 const auto hidlMeasure = NN_TRY(convert(measure));
Michael Butler4b276a72020-08-06 23:22:35 -0700195 const auto hidlDeadline = NN_TRY(convert(deadline));
196 const auto hidlLoopTimeoutDuration = NN_TRY(convert(loopTimeoutDuration));
197 const auto hidlTimeoutDurationAfterFence = NN_TRY(convert(timeoutDurationAfterFence));
198
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800199 return executeFencedInternal(hidlRequest, hidlWaitFor, hidlMeasure, hidlDeadline,
200 hidlLoopTimeoutDuration, hidlTimeoutDurationAfterFence,
201 relocation);
202}
203
204nn::GeneralResult<std::pair<nn::SyncFence, nn::ExecuteFencedInfoCallback>>
205PreparedModel::executeFencedInternal(const Request& request, const hidl_vec<hidl_handle>& waitFor,
206 V1_2::MeasureTiming measure, const OptionalTimePoint& deadline,
207 const OptionalTimeoutDuration& loopTimeoutDuration,
208 const OptionalTimeoutDuration& timeoutDurationAfterFence,
209 const hal::utils::RequestRelocation& relocation) const {
210 if (relocation.input) {
211 relocation.input->flush();
212 }
213
Michael Butler7fd03c22020-12-06 21:50:59 -0800214 auto cb = hal::utils::CallbackValue(fencedExecutionCallback);
Michael Butler4b276a72020-08-06 23:22:35 -0700215
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800216 const auto ret =
217 kPreparedModel->executeFenced(request, waitFor, measure, deadline, loopTimeoutDuration,
218 timeoutDurationAfterFence, cb);
Michael Butlercca3e202020-11-22 20:25:34 -0800219 HANDLE_TRANSPORT_FAILURE(ret);
Michael Butler7fd03c22020-12-06 21:50:59 -0800220 auto [syncFence, callback] = NN_TRY(cb.take());
Michael Butler4b276a72020-08-06 23:22:35 -0700221
222 // If executeFenced required the request memory to be moved into shared memory, block here until
223 // the fenced execution has completed and flush the memory back.
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800224 if (relocation.output) {
Michael Butler4b276a72020-08-06 23:22:35 -0700225 const auto state = syncFence.syncWait({});
226 if (state != nn::SyncFence::FenceState::SIGNALED) {
227 return NN_ERROR() << "syncWait failed with " << state;
228 }
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800229 relocation.output->flush();
Michael Butler4b276a72020-08-06 23:22:35 -0700230 }
231
232 return std::make_pair(std::move(syncFence), std::move(callback));
233}
234
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800235nn::GeneralResult<nn::SharedExecution> PreparedModel::createReusableExecution(
236 const nn::Request& request, nn::MeasureTiming measure,
Miao Wangb5c8a822021-10-26 20:03:05 +0000237 const nn::OptionalDuration& loopTimeoutDuration,
238 const std::vector<nn::TokenValuePair>& /*hints*/,
239 const std::vector<nn::ExtensionNameAndPrefix>& /*extensionNameToPrefix*/) const {
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800240 // Ensure that request is ready for IPC.
241 std::optional<nn::Request> maybeRequestInShared;
242 hal::utils::RequestRelocation relocation;
243 const nn::Request& requestInShared = NN_TRY(hal::utils::convertRequestFromPointerToShared(
Xusong Wange3d0dad2021-05-07 14:13:22 -0700244 &request, nn::kDefaultRequestMemoryAlignment, nn::kMinMemoryPadding,
245 &maybeRequestInShared, &relocation));
Xusong Wang5f6bedb2021-03-03 16:20:37 -0800246
247 auto hidlRequest = NN_TRY(convert(requestInShared));
248 auto hidlMeasure = NN_TRY(convert(measure));
249 auto hidlLoopTimeoutDuration = NN_TRY(convert(loopTimeoutDuration));
250 return Execution::create(shared_from_this(), std::move(hidlRequest), std::move(relocation),
251 hidlMeasure, std::move(hidlLoopTimeoutDuration));
252}
253
Michael Butler2a8b6792020-12-18 20:31:14 -0800254nn::GeneralResult<nn::SharedBurst> PreparedModel::configureExecutionBurst() const {
Michael Butler76e491f2020-12-19 01:55:32 -0800255 const auto pollingTimeWindow = V1_2::utils::getBurstControllerPollingTimeWindow();
Michael Butler137ee992021-11-01 16:40:31 -0700256 return V1_2::utils::Burst::create(shared_from_this(), kPreparedModel, pollingTimeWindow);
Michael Butler2a8b6792020-12-18 20:31:14 -0800257}
258
Michael Butler4b276a72020-08-06 23:22:35 -0700259std::any PreparedModel::getUnderlyingResource() const {
260 sp<V1_3::IPreparedModel> resource = kPreparedModel;
261 return resource;
262}
263
264} // namespace android::hardware::neuralnetworks::V1_3::utils