blob: 7a5035f6fcf1dd1716c70d3e1a2ba6742ac36ffb [file] [log] [blame]
Michael Butlera685c3d2020-02-22 22:37:59 -08001/*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "CommonUtils.h"
18
Michael Butler3670c382020-08-06 23:22:35 -070019#include "HandleError.h"
20
Michael Butlera685c3d2020-02-22 22:37:59 -080021#include <android-base/logging.h>
Slava Shklyaevd4290b82020-10-27 18:44:01 +000022#include <android-base/unique_fd.h>
Michael Butlerbbe43d92021-02-08 00:05:07 -080023#include <android/hardware_buffer.h>
24#include <hidl/HidlSupport.h>
Michael Butlera685c3d2020-02-22 22:37:59 -080025#include <nnapi/Result.h>
26#include <nnapi/SharedMemory.h>
27#include <nnapi/TypeUtils.h>
28#include <nnapi/Types.h>
29#include <nnapi/Validation.h>
Michael Butlerbbe43d92021-02-08 00:05:07 -080030#include <vndk/hardware_buffer.h>
Michael Butlera685c3d2020-02-22 22:37:59 -080031
32#include <algorithm>
33#include <any>
Michael Butler3670c382020-08-06 23:22:35 -070034#include <functional>
Michael Butlera685c3d2020-02-22 22:37:59 -080035#include <optional>
36#include <variant>
37#include <vector>
38
39namespace android::hardware::neuralnetworks::utils {
40namespace {
41
42bool hasNoPointerData(const nn::Operand& operand);
43bool hasNoPointerData(const nn::Model::Subgraph& subgraph);
44bool hasNoPointerData(const nn::Request::Argument& argument);
45
46template <typename Type>
47bool hasNoPointerData(const std::vector<Type>& objects) {
48 return std::all_of(objects.begin(), objects.end(),
49 [](const auto& object) { return hasNoPointerData(object); });
50}
51
52bool hasNoPointerData(const nn::DataLocation& location) {
53 return std::visit([](auto ptr) { return ptr == nullptr; }, location.pointer);
54}
55
56bool hasNoPointerData(const nn::Operand& operand) {
57 return hasNoPointerData(operand.location);
58}
59
60bool hasNoPointerData(const nn::Model::Subgraph& subgraph) {
61 return hasNoPointerData(subgraph.operands);
62}
63
64bool hasNoPointerData(const nn::Request::Argument& argument) {
65 return hasNoPointerData(argument.location);
66}
67
68void copyPointersToSharedMemory(nn::Operand* operand, nn::ConstantMemoryBuilder* memoryBuilder) {
69 CHECK(operand != nullptr);
70 CHECK(memoryBuilder != nullptr);
71
72 if (operand->lifetime != nn::Operand::LifeTime::POINTER) {
73 return;
74 }
75
76 const void* data = std::visit([](auto ptr) { return static_cast<const void*>(ptr); },
77 operand->location.pointer);
78 CHECK(data != nullptr);
79 operand->lifetime = nn::Operand::LifeTime::CONSTANT_REFERENCE;
80 operand->location = memoryBuilder->append(data, operand->location.length);
81}
82
83void copyPointersToSharedMemory(nn::Model::Subgraph* subgraph,
84 nn::ConstantMemoryBuilder* memoryBuilder) {
85 CHECK(subgraph != nullptr);
86 std::for_each(subgraph->operands.begin(), subgraph->operands.end(),
87 [memoryBuilder](auto& operand) {
88 copyPointersToSharedMemory(&operand, memoryBuilder);
89 });
90}
91
92} // anonymous namespace
93
94nn::Capabilities::OperandPerformanceTable makeQuantized8PerformanceConsistentWithP(
95 const nn::Capabilities::PerformanceInfo& float32Performance,
96 const nn::Capabilities::PerformanceInfo& quantized8Performance) {
97 // In Android P, most data types are treated as having the same performance as
98 // TENSOR_QUANT8_ASYMM. This collection must be in sorted order.
99 std::vector<nn::Capabilities::OperandPerformance> operandPerformances = {
100 {.type = nn::OperandType::FLOAT32, .info = float32Performance},
101 {.type = nn::OperandType::INT32, .info = quantized8Performance},
102 {.type = nn::OperandType::UINT32, .info = quantized8Performance},
103 {.type = nn::OperandType::TENSOR_FLOAT32, .info = float32Performance},
104 {.type = nn::OperandType::TENSOR_INT32, .info = quantized8Performance},
105 {.type = nn::OperandType::TENSOR_QUANT8_ASYMM, .info = quantized8Performance},
106 {.type = nn::OperandType::OEM, .info = quantized8Performance},
107 {.type = nn::OperandType::TENSOR_OEM_BYTE, .info = quantized8Performance},
108 };
109 return nn::Capabilities::OperandPerformanceTable::create(std::move(operandPerformances))
110 .value();
111}
112
113bool hasNoPointerData(const nn::Model& model) {
114 return hasNoPointerData(model.main) && hasNoPointerData(model.referenced);
115}
116
117bool hasNoPointerData(const nn::Request& request) {
118 return hasNoPointerData(request.inputs) && hasNoPointerData(request.outputs);
119}
120
Michael Butler3670c382020-08-06 23:22:35 -0700121nn::GeneralResult<std::reference_wrapper<const nn::Model>> flushDataFromPointerToShared(
122 const nn::Model* model, std::optional<nn::Model>* maybeModelInSharedOut) {
123 CHECK(model != nullptr);
124 CHECK(maybeModelInSharedOut != nullptr);
125
126 if (hasNoPointerData(*model)) {
127 return *model;
128 }
129
130 // Make a copy of the model in order to make modifications. The modified model is returned to
131 // the caller through `maybeModelInSharedOut` if the function succeeds.
132 nn::Model modelInShared = *model;
Michael Butlera685c3d2020-02-22 22:37:59 -0800133
134 nn::ConstantMemoryBuilder memoryBuilder(modelInShared.pools.size());
135 copyPointersToSharedMemory(&modelInShared.main, &memoryBuilder);
136 std::for_each(modelInShared.referenced.begin(), modelInShared.referenced.end(),
137 [&memoryBuilder](auto& subgraph) {
138 copyPointersToSharedMemory(&subgraph, &memoryBuilder);
139 });
140
141 if (!memoryBuilder.empty()) {
142 auto memory = NN_TRY(memoryBuilder.finish());
143 modelInShared.pools.push_back(std::move(memory));
144 }
145
Michael Butler3670c382020-08-06 23:22:35 -0700146 *maybeModelInSharedOut = modelInShared;
147 return **maybeModelInSharedOut;
Michael Butlera685c3d2020-02-22 22:37:59 -0800148}
149
Michael Butler3670c382020-08-06 23:22:35 -0700150nn::GeneralResult<std::reference_wrapper<const nn::Request>> flushDataFromPointerToShared(
151 const nn::Request* request, std::optional<nn::Request>* maybeRequestInSharedOut) {
152 CHECK(request != nullptr);
153 CHECK(maybeRequestInSharedOut != nullptr);
154
155 if (hasNoPointerData(*request)) {
156 return *request;
157 }
158
159 // Make a copy of the request in order to make modifications. The modified request is returned
160 // to the caller through `maybeRequestInSharedOut` if the function succeeds.
161 nn::Request requestInShared = *request;
Michael Butlera685c3d2020-02-22 22:37:59 -0800162
163 // Change input pointers to shared memory.
164 nn::ConstantMemoryBuilder inputBuilder(requestInShared.pools.size());
165 for (auto& input : requestInShared.inputs) {
166 const auto& location = input.location;
167 if (input.lifetime != nn::Request::Argument::LifeTime::POINTER) {
168 continue;
169 }
170
171 input.lifetime = nn::Request::Argument::LifeTime::POOL;
172 const void* data = std::visit([](auto ptr) { return static_cast<const void*>(ptr); },
173 location.pointer);
174 CHECK(data != nullptr);
175 input.location = inputBuilder.append(data, location.length);
176 }
177
178 // Allocate input memory.
179 if (!inputBuilder.empty()) {
180 auto memory = NN_TRY(inputBuilder.finish());
181 requestInShared.pools.push_back(std::move(memory));
182 }
183
184 // Change output pointers to shared memory.
185 nn::MutableMemoryBuilder outputBuilder(requestInShared.pools.size());
186 for (auto& output : requestInShared.outputs) {
187 const auto& location = output.location;
188 if (output.lifetime != nn::Request::Argument::LifeTime::POINTER) {
189 continue;
190 }
191
192 output.lifetime = nn::Request::Argument::LifeTime::POOL;
193 output.location = outputBuilder.append(location.length);
194 }
195
196 // Allocate output memory.
197 if (!outputBuilder.empty()) {
198 auto memory = NN_TRY(outputBuilder.finish());
199 requestInShared.pools.push_back(std::move(memory));
200 }
201
Michael Butler3670c382020-08-06 23:22:35 -0700202 *maybeRequestInSharedOut = requestInShared;
203 return **maybeRequestInSharedOut;
Michael Butlera685c3d2020-02-22 22:37:59 -0800204}
205
Michael Butler3670c382020-08-06 23:22:35 -0700206nn::GeneralResult<void> unflushDataFromSharedToPointer(
207 const nn::Request& request, const std::optional<nn::Request>& maybeRequestInShared) {
208 if (!maybeRequestInShared.has_value() || maybeRequestInShared->pools.empty() ||
Michael Butler79a16eb2021-02-07 00:11:13 -0800209 !std::holds_alternative<nn::SharedMemory>(maybeRequestInShared->pools.back())) {
Michael Butlera685c3d2020-02-22 22:37:59 -0800210 return {};
211 }
Michael Butler3670c382020-08-06 23:22:35 -0700212 const auto& requestInShared = *maybeRequestInShared;
Michael Butlera685c3d2020-02-22 22:37:59 -0800213
214 // Map the memory.
Michael Butler79a16eb2021-02-07 00:11:13 -0800215 const auto& outputMemory = std::get<nn::SharedMemory>(requestInShared.pools.back());
Michael Butlera685c3d2020-02-22 22:37:59 -0800216 const auto [pointer, size, context] = NN_TRY(map(outputMemory));
217 const uint8_t* constantPointer =
218 std::visit([](const auto& o) { return static_cast<const uint8_t*>(o); }, pointer);
219
220 // Flush each output pointer.
221 CHECK_EQ(request.outputs.size(), requestInShared.outputs.size());
222 for (size_t i = 0; i < request.outputs.size(); ++i) {
223 const auto& location = request.outputs[i].location;
224 const auto& locationInShared = requestInShared.outputs[i].location;
225 if (!std::holds_alternative<void*>(location.pointer)) {
226 continue;
227 }
228
229 // Get output pointer and size.
230 void* data = std::get<void*>(location.pointer);
231 CHECK(data != nullptr);
232 const size_t length = location.length;
233
234 // Get output pool location.
235 CHECK(requestInShared.outputs[i].lifetime == nn::Request::Argument::LifeTime::POOL);
236 const size_t index = locationInShared.poolIndex;
237 const size_t offset = locationInShared.offset;
238 const size_t outputPoolIndex = requestInShared.pools.size() - 1;
239 CHECK(locationInShared.length == length);
240 CHECK(index == outputPoolIndex);
241
242 // Flush memory.
243 std::memcpy(data, constantPointer + offset, length);
244 }
245
246 return {};
247}
248
249std::vector<uint32_t> countNumberOfConsumers(size_t numberOfOperands,
250 const std::vector<nn::Operation>& operations) {
251 return nn::countNumberOfConsumers(numberOfOperands, operations);
252}
253
Michael Butlerbbe43d92021-02-08 00:05:07 -0800254nn::GeneralResult<hidl_memory> createHidlMemoryFromSharedMemory(const nn::SharedMemory& memory) {
255 if (memory == nullptr) {
256 return NN_ERROR() << "Memory must be non-empty";
257 }
258 if (const auto* handle = std::get_if<nn::Handle>(&memory->handle)) {
259 return hidl_memory(memory->name, NN_TRY(hidlHandleFromSharedHandle(*handle)), memory->size);
Slava Shklyaevd4290b82020-10-27 18:44:01 +0000260 }
261
Michael Butlerbbe43d92021-02-08 00:05:07 -0800262 const auto* ahwb = std::get<nn::HardwareBufferHandle>(memory->handle).get();
263 AHardwareBuffer_Desc bufferDesc;
264 AHardwareBuffer_describe(ahwb, &bufferDesc);
265
266 if (bufferDesc.format == AHARDWAREBUFFER_FORMAT_BLOB) {
267 CHECK_EQ(memory->size, bufferDesc.width);
268 CHECK_EQ(memory->name, "hardware_buffer_blob");
269 } else {
270 CHECK_EQ(memory->size, 0u);
271 CHECK_EQ(memory->name, "hardware_buffer");
272 }
273
274 const native_handle_t* nativeHandle = AHardwareBuffer_getNativeHandle(ahwb);
275 const hidl_handle hidlHandle(nativeHandle);
276 hidl_handle handle(hidlHandle);
277
278 return hidl_memory(memory->name, std::move(handle), memory->size);
279}
280
281static uint32_t roundUpToMultiple(uint32_t value, uint32_t multiple) {
282 return (value + multiple - 1) / multiple * multiple;
283}
284
285nn::GeneralResult<nn::SharedMemory> createSharedMemoryFromHidlMemory(const hidl_memory& memory) {
286 CHECK_LE(memory.size(), std::numeric_limits<uint32_t>::max());
287
288 if (memory.name() != "hardware_buffer_blob") {
289 return std::make_shared<const nn::Memory>(nn::Memory{
290 .handle = NN_TRY(sharedHandleFromNativeHandle(memory.handle())),
291 .size = static_cast<uint32_t>(memory.size()),
292 .name = memory.name(),
293 });
294 }
295
296 const auto size = memory.size();
297 const auto format = AHARDWAREBUFFER_FORMAT_BLOB;
298 const auto usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN | AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN;
299 const uint32_t width = size;
300 const uint32_t height = 1; // height is always 1 for BLOB mode AHardwareBuffer.
301 const uint32_t layers = 1; // layers is always 1 for BLOB mode AHardwareBuffer.
302
303 // AHardwareBuffer_createFromHandle() might fail because an allocator
304 // expects a specific stride value. In that case, we try to guess it by
305 // aligning the width to small powers of 2.
306 // TODO(b/174120849): Avoid stride assumptions.
307 AHardwareBuffer* hardwareBuffer = nullptr;
308 status_t status = UNKNOWN_ERROR;
309 for (uint32_t alignment : {1, 4, 32, 64, 128, 2, 8, 16}) {
310 const uint32_t stride = roundUpToMultiple(width, alignment);
311 AHardwareBuffer_Desc desc{
312 .width = width,
313 .height = height,
314 .layers = layers,
315 .format = format,
316 .usage = usage,
317 .stride = stride,
318 };
319 status = AHardwareBuffer_createFromHandle(&desc, memory.handle(),
320 AHARDWAREBUFFER_CREATE_FROM_HANDLE_METHOD_CLONE,
321 &hardwareBuffer);
322 if (status == NO_ERROR) {
323 break;
324 }
325 }
326 if (status != NO_ERROR) {
327 return NN_ERROR(nn::ErrorStatus::GENERAL_FAILURE)
328 << "Can't create AHardwareBuffer from handle. Error: " << status;
329 }
330
331 return std::make_shared<const nn::Memory>(nn::Memory{
332 .handle = nn::HardwareBufferHandle(hardwareBuffer, /*takeOwnership=*/true),
333 .size = static_cast<uint32_t>(memory.size()),
334 .name = memory.name(),
335 });
336}
337
338nn::GeneralResult<hidl_handle> hidlHandleFromSharedHandle(const nn::Handle& handle) {
Slava Shklyaevd4290b82020-10-27 18:44:01 +0000339 std::vector<base::unique_fd> fds;
Michael Butlerbbe43d92021-02-08 00:05:07 -0800340 fds.reserve(handle.fds.size());
341 for (const auto& fd : handle.fds) {
342 const int dupFd = dup(fd);
Slava Shklyaevd4290b82020-10-27 18:44:01 +0000343 if (dupFd == -1) {
344 return NN_ERROR(nn::ErrorStatus::GENERAL_FAILURE) << "Failed to dup the fd";
345 }
346 fds.emplace_back(dupFd);
347 }
348
Michael Butlerbbe43d92021-02-08 00:05:07 -0800349 constexpr size_t kIntMax = std::numeric_limits<int>::max();
350 CHECK_LE(handle.fds.size(), kIntMax);
351 CHECK_LE(handle.ints.size(), kIntMax);
352 native_handle_t* nativeHandle = native_handle_create(static_cast<int>(handle.fds.size()),
353 static_cast<int>(handle.ints.size()));
Slava Shklyaevd4290b82020-10-27 18:44:01 +0000354 if (nativeHandle == nullptr) {
355 return NN_ERROR(nn::ErrorStatus::GENERAL_FAILURE) << "Failed to create native_handle";
356 }
357 for (size_t i = 0; i < fds.size(); ++i) {
358 nativeHandle->data[i] = fds[i].release();
359 }
Michael Butlerbbe43d92021-02-08 00:05:07 -0800360 std::copy(handle.ints.begin(), handle.ints.end(), &nativeHandle->data[nativeHandle->numFds]);
Slava Shklyaevd4290b82020-10-27 18:44:01 +0000361
362 hidl_handle hidlHandle;
363 hidlHandle.setTo(nativeHandle, /*shouldOwn=*/true);
364 return hidlHandle;
365}
366
Michael Butlerbbe43d92021-02-08 00:05:07 -0800367nn::GeneralResult<nn::Handle> sharedHandleFromNativeHandle(const native_handle_t* handle) {
Slava Shklyaevd4290b82020-10-27 18:44:01 +0000368 if (handle == nullptr) {
Michael Butlerbbe43d92021-02-08 00:05:07 -0800369 return NN_ERROR() << "sharedHandleFromNativeHandle failed because handle is nullptr";
Slava Shklyaevd4290b82020-10-27 18:44:01 +0000370 }
371
372 std::vector<base::unique_fd> fds;
373 fds.reserve(handle->numFds);
374 for (int i = 0; i < handle->numFds; ++i) {
Michael Butlerbbe43d92021-02-08 00:05:07 -0800375 const int dupFd = dup(handle->data[i]);
Slava Shklyaevd4290b82020-10-27 18:44:01 +0000376 if (dupFd == -1) {
377 return NN_ERROR(nn::ErrorStatus::GENERAL_FAILURE) << "Failed to dup the fd";
378 }
379 fds.emplace_back(dupFd);
380 }
381
382 std::vector<int> ints(&handle->data[handle->numFds],
383 &handle->data[handle->numFds + handle->numInts]);
384
Michael Butlerbbe43d92021-02-08 00:05:07 -0800385 return nn::Handle{.fds = std::move(fds), .ints = std::move(ints)};
Slava Shklyaevd4290b82020-10-27 18:44:01 +0000386}
387
388nn::GeneralResult<hidl_vec<hidl_handle>> convertSyncFences(
389 const std::vector<nn::SyncFence>& syncFences) {
390 hidl_vec<hidl_handle> handles(syncFences.size());
391 for (size_t i = 0; i < syncFences.size(); ++i) {
Michael Butlerbbe43d92021-02-08 00:05:07 -0800392 const auto& handle = syncFences[i].getSharedHandle();
393 if (handle == nullptr) {
394 return NN_ERROR() << "convertSyncFences failed because sync fence is empty";
395 }
396 handles[i] = NN_TRY(hidlHandleFromSharedHandle(*handle));
Slava Shklyaevd4290b82020-10-27 18:44:01 +0000397 }
398 return handles;
399}
400
Michael Butlera685c3d2020-02-22 22:37:59 -0800401} // namespace android::hardware::neuralnetworks::utils