blob: 83a8d94ba52bddc274c6dc541a4b8a62e270e10d [file] [log] [blame]
Lev Proleev13fdfcd2019-08-30 11:35:34 +01001/*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "GeneratedTestHarness.h"
18
19#include <android-base/logging.h>
20#include <android/hardware/neuralnetworks/1.0/IDevice.h>
21#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
22#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
23#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
24#include <android/hardware/neuralnetworks/1.0/types.h>
25#include <android/hardware/neuralnetworks/1.1/IDevice.h>
26#include <android/hardware/neuralnetworks/1.2/IDevice.h>
27#include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h>
28#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
29#include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h>
Lev Proleev26d1bc82019-08-30 11:57:18 +010030#include <android/hardware/neuralnetworks/1.2/types.h>
31#include <android/hardware/neuralnetworks/1.3/IDevice.h>
Miao Wang2b5c4cd2019-12-26 18:03:56 -080032#include <android/hardware/neuralnetworks/1.3/IFencedExecutionCallback.h>
Xusong Wang1b3f4262019-10-25 12:07:17 -070033#include <android/hardware/neuralnetworks/1.3/IPreparedModel.h>
Xusong Wangcc47dff2019-10-23 10:35:07 -070034#include <android/hardware/neuralnetworks/1.3/IPreparedModelCallback.h>
Lev Proleev26d1bc82019-08-30 11:57:18 +010035#include <android/hardware/neuralnetworks/1.3/types.h>
Lev Proleev13fdfcd2019-08-30 11:35:34 +010036#include <android/hidl/allocator/1.0/IAllocator.h>
37#include <android/hidl/memory/1.0/IMemory.h>
Miao Wang2b5c4cd2019-12-26 18:03:56 -080038#include <android/sync.h>
Lev Proleev56cda832019-12-05 14:49:47 +000039#include <gtest/gtest.h>
Lev Proleev13fdfcd2019-08-30 11:35:34 +010040#include <hidlmemory/mapping.h>
41
Lev Proleev13fdfcd2019-08-30 11:35:34 +010042#include <algorithm>
Michael Butler648ada52019-07-25 17:22:11 -070043#include <chrono>
Lev Proleev13fdfcd2019-08-30 11:35:34 +010044#include <iostream>
45#include <numeric>
Lev Proleev56cda832019-12-05 14:49:47 +000046#include <vector>
Lev Proleev13fdfcd2019-08-30 11:35:34 +010047
48#include "1.0/Utils.h"
Xusong Wangcc47dff2019-10-23 10:35:07 -070049#include "1.3/Callbacks.h"
Michael Butler95899b32020-01-07 14:52:44 -080050#include "1.3/Utils.h"
Lev Proleev13fdfcd2019-08-30 11:35:34 +010051#include "ExecutionBurstController.h"
52#include "MemoryUtils.h"
53#include "TestHarness.h"
54#include "Utils.h"
55#include "VtsHalNeuralnetworks.h"
56
Lev Proleev26d1bc82019-08-30 11:57:18 +010057namespace android::hardware::neuralnetworks::V1_3::vts::functional {
Lev Proleev13fdfcd2019-08-30 11:35:34 +010058
59using namespace test_helper;
60using hidl::memory::V1_0::IMemory;
Michael Butler79a41d72019-12-11 19:08:08 -080061using implementation::ExecutionCallback;
Xusong Wangcc47dff2019-10-23 10:35:07 -070062using implementation::PreparedModelCallback;
Lev Proleev13fdfcd2019-08-30 11:35:34 +010063using V1_0::DataLocation;
Xusong Wange9da9852020-01-13 11:44:45 -080064using V1_0::RequestArgument;
Lev Proleev13fdfcd2019-08-30 11:35:34 +010065using V1_1::ExecutionPreference;
Lev Proleev26d1bc82019-08-30 11:57:18 +010066using V1_2::Constant;
Lev Proleev26d1bc82019-08-30 11:57:18 +010067using V1_2::MeasureTiming;
Lev Proleev26d1bc82019-08-30 11:57:18 +010068using V1_2::OutputShape;
69using V1_2::SymmPerChannelQuantParams;
70using V1_2::Timing;
Lev Proleev13fdfcd2019-08-30 11:35:34 +010071using HidlToken = hidl_array<uint8_t, static_cast<uint32_t>(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
72
Lev Proleev0d4ba3f2019-10-02 17:32:06 +010073namespace {
74
Miao Wang2b5c4cd2019-12-26 18:03:56 -080075enum class Executor { ASYNC, SYNC, BURST, FENCED };
Lev Proleev0d4ba3f2019-10-02 17:32:06 +010076
Slava Shklyaeva37c2c82020-02-18 16:11:32 +000077enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT, MISSED_DEADLINE };
Lev Proleev13fdfcd2019-08-30 11:35:34 +010078
Xusong Wange9da9852020-01-13 11:44:45 -080079enum class MemoryType { SHARED, DEVICE };
80
81enum class IOType { INPUT, OUTPUT };
82
Miao Wang99753d92020-01-21 13:15:09 -080083static void waitForSyncFence(int syncFd) {
84 constexpr int kInfiniteTimeout = -1;
85 ASSERT_GT(syncFd, 0);
86 int r = sync_wait(syncFd, kInfiniteTimeout);
87 ASSERT_GE(r, 0);
88}
89
Lev Proleev0d4ba3f2019-10-02 17:32:06 +010090struct TestConfig {
91 Executor executor;
92 MeasureTiming measureTiming;
93 OutputType outputType;
Xusong Wange9da9852020-01-13 11:44:45 -080094 MemoryType memoryType;
Lev Proleev9226c1e2019-10-03 14:43:18 +010095 // `reportSkipping` indicates if a test should print an info message in case
96 // it is skipped. The field is set to true by default and is set to false in
97 // quantization coupling tests to suppress skipping a test
98 bool reportSkipping;
Xusong Wange9da9852020-01-13 11:44:45 -080099 TestConfig(Executor executor, MeasureTiming measureTiming, OutputType outputType,
100 MemoryType memoryType)
Lev Proleev9226c1e2019-10-03 14:43:18 +0100101 : executor(executor),
102 measureTiming(measureTiming),
103 outputType(outputType),
Xusong Wange9da9852020-01-13 11:44:45 -0800104 memoryType(memoryType),
Lev Proleev9226c1e2019-10-03 14:43:18 +0100105 reportSkipping(true) {}
106 TestConfig(Executor executor, MeasureTiming measureTiming, OutputType outputType,
Xusong Wange9da9852020-01-13 11:44:45 -0800107 MemoryType memoryType, bool reportSkipping)
Lev Proleev9226c1e2019-10-03 14:43:18 +0100108 : executor(executor),
109 measureTiming(measureTiming),
110 outputType(outputType),
Xusong Wange9da9852020-01-13 11:44:45 -0800111 memoryType(memoryType),
Lev Proleev9226c1e2019-10-03 14:43:18 +0100112 reportSkipping(reportSkipping) {}
Lev Proleev0d4ba3f2019-10-02 17:32:06 +0100113};
114
Xusong Wange9da9852020-01-13 11:44:45 -0800115class DeviceMemoryAllocator {
116 public:
117 DeviceMemoryAllocator(const sp<IDevice>& device, const sp<IPreparedModel>& preparedModel,
118 const TestModel& testModel)
119 : kDevice(device), kPreparedModel(preparedModel), kTestModel(testModel) {}
120
121 // Allocate device memory for a target input/output operand.
122 // Return {IBuffer object, token} if successful.
123 // Return {nullptr, 0} if device memory is not supported.
124 template <IOType ioType>
Michael Butlerc3e1a292020-02-04 16:15:04 -0800125 std::pair<sp<IBuffer>, uint32_t> allocate(uint32_t index) {
126 std::pair<sp<IBuffer>, uint32_t> buffer;
Xusong Wange9da9852020-01-13 11:44:45 -0800127 allocateInternal<ioType>(index, &buffer);
128 return buffer;
129 }
130
131 private:
132 template <IOType ioType>
Michael Butlerc3e1a292020-02-04 16:15:04 -0800133 void allocateInternal(uint32_t index, std::pair<sp<IBuffer>, uint32_t>* result) {
Xusong Wange9da9852020-01-13 11:44:45 -0800134 ASSERT_NE(result, nullptr);
135
136 // Prepare arguments.
137 BufferRole role = {.modelIndex = 0, .ioIndex = index, .frequency = 1.0f};
138 hidl_vec<BufferRole> inputRoles, outputRoles;
139 if constexpr (ioType == IOType::INPUT) {
140 inputRoles = {role};
141 } else {
142 outputRoles = {role};
143 }
144
145 // Allocate device memory.
146 ErrorStatus status;
147 sp<IBuffer> buffer;
Michael Butlerc3e1a292020-02-04 16:15:04 -0800148 uint32_t token;
149 auto cb = [&status, &buffer, &token](ErrorStatus error, const sp<IBuffer>& buf,
150 uint32_t tok) {
151 status = error;
152 buffer = buf;
153 token = tok;
154 };
155 const auto ret = kDevice->allocate({}, {kPreparedModel}, inputRoles, outputRoles, cb);
Xusong Wange9da9852020-01-13 11:44:45 -0800156
157 // Check allocation results.
158 ASSERT_TRUE(ret.isOk());
159 if (status == ErrorStatus::NONE) {
160 ASSERT_NE(buffer, nullptr);
161 ASSERT_GT(token, 0);
162 } else {
163 ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE);
164 ASSERT_EQ(buffer, nullptr);
165 ASSERT_EQ(token, 0);
166 }
167
168 // Initialize input data from TestBuffer.
169 if constexpr (ioType == IOType::INPUT) {
170 if (buffer != nullptr) {
171 // TestBuffer -> Shared memory.
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000172 const auto& testBuffer =
173 kTestModel.main.operands[kTestModel.main.inputIndexes[index]].data;
Xusong Wange9da9852020-01-13 11:44:45 -0800174 ASSERT_GT(testBuffer.size(), 0);
175 hidl_memory tmp = nn::allocateSharedMemory(testBuffer.size());
176 sp<IMemory> inputMemory = mapMemory(tmp);
177 ASSERT_NE(inputMemory.get(), nullptr);
178 uint8_t* inputPtr =
179 static_cast<uint8_t*>(static_cast<void*>(inputMemory->getPointer()));
180 ASSERT_NE(inputPtr, nullptr);
181 const uint8_t* begin = testBuffer.get<uint8_t>();
182 const uint8_t* end = begin + testBuffer.size();
183 std::copy(begin, end, inputPtr);
184
185 // Shared memory -> IBuffer.
186 auto ret = buffer->copyFrom(tmp, {});
187 ASSERT_TRUE(ret.isOk());
188 ASSERT_EQ(static_cast<ErrorStatus>(ret), ErrorStatus::NONE);
189 }
190 }
191 *result = {std::move(buffer), token};
192 }
193
194 const sp<IDevice> kDevice;
195 const sp<IPreparedModel> kPreparedModel;
196 const TestModel& kTestModel;
197};
198
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000199Subgraph createSubgraph(const TestSubgraph& testSubgraph, uint32_t* constCopySize,
200 std::vector<const TestBuffer*>* constCopies, uint32_t* constRefSize,
201 std::vector<const TestBuffer*>* constReferences) {
202 CHECK(constCopySize != nullptr);
203 CHECK(constCopies != nullptr);
204 CHECK(constRefSize != nullptr);
205 CHECK(constReferences != nullptr);
Lev Proleev0d4ba3f2019-10-02 17:32:06 +0100206
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000207 // Operands.
208 hidl_vec<Operand> operands(testSubgraph.operands.size());
209 for (uint32_t i = 0; i < testSubgraph.operands.size(); i++) {
210 const auto& op = testSubgraph.operands[i];
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100211
212 DataLocation loc = {};
213 if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000214 loc = {
215 .poolIndex = 0,
216 .offset = *constCopySize,
217 .length = static_cast<uint32_t>(op.data.size()),
218 };
219 constCopies->push_back(&op.data);
220 *constCopySize += op.data.alignedSize();
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100221 } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000222 loc = {
223 .poolIndex = 0,
224 .offset = *constRefSize,
225 .length = static_cast<uint32_t>(op.data.size()),
226 };
227 constReferences->push_back(&op.data);
228 *constRefSize += op.data.alignedSize();
229 } else if (op.lifetime == TestOperandLifeTime::SUBGRAPH) {
230 loc = {
231 .poolIndex = 0,
232 .offset = *op.data.get<uint32_t>(),
233 .length = 0,
234 };
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100235 }
236
Michael Butlerc3e1a292020-02-04 16:15:04 -0800237 V1_2::Operand::ExtraParams extraParams;
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100238 if (op.type == TestOperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
239 extraParams.channelQuant(SymmPerChannelQuantParams{
240 .scales = op.channelQuant.scales, .channelDim = op.channelQuant.channelDim});
241 }
242
243 operands[i] = {.type = static_cast<OperandType>(op.type),
244 .dimensions = op.dimensions,
245 .numberOfConsumers = op.numberOfConsumers,
246 .scale = op.scale,
247 .zeroPoint = op.zeroPoint,
248 .lifetime = static_cast<OperandLifeTime>(op.lifetime),
249 .location = loc,
250 .extraParams = std::move(extraParams)};
251 }
252
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000253 // Operations.
254 hidl_vec<Operation> operations(testSubgraph.operations.size());
255 std::transform(testSubgraph.operations.begin(), testSubgraph.operations.end(),
256 operations.begin(), [](const TestOperation& op) -> Operation {
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100257 return {.type = static_cast<OperationType>(op.type),
258 .inputs = op.inputs,
259 .outputs = op.outputs};
260 });
261
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000262 return {.operands = std::move(operands),
263 .operations = std::move(operations),
264 .inputIndexes = testSubgraph.inputIndexes,
265 .outputIndexes = testSubgraph.outputIndexes};
266}
267
268void copyTestBuffers(const std::vector<const TestBuffer*>& buffers, uint8_t* output) {
269 uint32_t offset = 0;
270 for (const TestBuffer* buffer : buffers) {
271 const uint8_t* begin = buffer->get<uint8_t>();
272 const uint8_t* end = begin + buffer->size();
273 std::copy(begin, end, output + offset);
274 offset += buffer->alignedSize();
275 }
276}
277
278} // namespace
279
280Model createModel(const TestModel& testModel) {
281 uint32_t constCopySize = 0;
282 uint32_t constRefSize = 0;
283 std::vector<const TestBuffer*> constCopies;
284 std::vector<const TestBuffer*> constReferences;
285
286 Subgraph mainSubgraph = createSubgraph(testModel.main, &constCopySize, &constCopies,
287 &constRefSize, &constReferences);
288 hidl_vec<Subgraph> refSubgraphs(testModel.referenced.size());
289 std::transform(testModel.referenced.begin(), testModel.referenced.end(), refSubgraphs.begin(),
290 [&constCopySize, &constCopies, &constRefSize,
291 &constReferences](const TestSubgraph& testSubgraph) {
292 return createSubgraph(testSubgraph, &constCopySize, &constCopies,
293 &constRefSize, &constReferences);
294 });
295
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100296 // Constant copies.
297 hidl_vec<uint8_t> operandValues(constCopySize);
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000298 copyTestBuffers(constCopies, operandValues.data());
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100299
300 // Shared memory.
301 hidl_vec<hidl_memory> pools = {};
302 if (constRefSize > 0) {
303 hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
304 CHECK_NE(pools[0].size(), 0u);
305
306 // load data
307 sp<IMemory> mappedMemory = mapMemory(pools[0]);
308 CHECK(mappedMemory.get() != nullptr);
309 uint8_t* mappedPtr =
310 reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));
311 CHECK(mappedPtr != nullptr);
312
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000313 copyTestBuffers(constReferences, mappedPtr);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100314 }
315
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000316 return {.main = std::move(mainSubgraph),
317 .referenced = std::move(refSubgraphs),
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100318 .operandValues = std::move(operandValues),
319 .pools = std::move(pools),
320 .relaxComputationFloat32toFloat16 = testModel.isRelaxed};
321}
322
323static bool isOutputSizeGreaterThanOne(const TestModel& testModel, uint32_t index) {
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000324 const auto byteSize = testModel.main.operands[testModel.main.outputIndexes[index]].data.size();
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100325 return byteSize > 1u;
326}
327
Xusong Wange9da9852020-01-13 11:44:45 -0800328static void makeOutputInsufficientSize(uint32_t outputIndex, Request* request) {
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100329 auto& length = request->outputs[outputIndex].location.length;
330 ASSERT_GT(length, 1u);
331 length -= 1u;
332}
333
334static void makeOutputDimensionsUnspecified(Model* model) {
Slava Shklyaevf8124a82019-12-13 12:24:35 +0000335 for (auto i : model->main.outputIndexes) {
336 auto& dims = model->main.operands[i].dimensions;
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100337 std::fill(dims.begin(), dims.end(), 0);
338 }
339}
340
Xusong Wange9da9852020-01-13 11:44:45 -0800341constexpr uint32_t kInputPoolIndex = 0;
342constexpr uint32_t kOutputPoolIndex = 1;
343constexpr uint32_t kDeviceMemoryBeginIndex = 2;
344
345static std::pair<Request, std::vector<sp<IBuffer>>> createRequest(
346 const sp<IDevice>& device, const sp<IPreparedModel>& preparedModel,
347 const TestModel& testModel, bool preferDeviceMemory) {
348 // Memory pools are organized as:
349 // - 0: Input shared memory pool
350 // - 1: Output shared memory pool
351 // - [2, 2+i): Input device memories
352 // - [2+i, 2+i+o): Output device memories
353 DeviceMemoryAllocator allocator(device, preparedModel, testModel);
354 std::vector<sp<IBuffer>> buffers;
Michael Butlerc3e1a292020-02-04 16:15:04 -0800355 std::vector<uint32_t> tokens;
Xusong Wange9da9852020-01-13 11:44:45 -0800356
357 // Model inputs.
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000358 hidl_vec<RequestArgument> inputs(testModel.main.inputIndexes.size());
Xusong Wange9da9852020-01-13 11:44:45 -0800359 size_t inputSize = 0;
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000360 for (uint32_t i = 0; i < testModel.main.inputIndexes.size(); i++) {
361 const auto& op = testModel.main.operands[testModel.main.inputIndexes[i]];
Xusong Wange9da9852020-01-13 11:44:45 -0800362 if (op.data.size() == 0) {
363 // Omitted input.
364 inputs[i] = {.hasNoValue = true};
365 continue;
366 } else if (preferDeviceMemory) {
367 SCOPED_TRACE("Input index = " + std::to_string(i));
368 auto [buffer, token] = allocator.allocate<IOType::INPUT>(i);
369 if (buffer != nullptr) {
370 DataLocation loc = {.poolIndex = static_cast<uint32_t>(buffers.size() +
371 kDeviceMemoryBeginIndex)};
372 buffers.push_back(std::move(buffer));
373 tokens.push_back(token);
374 inputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}};
375 continue;
376 }
377 }
378
379 // Reserve shared memory for input.
380 DataLocation loc = {.poolIndex = kInputPoolIndex,
381 .offset = static_cast<uint32_t>(inputSize),
382 .length = static_cast<uint32_t>(op.data.size())};
383 inputSize += op.data.alignedSize();
384 inputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}};
385 }
386
387 // Model outputs.
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000388 hidl_vec<RequestArgument> outputs(testModel.main.outputIndexes.size());
Xusong Wange9da9852020-01-13 11:44:45 -0800389 size_t outputSize = 0;
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000390 for (uint32_t i = 0; i < testModel.main.outputIndexes.size(); i++) {
391 const auto& op = testModel.main.operands[testModel.main.outputIndexes[i]];
Xusong Wange9da9852020-01-13 11:44:45 -0800392 if (preferDeviceMemory) {
393 SCOPED_TRACE("Output index = " + std::to_string(i));
394 auto [buffer, token] = allocator.allocate<IOType::OUTPUT>(i);
395 if (buffer != nullptr) {
396 DataLocation loc = {.poolIndex = static_cast<uint32_t>(buffers.size() +
397 kDeviceMemoryBeginIndex)};
398 buffers.push_back(std::move(buffer));
399 tokens.push_back(token);
400 outputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}};
401 continue;
402 }
403 }
404
405 // In the case of zero-sized output, we should at least provide a one-byte buffer.
406 // This is because zero-sized tensors are only supported internally to the driver, or
407 // reported in output shapes. It is illegal for the client to pre-specify a zero-sized
408 // tensor as model output. Otherwise, we will have two semantic conflicts:
409 // - "Zero dimension" conflicts with "unspecified dimension".
410 // - "Omitted operand buffer" conflicts with "zero-sized operand buffer".
411 size_t bufferSize = std::max<size_t>(op.data.size(), 1);
412
413 // Reserve shared memory for output.
414 DataLocation loc = {.poolIndex = kOutputPoolIndex,
415 .offset = static_cast<uint32_t>(outputSize),
416 .length = static_cast<uint32_t>(bufferSize)};
417 outputSize += op.data.size() == 0 ? TestBuffer::kAlignment : op.data.alignedSize();
418 outputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}};
419 }
420
421 // Memory pools.
422 hidl_vec<Request::MemoryPool> pools(kDeviceMemoryBeginIndex + buffers.size());
423 pools[kInputPoolIndex].hidlMemory(nn::allocateSharedMemory(std::max<size_t>(inputSize, 1)));
424 pools[kOutputPoolIndex].hidlMemory(nn::allocateSharedMemory(std::max<size_t>(outputSize, 1)));
425 CHECK_NE(pools[kInputPoolIndex].hidlMemory().size(), 0u);
426 CHECK_NE(pools[kOutputPoolIndex].hidlMemory().size(), 0u);
427 for (uint32_t i = 0; i < buffers.size(); i++) {
428 pools[kDeviceMemoryBeginIndex + i].token(tokens[i]);
429 }
430
431 // Copy input data to the input shared memory pool.
432 sp<IMemory> inputMemory = mapMemory(pools[kInputPoolIndex].hidlMemory());
433 CHECK(inputMemory.get() != nullptr);
434 uint8_t* inputPtr = static_cast<uint8_t*>(static_cast<void*>(inputMemory->getPointer()));
435 CHECK(inputPtr != nullptr);
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000436 for (uint32_t i = 0; i < testModel.main.inputIndexes.size(); i++) {
Xusong Wange9da9852020-01-13 11:44:45 -0800437 if (!inputs[i].hasNoValue && inputs[i].location.poolIndex == kInputPoolIndex) {
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000438 const auto& op = testModel.main.operands[testModel.main.inputIndexes[i]];
Xusong Wange9da9852020-01-13 11:44:45 -0800439 const uint8_t* begin = op.data.get<uint8_t>();
440 const uint8_t* end = begin + op.data.size();
441 std::copy(begin, end, inputPtr + inputs[i].location.offset);
442 }
443 }
444
445 Request request = {
446 .inputs = std::move(inputs), .outputs = std::move(outputs), .pools = std::move(pools)};
447 return {std::move(request), std::move(buffers)};
448}
449
450// Get a TestBuffer with data copied from an IBuffer object.
451static void getBuffer(const sp<IBuffer>& buffer, size_t size, TestBuffer* testBuffer) {
452 // IBuffer -> Shared memory.
453 hidl_memory tmp = nn::allocateSharedMemory(size);
454 const auto ret = buffer->copyTo(tmp);
455 ASSERT_TRUE(ret.isOk());
456 ASSERT_EQ(static_cast<ErrorStatus>(ret), ErrorStatus::NONE);
457
458 // Shared memory -> TestBuffer.
459 sp<IMemory> outputMemory = mapMemory(tmp);
460 ASSERT_NE(outputMemory.get(), nullptr);
461 uint8_t* outputPtr = static_cast<uint8_t*>(static_cast<void*>(outputMemory->getPointer()));
462 ASSERT_NE(outputPtr, nullptr);
463 ASSERT_NE(testBuffer, nullptr);
464 *testBuffer = TestBuffer(size, outputPtr);
465}
466
467static std::vector<TestBuffer> getOutputBuffers(const TestModel& testModel, const Request& request,
468 const std::vector<sp<IBuffer>>& buffers) {
469 sp<IMemory> outputMemory = mapMemory(request.pools[kOutputPoolIndex].hidlMemory());
470 CHECK(outputMemory.get() != nullptr);
471 uint8_t* outputPtr = static_cast<uint8_t*>(static_cast<void*>(outputMemory->getPointer()));
472 CHECK(outputPtr != nullptr);
473
474 // Copy out output results.
475 std::vector<TestBuffer> outputBuffers;
476 for (uint32_t i = 0; i < request.outputs.size(); i++) {
477 const auto& outputLoc = request.outputs[i].location;
478 if (outputLoc.poolIndex == kOutputPoolIndex) {
479 outputBuffers.emplace_back(outputLoc.length, outputPtr + outputLoc.offset);
480 } else {
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000481 const auto& op = testModel.main.operands[testModel.main.outputIndexes[i]];
Xusong Wange9da9852020-01-13 11:44:45 -0800482 if (op.data.size() == 0) {
483 outputBuffers.emplace_back();
484 } else {
485 SCOPED_TRACE("Output index = " + std::to_string(i));
486 const uint32_t bufferIndex = outputLoc.poolIndex - kDeviceMemoryBeginIndex;
487 TestBuffer buffer;
488 getBuffer(buffers[bufferIndex], op.data.size(), &buffer);
489 outputBuffers.push_back(std::move(buffer));
490 }
491 }
492 }
493 return outputBuffers;
494}
495
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100496static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
497 const Request& request, MeasureTiming measure,
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000498 const OptionalTimeoutDuration& loopTimeoutDuration,
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100499 sp<ExecutionCallback>& callback) {
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000500 return preparedModel->execute_1_3(request, measure, {}, loopTimeoutDuration, callback);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100501}
502static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
503 const Request& request, MeasureTiming measure,
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000504 const OptionalTimeoutDuration& loopTimeoutDuration,
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100505 hidl_vec<OutputShape>* outputShapes,
506 Timing* timing) {
507 ErrorStatus result;
Xusong Wangd4a060b2019-10-28 11:11:19 -0700508 Return<void> ret = preparedModel->executeSynchronously_1_3(
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000509 request, measure, {}, loopTimeoutDuration,
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100510 [&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes,
511 const Timing& time) {
512 result = error;
513 *outputShapes = shapes;
514 *timing = time;
515 });
516 if (!ret.isOk()) {
517 return ErrorStatus::GENERAL_FAILURE;
518 }
519 return result;
520}
521static std::shared_ptr<::android::nn::ExecutionBurstController> CreateBurst(
522 const sp<IPreparedModel>& preparedModel) {
Michael Butler648ada52019-07-25 17:22:11 -0700523 return android::nn::ExecutionBurstController::create(preparedModel,
524 std::chrono::microseconds{0});
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100525}
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100526
Xusong Wange9da9852020-01-13 11:44:45 -0800527void EvaluatePreparedModel(const sp<IDevice>& device, const sp<IPreparedModel>& preparedModel,
528 const TestModel& testModel, const TestConfig& testConfig,
529 bool* skipped = nullptr) {
Lev Proleev9226c1e2019-10-03 14:43:18 +0100530 if (skipped != nullptr) {
531 *skipped = false;
532 }
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100533 // If output0 does not have size larger than one byte, we can not test with insufficient buffer.
Lev Proleev0d4ba3f2019-10-02 17:32:06 +0100534 if (testConfig.outputType == OutputType::INSUFFICIENT &&
535 !isOutputSizeGreaterThanOne(testModel, 0)) {
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100536 return;
537 }
538
Xusong Wange9da9852020-01-13 11:44:45 -0800539 auto [request, buffers] =
540 createRequest(device, preparedModel, testModel,
541 /*preferDeviceMemory=*/testConfig.memoryType == MemoryType::DEVICE);
542 // Skip if testing memory domain but no device memory has been allocated.
543 if (testConfig.memoryType == MemoryType::DEVICE && buffers.empty()) {
544 return;
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100545 }
Xusong Wange9da9852020-01-13 11:44:45 -0800546 if (testConfig.outputType == OutputType::INSUFFICIENT) {
547 makeOutputInsufficientSize(/*outputIndex=*/0, &request);
548 }
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100549
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000550 OptionalTimeoutDuration loopTimeoutDuration;
551 // OutputType::MISSED_DEADLINE is only used by
552 // TestKind::INTINITE_LOOP_TIMEOUT tests to verify that an infinite loop is
553 // aborted after a timeout.
554 if (testConfig.outputType == OutputType::MISSED_DEADLINE) {
555 // Override the default loop timeout duration with a small value to
556 // speed up test execution.
557 constexpr uint64_t kMillisecond = 1'000'000;
558 loopTimeoutDuration.nanoseconds(1 * kMillisecond);
559 }
560
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100561 ErrorStatus executionStatus;
562 hidl_vec<OutputShape> outputShapes;
563 Timing timing;
Lev Proleev0d4ba3f2019-10-02 17:32:06 +0100564 switch (testConfig.executor) {
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100565 case Executor::ASYNC: {
566 SCOPED_TRACE("asynchronous");
567
568 // launch execution
569 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000570 Return<ErrorStatus> executionLaunchStatus =
571 ExecutePreparedModel(preparedModel, request, testConfig.measureTiming,
572 loopTimeoutDuration, executionCallback);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100573 ASSERT_TRUE(executionLaunchStatus.isOk());
574 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
575
576 // retrieve execution status
577 executionCallback->wait();
578 executionStatus = executionCallback->getStatus();
579 outputShapes = executionCallback->getOutputShapes();
580 timing = executionCallback->getTiming();
581
582 break;
583 }
584 case Executor::SYNC: {
585 SCOPED_TRACE("synchronous");
586
587 // execute
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000588 Return<ErrorStatus> executionReturnStatus =
589 ExecutePreparedModel(preparedModel, request, testConfig.measureTiming,
590 loopTimeoutDuration, &outputShapes, &timing);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100591 ASSERT_TRUE(executionReturnStatus.isOk());
592 executionStatus = static_cast<ErrorStatus>(executionReturnStatus);
593
594 break;
595 }
596 case Executor::BURST: {
Xusong Wangb345a462019-11-27 12:46:48 -0800597 // TODO(butlermichael): Check if we need to test burst in V1_3 if the interface remains
598 // V1_2.
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100599 SCOPED_TRACE("burst");
600
Xusong Wange9da9852020-01-13 11:44:45 -0800601 // check compliance
602 ASSERT_TRUE(nn::compliantWithV1_0(request));
603 V1_0::Request request10 = nn::convertToV1_0(request);
604
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100605 // create burst
606 const std::shared_ptr<::android::nn::ExecutionBurstController> controller =
607 CreateBurst(preparedModel);
608 ASSERT_NE(nullptr, controller.get());
609
610 // create memory keys
Xusong Wangb345a462019-11-27 12:46:48 -0800611 std::vector<intptr_t> keys(request10.pools.size());
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100612 for (size_t i = 0; i < keys.size(); ++i) {
Xusong Wangb345a462019-11-27 12:46:48 -0800613 keys[i] = reinterpret_cast<intptr_t>(&request10.pools[i]);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100614 }
615
616 // execute burst
Michael Butler648ada52019-07-25 17:22:11 -0700617 int n;
618 std::tie(n, outputShapes, timing, std::ignore) =
Xusong Wangb345a462019-11-27 12:46:48 -0800619 controller->compute(request10, testConfig.measureTiming, keys);
Michael Butler648ada52019-07-25 17:22:11 -0700620 executionStatus = nn::convertResultCodeToErrorStatus(n);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100621
622 break;
623 }
Miao Wang2b5c4cd2019-12-26 18:03:56 -0800624 case Executor::FENCED: {
625 SCOPED_TRACE("fenced");
626 ErrorStatus result;
Miao Wang99753d92020-01-21 13:15:09 -0800627 hidl_handle syncFenceHandle;
628 sp<IFencedExecutionCallback> fencedCallback;
Miao Wang2d74d7d2020-02-25 15:54:08 -0800629 auto callbackFunc = [&result, &syncFenceHandle, &fencedCallback](
630 ErrorStatus error, const hidl_handle& handle,
631 const sp<IFencedExecutionCallback>& callback) {
632 result = error;
633 syncFenceHandle = handle;
634 fencedCallback = callback;
635 };
636 Return<void> ret =
637 preparedModel->executeFenced(request, {}, testConfig.measureTiming, {},
638 loopTimeoutDuration, {}, callbackFunc);
Miao Wang2b5c4cd2019-12-26 18:03:56 -0800639 ASSERT_TRUE(ret.isOk());
640 if (result != ErrorStatus::NONE) {
Miao Wang99753d92020-01-21 13:15:09 -0800641 ASSERT_EQ(syncFenceHandle.getNativeHandle(), nullptr);
642 ASSERT_EQ(fencedCallback, nullptr);
Miao Wang40689072020-02-25 16:57:53 -0800643 executionStatus = result;
Miao Wang99753d92020-01-21 13:15:09 -0800644 } else if (syncFenceHandle.getNativeHandle()) {
Miao Wang2d74d7d2020-02-25 15:54:08 -0800645 // If a sync fence is returned, try start another run waiting for the sync fence.
646 ret = preparedModel->executeFenced(request, {syncFenceHandle},
647 testConfig.measureTiming, {},
648 loopTimeoutDuration, {}, callbackFunc);
649 ASSERT_TRUE(ret.isOk());
650 ASSERT_EQ(result, ErrorStatus::NONE);
Miao Wang99753d92020-01-21 13:15:09 -0800651 waitForSyncFence(syncFenceHandle.getNativeHandle()->data[0]);
Miao Wang2b5c4cd2019-12-26 18:03:56 -0800652 }
653 if (result == ErrorStatus::NONE) {
Miao Wang99753d92020-01-21 13:15:09 -0800654 ASSERT_NE(fencedCallback, nullptr);
655 Return<void> ret = fencedCallback->getExecutionInfo(
656 [&executionStatus, &timing](ErrorStatus error, Timing t, Timing) {
Miao Wang2b5c4cd2019-12-26 18:03:56 -0800657 executionStatus = error;
658 timing = t;
659 });
660 ASSERT_TRUE(ret.isOk());
661 }
662 break;
663 }
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100664 }
665
Miao Wang40689072020-02-25 16:57:53 -0800666 if (testConfig.outputType != OutputType::FULLY_SPECIFIED &&
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100667 executionStatus == ErrorStatus::GENERAL_FAILURE) {
Lev Proleev9226c1e2019-10-03 14:43:18 +0100668 if (skipped != nullptr) {
669 *skipped = true;
670 }
671 if (!testConfig.reportSkipping) {
672 return;
673 }
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100674 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
675 "execute model that it does not support.";
676 std::cout << "[ ] Early termination of test because vendor service cannot "
677 "execute model that it does not support."
678 << std::endl;
679 GTEST_SKIP();
680 }
Lev Proleev0d4ba3f2019-10-02 17:32:06 +0100681 if (testConfig.measureTiming == MeasureTiming::NO) {
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100682 EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);
683 EXPECT_EQ(UINT64_MAX, timing.timeInDriver);
684 } else {
685 if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) {
686 EXPECT_LE(timing.timeOnDevice, timing.timeInDriver);
687 }
688 }
689
Lev Proleev0d4ba3f2019-10-02 17:32:06 +0100690 switch (testConfig.outputType) {
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100691 case OutputType::FULLY_SPECIFIED:
692 // If the model output operands are fully specified, outputShapes must be either
693 // either empty, or have the same number of elements as the number of outputs.
694 ASSERT_EQ(ErrorStatus::NONE, executionStatus);
695 ASSERT_TRUE(outputShapes.size() == 0 ||
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000696 outputShapes.size() == testModel.main.outputIndexes.size());
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100697 break;
698 case OutputType::UNSPECIFIED:
Miao Wang40689072020-02-25 16:57:53 -0800699 if (testConfig.executor == Executor::FENCED) {
700 // For Executor::FENCED, the output shape must be fully specified.
701 ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionStatus);
702 return;
703 }
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100704 // If the model output operands are not fully specified, outputShapes must have
705 // the same number of elements as the number of outputs.
706 ASSERT_EQ(ErrorStatus::NONE, executionStatus);
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000707 ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size());
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100708 break;
709 case OutputType::INSUFFICIENT:
Miao Wang40689072020-02-25 16:57:53 -0800710 if (testConfig.executor == Executor::FENCED) {
711 // For Executor::FENCED, the output shape must be fully specified.
712 ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionStatus);
713 return;
714 }
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100715 ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus);
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000716 ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size());
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100717 ASSERT_FALSE(outputShapes[0].isSufficient);
718 return;
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000719 case OutputType::MISSED_DEADLINE:
720 ASSERT_TRUE(executionStatus == ErrorStatus::MISSED_DEADLINE_TRANSIENT ||
721 executionStatus == ErrorStatus::MISSED_DEADLINE_PERSISTENT)
722 << "executionStatus = " << executionStatus;
723 return;
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100724 }
725
726 // Go through all outputs, check returned output shapes.
727 for (uint32_t i = 0; i < outputShapes.size(); i++) {
728 EXPECT_TRUE(outputShapes[i].isSufficient);
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000729 const auto& expect = testModel.main.operands[testModel.main.outputIndexes[i]].dimensions;
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100730 const std::vector<uint32_t> actual = outputShapes[i].dimensions;
731 EXPECT_EQ(expect, actual);
732 }
733
734 // Retrieve execution results.
Xusong Wange9da9852020-01-13 11:44:45 -0800735 const std::vector<TestBuffer> outputs = getOutputBuffers(testModel, request, buffers);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100736
737 // We want "close-enough" results.
738 checkResults(testModel, outputs);
739}
740
Xusong Wange9da9852020-01-13 11:44:45 -0800741void EvaluatePreparedModel(const sp<IDevice>& device, const sp<IPreparedModel>& preparedModel,
742 const TestModel& testModel, TestKind testKind) {
Lev Proleev56cda832019-12-05 14:49:47 +0000743 std::vector<OutputType> outputTypesList;
744 std::vector<MeasureTiming> measureTimingList;
745 std::vector<Executor> executorList;
Xusong Wange9da9852020-01-13 11:44:45 -0800746 MemoryType memoryType = MemoryType::SHARED;
Lev Proleev0d4ba3f2019-10-02 17:32:06 +0100747
Lev Proleev9226c1e2019-10-03 14:43:18 +0100748 switch (testKind) {
749 case TestKind::GENERAL: {
750 outputTypesList = {OutputType::FULLY_SPECIFIED};
751 measureTimingList = {MeasureTiming::NO, MeasureTiming::YES};
Miao Wange51e4a92020-01-23 13:53:21 -0800752 executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST};
Lev Proleev9226c1e2019-10-03 14:43:18 +0100753 } break;
754 case TestKind::DYNAMIC_SHAPE: {
755 outputTypesList = {OutputType::UNSPECIFIED, OutputType::INSUFFICIENT};
756 measureTimingList = {MeasureTiming::NO, MeasureTiming::YES};
Miao Wang40689072020-02-25 16:57:53 -0800757 executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST, Executor::FENCED};
Lev Proleev9226c1e2019-10-03 14:43:18 +0100758 } break;
Xusong Wange9da9852020-01-13 11:44:45 -0800759 case TestKind::MEMORY_DOMAIN: {
760 outputTypesList = {OutputType::FULLY_SPECIFIED};
761 measureTimingList = {MeasureTiming::NO};
Miao Wang2d74d7d2020-02-25 15:54:08 -0800762 executorList = {Executor::ASYNC, Executor::SYNC, Executor::FENCED};
Xusong Wange9da9852020-01-13 11:44:45 -0800763 memoryType = MemoryType::DEVICE;
764 } break;
Miao Wange51e4a92020-01-23 13:53:21 -0800765 case TestKind::FENCED_COMPUTE: {
766 outputTypesList = {OutputType::FULLY_SPECIFIED};
767 measureTimingList = {MeasureTiming::NO, MeasureTiming::YES};
768 executorList = {Executor::FENCED};
769 } break;
Lev Proleev9226c1e2019-10-03 14:43:18 +0100770 case TestKind::QUANTIZATION_COUPLING: {
771 LOG(FATAL) << "Wrong TestKind for EvaluatePreparedModel";
772 return;
773 } break;
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000774 case TestKind::INTINITE_LOOP_TIMEOUT: {
775 outputTypesList = {OutputType::MISSED_DEADLINE};
776 measureTimingList = {MeasureTiming::NO, MeasureTiming::YES};
777 // Burst does not support V1_3 loop timeout.
778 executorList = {Executor::ASYNC, Executor::SYNC, Executor::FENCED};
779 } break;
Lev Proleev0d4ba3f2019-10-02 17:32:06 +0100780 }
781
782 for (const OutputType outputType : outputTypesList) {
783 for (const MeasureTiming measureTiming : measureTimingList) {
784 for (const Executor executor : executorList) {
Xusong Wange9da9852020-01-13 11:44:45 -0800785 const TestConfig testConfig(executor, measureTiming, outputType, memoryType);
786 EvaluatePreparedModel(device, preparedModel, testModel, testConfig);
Lev Proleev0d4ba3f2019-10-02 17:32:06 +0100787 }
788 }
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100789 }
790}
791
Xusong Wange9da9852020-01-13 11:44:45 -0800792void EvaluatePreparedCoupledModels(const sp<IDevice>& device,
793 const sp<IPreparedModel>& preparedModel,
Lev Proleev9226c1e2019-10-03 14:43:18 +0100794 const TestModel& testModel,
795 const sp<IPreparedModel>& preparedCoupledModel,
796 const TestModel& coupledModel) {
Lev Proleev56cda832019-12-05 14:49:47 +0000797 const std::vector<OutputType> outputTypesList = {OutputType::FULLY_SPECIFIED};
798 const std::vector<MeasureTiming> measureTimingList = {MeasureTiming::NO, MeasureTiming::YES};
Miao Wang2b5c4cd2019-12-26 18:03:56 -0800799 const std::vector<Executor> executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST,
800 Executor::FENCED};
Lev Proleev9226c1e2019-10-03 14:43:18 +0100801
802 for (const OutputType outputType : outputTypesList) {
803 for (const MeasureTiming measureTiming : measureTimingList) {
804 for (const Executor executor : executorList) {
Xusong Wange9da9852020-01-13 11:44:45 -0800805 const TestConfig testConfig(executor, measureTiming, outputType, MemoryType::SHARED,
Lev Proleev9226c1e2019-10-03 14:43:18 +0100806 /*reportSkipping=*/false);
807 bool baseSkipped = false;
Xusong Wange9da9852020-01-13 11:44:45 -0800808 EvaluatePreparedModel(device, preparedModel, testModel, testConfig, &baseSkipped);
Lev Proleev9226c1e2019-10-03 14:43:18 +0100809 bool coupledSkipped = false;
Xusong Wange9da9852020-01-13 11:44:45 -0800810 EvaluatePreparedModel(device, preparedCoupledModel, coupledModel, testConfig,
Lev Proleev9226c1e2019-10-03 14:43:18 +0100811 &coupledSkipped);
812 ASSERT_EQ(baseSkipped, coupledSkipped);
813 if (baseSkipped) {
814 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
815 "execute model that it does not support.";
816 std::cout << "[ ] Early termination of test because vendor service "
817 "cannot "
818 "execute model that it does not support."
819 << std::endl;
820 GTEST_SKIP();
821 }
822 }
823 }
824 }
825}
826
827void Execute(const sp<IDevice>& device, const TestModel& testModel, TestKind testKind) {
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100828 Model model = createModel(testModel);
Lev Proleev9226c1e2019-10-03 14:43:18 +0100829 if (testKind == TestKind::DYNAMIC_SHAPE) {
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100830 makeOutputDimensionsUnspecified(&model);
831 }
832
833 sp<IPreparedModel> preparedModel;
Lev Proleev9226c1e2019-10-03 14:43:18 +0100834 switch (testKind) {
Xusong Wange9da9852020-01-13 11:44:45 -0800835 case TestKind::GENERAL:
836 case TestKind::DYNAMIC_SHAPE:
Miao Wange51e4a92020-01-23 13:53:21 -0800837 case TestKind::MEMORY_DOMAIN:
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000838 case TestKind::FENCED_COMPUTE:
839 case TestKind::INTINITE_LOOP_TIMEOUT: {
Lev Proleev9226c1e2019-10-03 14:43:18 +0100840 createPreparedModel(device, model, &preparedModel);
841 if (preparedModel == nullptr) return;
Xusong Wange9da9852020-01-13 11:44:45 -0800842 EvaluatePreparedModel(device, preparedModel, testModel, testKind);
Lev Proleev9226c1e2019-10-03 14:43:18 +0100843 } break;
844 case TestKind::QUANTIZATION_COUPLING: {
Lev Proleev673fdcf2020-01-02 18:22:30 +0000845 ASSERT_TRUE(testModel.hasQuant8CoupledOperands());
Michael Butler95899b32020-01-07 14:52:44 -0800846 createPreparedModel(device, model, &preparedModel,
847 /*reportSkipping*/ false);
Lev Proleev9226c1e2019-10-03 14:43:18 +0100848 TestModel signedQuantizedModel = convertQuant8AsymmOperandsToSigned(testModel);
849 sp<IPreparedModel> preparedCoupledModel;
850 createPreparedModel(device, createModel(signedQuantizedModel), &preparedCoupledModel,
851 /*reportSkipping*/ false);
852 // If we couldn't prepare a model with unsigned quantization, we must
853 // fail to prepare a model with signed quantization as well.
854 if (preparedModel == nullptr) {
855 ASSERT_EQ(preparedCoupledModel, nullptr);
856 // If we failed to prepare both of the models, we can safely skip
857 // the test.
858 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
859 "prepare model that it does not support.";
860 std::cout
861 << "[ ] Early termination of test because vendor service cannot "
862 "prepare model that it does not support."
863 << std::endl;
864 GTEST_SKIP();
865 }
866 ASSERT_NE(preparedCoupledModel, nullptr);
Xusong Wange9da9852020-01-13 11:44:45 -0800867 EvaluatePreparedCoupledModels(device, preparedModel, testModel, preparedCoupledModel,
Lev Proleev9226c1e2019-10-03 14:43:18 +0100868 signedQuantizedModel);
869 } break;
870 }
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100871}
872
873void GeneratedTestBase::SetUp() {
874 testing::TestWithParam<GeneratedTestParam>::SetUp();
875 ASSERT_NE(kDevice, nullptr);
876}
877
878std::vector<NamedModel> getNamedModels(const FilterFn& filter) {
879 return TestModelManager::get().getTestModels(filter);
880}
881
882std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info) {
883 const auto& [namedDevice, namedModel] = info.param;
884 return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel));
885}
886
887// Tag for the generated tests
888class GeneratedTest : public GeneratedTestBase {};
889
890// Tag for the dynamic output shape tests
891class DynamicOutputShapeTest : public GeneratedTest {};
892
Xusong Wange9da9852020-01-13 11:44:45 -0800893// Tag for the memory domain tests
894class MemoryDomainTest : public GeneratedTest {};
895
Miao Wange51e4a92020-01-23 13:53:21 -0800896// Tag for the fenced compute tests
897class FencedComputeTest : public GeneratedTest {};
898
Lev Proleev9226c1e2019-10-03 14:43:18 +0100899// Tag for the dynamic output shape tests
Lev Proleev3c68b342020-01-09 16:37:28 +0000900class QuantizationCouplingTest : public GeneratedTest {};
Lev Proleev9226c1e2019-10-03 14:43:18 +0100901
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000902// Tag for the loop timeout tests
903class InfiniteLoopTimeoutTest : public GeneratedTest {};
904
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100905TEST_P(GeneratedTest, Test) {
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000906 Execute(kDevice, kTestModel, TestKind::GENERAL);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100907}
908
909TEST_P(DynamicOutputShapeTest, Test) {
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000910 Execute(kDevice, kTestModel, TestKind::DYNAMIC_SHAPE);
Lev Proleev9226c1e2019-10-03 14:43:18 +0100911}
912
Xusong Wange9da9852020-01-13 11:44:45 -0800913TEST_P(MemoryDomainTest, Test) {
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000914 Execute(kDevice, kTestModel, TestKind::MEMORY_DOMAIN);
Xusong Wange9da9852020-01-13 11:44:45 -0800915}
916
Miao Wange51e4a92020-01-23 13:53:21 -0800917TEST_P(FencedComputeTest, Test) {
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000918 Execute(kDevice, kTestModel, TestKind::FENCED_COMPUTE);
Miao Wange51e4a92020-01-23 13:53:21 -0800919}
920
Lev Proleev3c68b342020-01-09 16:37:28 +0000921TEST_P(QuantizationCouplingTest, Test) {
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000922 Execute(kDevice, kTestModel, TestKind::QUANTIZATION_COUPLING);
923}
924
925TEST_P(InfiniteLoopTimeoutTest, Test) {
926 Execute(kDevice, kTestModel, TestKind::INTINITE_LOOP_TIMEOUT);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100927}
928
929INSTANTIATE_GENERATED_TEST(GeneratedTest,
930 [](const TestModel& testModel) { return !testModel.expectFailure; });
931
Lev Proleev53a51cb2020-01-20 18:54:46 +0000932INSTANTIATE_GENERATED_TEST(DynamicOutputShapeTest, [](const TestModel& testModel) {
933 return !testModel.expectFailure && !testModel.hasScalarOutputs();
934});
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100935
Xusong Wange9da9852020-01-13 11:44:45 -0800936INSTANTIATE_GENERATED_TEST(MemoryDomainTest,
937 [](const TestModel& testModel) { return !testModel.expectFailure; });
938
Miao Wang40689072020-02-25 16:57:53 -0800939INSTANTIATE_GENERATED_TEST(FencedComputeTest, [](const TestModel& testModel) {
940 return !testModel.expectFailure &&
941 std::all_of(testModel.main.outputIndexes.begin(), testModel.main.outputIndexes.end(),
942 [&testModel](uint32_t index) {
943 return testModel.main.operands[index].data.size() > 0;
944 });
945});
Miao Wange51e4a92020-01-23 13:53:21 -0800946
Lev Proleev3c68b342020-01-09 16:37:28 +0000947INSTANTIATE_GENERATED_TEST(QuantizationCouplingTest, [](const TestModel& testModel) {
Slava Shklyaev0fff59b2020-01-31 15:14:24 +0000948 return testModel.hasQuant8CoupledOperands() && testModel.main.operations.size() == 1;
Lev Proleev9226c1e2019-10-03 14:43:18 +0100949});
950
Slava Shklyaeva37c2c82020-02-18 16:11:32 +0000951INSTANTIATE_GENERATED_TEST(InfiniteLoopTimeoutTest, [](const TestModel& testModel) {
952 return testModel.isInfiniteLoopTimeoutTest();
953});
954
Lev Proleev26d1bc82019-08-30 11:57:18 +0100955} // namespace android::hardware::neuralnetworks::V1_3::vts::functional