blob: 8a7ed24c9df5cf2cac1a05edd4c9abc4e12a5cb5 [file] [log] [blame]
Lev Proleev13fdfcd2019-08-30 11:35:34 +01001/*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "GeneratedTestHarness.h"
18
19#include <android-base/logging.h>
20#include <android/hardware/neuralnetworks/1.0/IDevice.h>
21#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
22#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
23#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
24#include <android/hardware/neuralnetworks/1.0/types.h>
25#include <android/hardware/neuralnetworks/1.1/IDevice.h>
26#include <android/hardware/neuralnetworks/1.2/IDevice.h>
27#include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h>
28#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
29#include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h>
Lev Proleev26d1bc82019-08-30 11:57:18 +010030#include <android/hardware/neuralnetworks/1.2/types.h>
31#include <android/hardware/neuralnetworks/1.3/IDevice.h>
32#include <android/hardware/neuralnetworks/1.3/types.h>
Lev Proleev13fdfcd2019-08-30 11:35:34 +010033#include <android/hidl/allocator/1.0/IAllocator.h>
34#include <android/hidl/memory/1.0/IMemory.h>
35#include <hidlmemory/mapping.h>
36
37#include <gtest/gtest.h>
38#include <algorithm>
Michael Butler648ada52019-07-25 17:22:11 -070039#include <chrono>
Lev Proleev13fdfcd2019-08-30 11:35:34 +010040#include <iostream>
41#include <numeric>
42
43#include "1.0/Utils.h"
44#include "1.2/Callbacks.h"
45#include "ExecutionBurstController.h"
46#include "MemoryUtils.h"
47#include "TestHarness.h"
48#include "Utils.h"
49#include "VtsHalNeuralnetworks.h"
50
Lev Proleev26d1bc82019-08-30 11:57:18 +010051namespace android::hardware::neuralnetworks::V1_3::vts::functional {
Lev Proleev13fdfcd2019-08-30 11:35:34 +010052
53using namespace test_helper;
54using hidl::memory::V1_0::IMemory;
Lev Proleev13fdfcd2019-08-30 11:35:34 +010055using V1_0::DataLocation;
56using V1_0::ErrorStatus;
57using V1_0::OperandLifeTime;
58using V1_0::Request;
59using V1_1::ExecutionPreference;
Lev Proleev26d1bc82019-08-30 11:57:18 +010060using V1_2::Constant;
61using V1_2::IPreparedModel;
62using V1_2::MeasureTiming;
63using V1_2::OperationType;
64using V1_2::OutputShape;
65using V1_2::SymmPerChannelQuantParams;
66using V1_2::Timing;
67using V1_2::implementation::ExecutionCallback;
68using V1_2::implementation::PreparedModelCallback;
Lev Proleev13fdfcd2019-08-30 11:35:34 +010069using HidlToken = hidl_array<uint8_t, static_cast<uint32_t>(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
70
71enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT };
72
73Model createModel(const TestModel& testModel) {
74 // Model operands.
75 hidl_vec<Operand> operands(testModel.operands.size());
76 size_t constCopySize = 0, constRefSize = 0;
77 for (uint32_t i = 0; i < testModel.operands.size(); i++) {
78 const auto& op = testModel.operands[i];
79
80 DataLocation loc = {};
81 if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
82 loc = {.poolIndex = 0,
83 .offset = static_cast<uint32_t>(constCopySize),
84 .length = static_cast<uint32_t>(op.data.size())};
85 constCopySize += op.data.alignedSize();
86 } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
87 loc = {.poolIndex = 0,
88 .offset = static_cast<uint32_t>(constRefSize),
89 .length = static_cast<uint32_t>(op.data.size())};
90 constRefSize += op.data.alignedSize();
91 }
92
93 Operand::ExtraParams extraParams;
94 if (op.type == TestOperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
95 extraParams.channelQuant(SymmPerChannelQuantParams{
96 .scales = op.channelQuant.scales, .channelDim = op.channelQuant.channelDim});
97 }
98
99 operands[i] = {.type = static_cast<OperandType>(op.type),
100 .dimensions = op.dimensions,
101 .numberOfConsumers = op.numberOfConsumers,
102 .scale = op.scale,
103 .zeroPoint = op.zeroPoint,
104 .lifetime = static_cast<OperandLifeTime>(op.lifetime),
105 .location = loc,
106 .extraParams = std::move(extraParams)};
107 }
108
109 // Model operations.
110 hidl_vec<Operation> operations(testModel.operations.size());
111 std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),
112 [](const TestOperation& op) -> Operation {
113 return {.type = static_cast<OperationType>(op.type),
114 .inputs = op.inputs,
115 .outputs = op.outputs};
116 });
117
118 // Constant copies.
119 hidl_vec<uint8_t> operandValues(constCopySize);
120 for (uint32_t i = 0; i < testModel.operands.size(); i++) {
121 const auto& op = testModel.operands[i];
122 if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
123 const uint8_t* begin = op.data.get<uint8_t>();
124 const uint8_t* end = begin + op.data.size();
125 std::copy(begin, end, operandValues.data() + operands[i].location.offset);
126 }
127 }
128
129 // Shared memory.
130 hidl_vec<hidl_memory> pools = {};
131 if (constRefSize > 0) {
132 hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
133 CHECK_NE(pools[0].size(), 0u);
134
135 // load data
136 sp<IMemory> mappedMemory = mapMemory(pools[0]);
137 CHECK(mappedMemory.get() != nullptr);
138 uint8_t* mappedPtr =
139 reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));
140 CHECK(mappedPtr != nullptr);
141
142 for (uint32_t i = 0; i < testModel.operands.size(); i++) {
143 const auto& op = testModel.operands[i];
144 if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
145 const uint8_t* begin = op.data.get<uint8_t>();
146 const uint8_t* end = begin + op.data.size();
147 std::copy(begin, end, mappedPtr + operands[i].location.offset);
148 }
149 }
150 }
151
152 return {.operands = std::move(operands),
153 .operations = std::move(operations),
154 .inputIndexes = testModel.inputIndexes,
155 .outputIndexes = testModel.outputIndexes,
156 .operandValues = std::move(operandValues),
157 .pools = std::move(pools),
158 .relaxComputationFloat32toFloat16 = testModel.isRelaxed};
159}
160
161static bool isOutputSizeGreaterThanOne(const TestModel& testModel, uint32_t index) {
162 const auto byteSize = testModel.operands[testModel.outputIndexes[index]].data.size();
163 return byteSize > 1u;
164}
165
166static void makeOutputInsufficientSize(uint32_t outputIndex, Request* request) {
167 auto& length = request->outputs[outputIndex].location.length;
168 ASSERT_GT(length, 1u);
169 length -= 1u;
170}
171
172static void makeOutputDimensionsUnspecified(Model* model) {
173 for (auto i : model->outputIndexes) {
174 auto& dims = model->operands[i].dimensions;
175 std::fill(dims.begin(), dims.end(), 0);
176 }
177}
178
179static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
180 const Request& request, MeasureTiming measure,
181 sp<ExecutionCallback>& callback) {
182 return preparedModel->execute_1_2(request, measure, callback);
183}
184static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
185 const Request& request, MeasureTiming measure,
186 hidl_vec<OutputShape>* outputShapes,
187 Timing* timing) {
188 ErrorStatus result;
189 Return<void> ret = preparedModel->executeSynchronously(
190 request, measure,
191 [&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes,
192 const Timing& time) {
193 result = error;
194 *outputShapes = shapes;
195 *timing = time;
196 });
197 if (!ret.isOk()) {
198 return ErrorStatus::GENERAL_FAILURE;
199 }
200 return result;
201}
202static std::shared_ptr<::android::nn::ExecutionBurstController> CreateBurst(
203 const sp<IPreparedModel>& preparedModel) {
Michael Butler648ada52019-07-25 17:22:11 -0700204 return android::nn::ExecutionBurstController::create(preparedModel,
205 std::chrono::microseconds{0});
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100206}
207enum class Executor { ASYNC, SYNC, BURST };
208
209void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
210 Executor executor, MeasureTiming measure, OutputType outputType) {
211 // If output0 does not have size larger than one byte, we can not test with insufficient buffer.
212 if (outputType == OutputType::INSUFFICIENT && !isOutputSizeGreaterThanOne(testModel, 0)) {
213 return;
214 }
215
216 Request request = createRequest(testModel);
217 if (outputType == OutputType::INSUFFICIENT) {
218 makeOutputInsufficientSize(/*outputIndex=*/0, &request);
219 }
220
221 ErrorStatus executionStatus;
222 hidl_vec<OutputShape> outputShapes;
223 Timing timing;
224 switch (executor) {
225 case Executor::ASYNC: {
226 SCOPED_TRACE("asynchronous");
227
228 // launch execution
229 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
230 Return<ErrorStatus> executionLaunchStatus =
231 ExecutePreparedModel(preparedModel, request, measure, executionCallback);
232 ASSERT_TRUE(executionLaunchStatus.isOk());
233 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
234
235 // retrieve execution status
236 executionCallback->wait();
237 executionStatus = executionCallback->getStatus();
238 outputShapes = executionCallback->getOutputShapes();
239 timing = executionCallback->getTiming();
240
241 break;
242 }
243 case Executor::SYNC: {
244 SCOPED_TRACE("synchronous");
245
246 // execute
247 Return<ErrorStatus> executionReturnStatus =
248 ExecutePreparedModel(preparedModel, request, measure, &outputShapes, &timing);
249 ASSERT_TRUE(executionReturnStatus.isOk());
250 executionStatus = static_cast<ErrorStatus>(executionReturnStatus);
251
252 break;
253 }
254 case Executor::BURST: {
255 SCOPED_TRACE("burst");
256
257 // create burst
258 const std::shared_ptr<::android::nn::ExecutionBurstController> controller =
259 CreateBurst(preparedModel);
260 ASSERT_NE(nullptr, controller.get());
261
262 // create memory keys
263 std::vector<intptr_t> keys(request.pools.size());
264 for (size_t i = 0; i < keys.size(); ++i) {
265 keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
266 }
267
268 // execute burst
Michael Butler648ada52019-07-25 17:22:11 -0700269 int n;
270 std::tie(n, outputShapes, timing, std::ignore) =
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100271 controller->compute(request, measure, keys);
Michael Butler648ada52019-07-25 17:22:11 -0700272 executionStatus = nn::convertResultCodeToErrorStatus(n);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100273
274 break;
275 }
276 }
277
278 if (outputType != OutputType::FULLY_SPECIFIED &&
279 executionStatus == ErrorStatus::GENERAL_FAILURE) {
280 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
281 "execute model that it does not support.";
282 std::cout << "[ ] Early termination of test because vendor service cannot "
283 "execute model that it does not support."
284 << std::endl;
285 GTEST_SKIP();
286 }
287 if (measure == MeasureTiming::NO) {
288 EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);
289 EXPECT_EQ(UINT64_MAX, timing.timeInDriver);
290 } else {
291 if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) {
292 EXPECT_LE(timing.timeOnDevice, timing.timeInDriver);
293 }
294 }
295
296 switch (outputType) {
297 case OutputType::FULLY_SPECIFIED:
298 // If the model output operands are fully specified, outputShapes must be either
299 // either empty, or have the same number of elements as the number of outputs.
300 ASSERT_EQ(ErrorStatus::NONE, executionStatus);
301 ASSERT_TRUE(outputShapes.size() == 0 ||
302 outputShapes.size() == testModel.outputIndexes.size());
303 break;
304 case OutputType::UNSPECIFIED:
305 // If the model output operands are not fully specified, outputShapes must have
306 // the same number of elements as the number of outputs.
307 ASSERT_EQ(ErrorStatus::NONE, executionStatus);
308 ASSERT_EQ(outputShapes.size(), testModel.outputIndexes.size());
309 break;
310 case OutputType::INSUFFICIENT:
311 ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus);
312 ASSERT_EQ(outputShapes.size(), testModel.outputIndexes.size());
313 ASSERT_FALSE(outputShapes[0].isSufficient);
314 return;
315 }
316
317 // Go through all outputs, check returned output shapes.
318 for (uint32_t i = 0; i < outputShapes.size(); i++) {
319 EXPECT_TRUE(outputShapes[i].isSufficient);
320 const auto& expect = testModel.operands[testModel.outputIndexes[i]].dimensions;
321 const std::vector<uint32_t> actual = outputShapes[i].dimensions;
322 EXPECT_EQ(expect, actual);
323 }
324
325 // Retrieve execution results.
326 const std::vector<TestBuffer> outputs = getOutputBuffers(request);
327
328 // We want "close-enough" results.
329 checkResults(testModel, outputs);
330}
331
332void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
333 bool testDynamicOutputShape) {
334 if (testDynamicOutputShape) {
335 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
336 OutputType::UNSPECIFIED);
337 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
338 OutputType::UNSPECIFIED);
339 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
340 OutputType::UNSPECIFIED);
341 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
342 OutputType::UNSPECIFIED);
343 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
344 OutputType::UNSPECIFIED);
345 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
346 OutputType::UNSPECIFIED);
347 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
348 OutputType::INSUFFICIENT);
349 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
350 OutputType::INSUFFICIENT);
351 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
352 OutputType::INSUFFICIENT);
353 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
354 OutputType::INSUFFICIENT);
355 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
356 OutputType::INSUFFICIENT);
357 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
358 OutputType::INSUFFICIENT);
359 } else {
360 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
361 OutputType::FULLY_SPECIFIED);
362 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
363 OutputType::FULLY_SPECIFIED);
364 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
365 OutputType::FULLY_SPECIFIED);
366 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
367 OutputType::FULLY_SPECIFIED);
368 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
369 OutputType::FULLY_SPECIFIED);
370 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
371 OutputType::FULLY_SPECIFIED);
372 }
373}
374
375void Execute(const sp<IDevice>& device, const TestModel& testModel, bool testDynamicOutputShape) {
376 Model model = createModel(testModel);
377 if (testDynamicOutputShape) {
378 makeOutputDimensionsUnspecified(&model);
379 }
380
381 sp<IPreparedModel> preparedModel;
382 createPreparedModel(device, model, &preparedModel);
383 if (preparedModel == nullptr) return;
384
385 EvaluatePreparedModel(preparedModel, testModel, testDynamicOutputShape);
386}
387
388void GeneratedTestBase::SetUp() {
389 testing::TestWithParam<GeneratedTestParam>::SetUp();
390 ASSERT_NE(kDevice, nullptr);
391}
392
393std::vector<NamedModel> getNamedModels(const FilterFn& filter) {
394 return TestModelManager::get().getTestModels(filter);
395}
396
397std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info) {
398 const auto& [namedDevice, namedModel] = info.param;
399 return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel));
400}
401
402// Tag for the generated tests
403class GeneratedTest : public GeneratedTestBase {};
404
405// Tag for the dynamic output shape tests
406class DynamicOutputShapeTest : public GeneratedTest {};
407
408TEST_P(GeneratedTest, Test) {
409 Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/false);
410}
411
412TEST_P(DynamicOutputShapeTest, Test) {
413 Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/true);
414}
415
416INSTANTIATE_GENERATED_TEST(GeneratedTest,
417 [](const TestModel& testModel) { return !testModel.expectFailure; });
418
419INSTANTIATE_GENERATED_TEST(DynamicOutputShapeTest,
420 [](const TestModel& testModel) { return !testModel.expectFailure; });
421
Lev Proleev26d1bc82019-08-30 11:57:18 +0100422} // namespace android::hardware::neuralnetworks::V1_3::vts::functional