blob: a1e04c58de5db86439bb86ca7410f795014cd19a [file] [log] [blame]
Lev Proleev13fdfcd2019-08-30 11:35:34 +01001/*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "GeneratedTestHarness.h"
18
19#include <android-base/logging.h>
20#include <android/hardware/neuralnetworks/1.0/IDevice.h>
21#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
22#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
23#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
24#include <android/hardware/neuralnetworks/1.0/types.h>
25#include <android/hardware/neuralnetworks/1.1/IDevice.h>
26#include <android/hardware/neuralnetworks/1.2/IDevice.h>
27#include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h>
28#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
29#include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h>
Lev Proleev26d1bc82019-08-30 11:57:18 +010030#include <android/hardware/neuralnetworks/1.2/types.h>
31#include <android/hardware/neuralnetworks/1.3/IDevice.h>
Xusong Wang1b3f4262019-10-25 12:07:17 -070032#include <android/hardware/neuralnetworks/1.3/IPreparedModel.h>
Xusong Wangcc47dff2019-10-23 10:35:07 -070033#include <android/hardware/neuralnetworks/1.3/IPreparedModelCallback.h>
Lev Proleev26d1bc82019-08-30 11:57:18 +010034#include <android/hardware/neuralnetworks/1.3/types.h>
Lev Proleev13fdfcd2019-08-30 11:35:34 +010035#include <android/hidl/allocator/1.0/IAllocator.h>
36#include <android/hidl/memory/1.0/IMemory.h>
37#include <hidlmemory/mapping.h>
38
39#include <gtest/gtest.h>
40#include <algorithm>
Michael Butler648ada52019-07-25 17:22:11 -070041#include <chrono>
Lev Proleev13fdfcd2019-08-30 11:35:34 +010042#include <iostream>
43#include <numeric>
44
45#include "1.0/Utils.h"
46#include "1.2/Callbacks.h"
Xusong Wangcc47dff2019-10-23 10:35:07 -070047#include "1.3/Callbacks.h"
Lev Proleev13fdfcd2019-08-30 11:35:34 +010048#include "ExecutionBurstController.h"
49#include "MemoryUtils.h"
50#include "TestHarness.h"
51#include "Utils.h"
52#include "VtsHalNeuralnetworks.h"
53
Lev Proleev26d1bc82019-08-30 11:57:18 +010054namespace android::hardware::neuralnetworks::V1_3::vts::functional {
Lev Proleev13fdfcd2019-08-30 11:35:34 +010055
56using namespace test_helper;
57using hidl::memory::V1_0::IMemory;
Xusong Wangcc47dff2019-10-23 10:35:07 -070058using implementation::PreparedModelCallback;
Lev Proleev13fdfcd2019-08-30 11:35:34 +010059using V1_0::DataLocation;
60using V1_0::ErrorStatus;
61using V1_0::OperandLifeTime;
62using V1_0::Request;
63using V1_1::ExecutionPreference;
Lev Proleev26d1bc82019-08-30 11:57:18 +010064using V1_2::Constant;
Lev Proleev26d1bc82019-08-30 11:57:18 +010065using V1_2::MeasureTiming;
Lev Proleev26d1bc82019-08-30 11:57:18 +010066using V1_2::OutputShape;
67using V1_2::SymmPerChannelQuantParams;
68using V1_2::Timing;
69using V1_2::implementation::ExecutionCallback;
Lev Proleev13fdfcd2019-08-30 11:35:34 +010070using HidlToken = hidl_array<uint8_t, static_cast<uint32_t>(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
71
72enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT };
73
74Model createModel(const TestModel& testModel) {
75 // Model operands.
76 hidl_vec<Operand> operands(testModel.operands.size());
77 size_t constCopySize = 0, constRefSize = 0;
78 for (uint32_t i = 0; i < testModel.operands.size(); i++) {
79 const auto& op = testModel.operands[i];
80
81 DataLocation loc = {};
82 if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
83 loc = {.poolIndex = 0,
84 .offset = static_cast<uint32_t>(constCopySize),
85 .length = static_cast<uint32_t>(op.data.size())};
86 constCopySize += op.data.alignedSize();
87 } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
88 loc = {.poolIndex = 0,
89 .offset = static_cast<uint32_t>(constRefSize),
90 .length = static_cast<uint32_t>(op.data.size())};
91 constRefSize += op.data.alignedSize();
92 }
93
94 Operand::ExtraParams extraParams;
95 if (op.type == TestOperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
96 extraParams.channelQuant(SymmPerChannelQuantParams{
97 .scales = op.channelQuant.scales, .channelDim = op.channelQuant.channelDim});
98 }
99
100 operands[i] = {.type = static_cast<OperandType>(op.type),
101 .dimensions = op.dimensions,
102 .numberOfConsumers = op.numberOfConsumers,
103 .scale = op.scale,
104 .zeroPoint = op.zeroPoint,
105 .lifetime = static_cast<OperandLifeTime>(op.lifetime),
106 .location = loc,
107 .extraParams = std::move(extraParams)};
108 }
109
110 // Model operations.
111 hidl_vec<Operation> operations(testModel.operations.size());
112 std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),
113 [](const TestOperation& op) -> Operation {
114 return {.type = static_cast<OperationType>(op.type),
115 .inputs = op.inputs,
116 .outputs = op.outputs};
117 });
118
119 // Constant copies.
120 hidl_vec<uint8_t> operandValues(constCopySize);
121 for (uint32_t i = 0; i < testModel.operands.size(); i++) {
122 const auto& op = testModel.operands[i];
123 if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
124 const uint8_t* begin = op.data.get<uint8_t>();
125 const uint8_t* end = begin + op.data.size();
126 std::copy(begin, end, operandValues.data() + operands[i].location.offset);
127 }
128 }
129
130 // Shared memory.
131 hidl_vec<hidl_memory> pools = {};
132 if (constRefSize > 0) {
133 hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
134 CHECK_NE(pools[0].size(), 0u);
135
136 // load data
137 sp<IMemory> mappedMemory = mapMemory(pools[0]);
138 CHECK(mappedMemory.get() != nullptr);
139 uint8_t* mappedPtr =
140 reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));
141 CHECK(mappedPtr != nullptr);
142
143 for (uint32_t i = 0; i < testModel.operands.size(); i++) {
144 const auto& op = testModel.operands[i];
145 if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
146 const uint8_t* begin = op.data.get<uint8_t>();
147 const uint8_t* end = begin + op.data.size();
148 std::copy(begin, end, mappedPtr + operands[i].location.offset);
149 }
150 }
151 }
152
153 return {.operands = std::move(operands),
154 .operations = std::move(operations),
155 .inputIndexes = testModel.inputIndexes,
156 .outputIndexes = testModel.outputIndexes,
157 .operandValues = std::move(operandValues),
158 .pools = std::move(pools),
159 .relaxComputationFloat32toFloat16 = testModel.isRelaxed};
160}
161
162static bool isOutputSizeGreaterThanOne(const TestModel& testModel, uint32_t index) {
163 const auto byteSize = testModel.operands[testModel.outputIndexes[index]].data.size();
164 return byteSize > 1u;
165}
166
167static void makeOutputInsufficientSize(uint32_t outputIndex, Request* request) {
168 auto& length = request->outputs[outputIndex].location.length;
169 ASSERT_GT(length, 1u);
170 length -= 1u;
171}
172
173static void makeOutputDimensionsUnspecified(Model* model) {
174 for (auto i : model->outputIndexes) {
175 auto& dims = model->operands[i].dimensions;
176 std::fill(dims.begin(), dims.end(), 0);
177 }
178}
179
180static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
181 const Request& request, MeasureTiming measure,
182 sp<ExecutionCallback>& callback) {
Xusong Wang1b3f4262019-10-25 12:07:17 -0700183 return preparedModel->execute_1_3(request, measure, callback);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100184}
185static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
186 const Request& request, MeasureTiming measure,
187 hidl_vec<OutputShape>* outputShapes,
188 Timing* timing) {
189 ErrorStatus result;
190 Return<void> ret = preparedModel->executeSynchronously(
191 request, measure,
192 [&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes,
193 const Timing& time) {
194 result = error;
195 *outputShapes = shapes;
196 *timing = time;
197 });
198 if (!ret.isOk()) {
199 return ErrorStatus::GENERAL_FAILURE;
200 }
201 return result;
202}
203static std::shared_ptr<::android::nn::ExecutionBurstController> CreateBurst(
204 const sp<IPreparedModel>& preparedModel) {
Michael Butler648ada52019-07-25 17:22:11 -0700205 return android::nn::ExecutionBurstController::create(preparedModel,
206 std::chrono::microseconds{0});
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100207}
208enum class Executor { ASYNC, SYNC, BURST };
209
210void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
211 Executor executor, MeasureTiming measure, OutputType outputType) {
212 // If output0 does not have size larger than one byte, we can not test with insufficient buffer.
213 if (outputType == OutputType::INSUFFICIENT && !isOutputSizeGreaterThanOne(testModel, 0)) {
214 return;
215 }
216
217 Request request = createRequest(testModel);
218 if (outputType == OutputType::INSUFFICIENT) {
219 makeOutputInsufficientSize(/*outputIndex=*/0, &request);
220 }
221
222 ErrorStatus executionStatus;
223 hidl_vec<OutputShape> outputShapes;
224 Timing timing;
225 switch (executor) {
226 case Executor::ASYNC: {
227 SCOPED_TRACE("asynchronous");
228
229 // launch execution
230 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
231 Return<ErrorStatus> executionLaunchStatus =
232 ExecutePreparedModel(preparedModel, request, measure, executionCallback);
233 ASSERT_TRUE(executionLaunchStatus.isOk());
234 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
235
236 // retrieve execution status
237 executionCallback->wait();
238 executionStatus = executionCallback->getStatus();
239 outputShapes = executionCallback->getOutputShapes();
240 timing = executionCallback->getTiming();
241
242 break;
243 }
244 case Executor::SYNC: {
245 SCOPED_TRACE("synchronous");
246
247 // execute
248 Return<ErrorStatus> executionReturnStatus =
249 ExecutePreparedModel(preparedModel, request, measure, &outputShapes, &timing);
250 ASSERT_TRUE(executionReturnStatus.isOk());
251 executionStatus = static_cast<ErrorStatus>(executionReturnStatus);
252
253 break;
254 }
255 case Executor::BURST: {
256 SCOPED_TRACE("burst");
257
258 // create burst
259 const std::shared_ptr<::android::nn::ExecutionBurstController> controller =
260 CreateBurst(preparedModel);
261 ASSERT_NE(nullptr, controller.get());
262
263 // create memory keys
264 std::vector<intptr_t> keys(request.pools.size());
265 for (size_t i = 0; i < keys.size(); ++i) {
266 keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
267 }
268
269 // execute burst
Michael Butler648ada52019-07-25 17:22:11 -0700270 int n;
271 std::tie(n, outputShapes, timing, std::ignore) =
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100272 controller->compute(request, measure, keys);
Michael Butler648ada52019-07-25 17:22:11 -0700273 executionStatus = nn::convertResultCodeToErrorStatus(n);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100274
275 break;
276 }
277 }
278
279 if (outputType != OutputType::FULLY_SPECIFIED &&
280 executionStatus == ErrorStatus::GENERAL_FAILURE) {
281 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
282 "execute model that it does not support.";
283 std::cout << "[ ] Early termination of test because vendor service cannot "
284 "execute model that it does not support."
285 << std::endl;
286 GTEST_SKIP();
287 }
288 if (measure == MeasureTiming::NO) {
289 EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);
290 EXPECT_EQ(UINT64_MAX, timing.timeInDriver);
291 } else {
292 if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) {
293 EXPECT_LE(timing.timeOnDevice, timing.timeInDriver);
294 }
295 }
296
297 switch (outputType) {
298 case OutputType::FULLY_SPECIFIED:
299 // If the model output operands are fully specified, outputShapes must be either
300 // either empty, or have the same number of elements as the number of outputs.
301 ASSERT_EQ(ErrorStatus::NONE, executionStatus);
302 ASSERT_TRUE(outputShapes.size() == 0 ||
303 outputShapes.size() == testModel.outputIndexes.size());
304 break;
305 case OutputType::UNSPECIFIED:
306 // If the model output operands are not fully specified, outputShapes must have
307 // the same number of elements as the number of outputs.
308 ASSERT_EQ(ErrorStatus::NONE, executionStatus);
309 ASSERT_EQ(outputShapes.size(), testModel.outputIndexes.size());
310 break;
311 case OutputType::INSUFFICIENT:
312 ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus);
313 ASSERT_EQ(outputShapes.size(), testModel.outputIndexes.size());
314 ASSERT_FALSE(outputShapes[0].isSufficient);
315 return;
316 }
317
318 // Go through all outputs, check returned output shapes.
319 for (uint32_t i = 0; i < outputShapes.size(); i++) {
320 EXPECT_TRUE(outputShapes[i].isSufficient);
321 const auto& expect = testModel.operands[testModel.outputIndexes[i]].dimensions;
322 const std::vector<uint32_t> actual = outputShapes[i].dimensions;
323 EXPECT_EQ(expect, actual);
324 }
325
326 // Retrieve execution results.
327 const std::vector<TestBuffer> outputs = getOutputBuffers(request);
328
329 // We want "close-enough" results.
330 checkResults(testModel, outputs);
331}
332
333void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
334 bool testDynamicOutputShape) {
335 if (testDynamicOutputShape) {
336 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
337 OutputType::UNSPECIFIED);
338 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
339 OutputType::UNSPECIFIED);
340 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
341 OutputType::UNSPECIFIED);
342 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
343 OutputType::UNSPECIFIED);
344 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
345 OutputType::UNSPECIFIED);
346 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
347 OutputType::UNSPECIFIED);
348 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
349 OutputType::INSUFFICIENT);
350 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
351 OutputType::INSUFFICIENT);
352 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
353 OutputType::INSUFFICIENT);
354 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
355 OutputType::INSUFFICIENT);
356 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
357 OutputType::INSUFFICIENT);
358 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
359 OutputType::INSUFFICIENT);
360 } else {
361 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
362 OutputType::FULLY_SPECIFIED);
363 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
364 OutputType::FULLY_SPECIFIED);
365 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
366 OutputType::FULLY_SPECIFIED);
367 EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
368 OutputType::FULLY_SPECIFIED);
369 EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
370 OutputType::FULLY_SPECIFIED);
371 EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
372 OutputType::FULLY_SPECIFIED);
373 }
374}
375
376void Execute(const sp<IDevice>& device, const TestModel& testModel, bool testDynamicOutputShape) {
377 Model model = createModel(testModel);
378 if (testDynamicOutputShape) {
379 makeOutputDimensionsUnspecified(&model);
380 }
381
382 sp<IPreparedModel> preparedModel;
383 createPreparedModel(device, model, &preparedModel);
384 if (preparedModel == nullptr) return;
385
386 EvaluatePreparedModel(preparedModel, testModel, testDynamicOutputShape);
387}
388
389void GeneratedTestBase::SetUp() {
390 testing::TestWithParam<GeneratedTestParam>::SetUp();
391 ASSERT_NE(kDevice, nullptr);
392}
393
394std::vector<NamedModel> getNamedModels(const FilterFn& filter) {
395 return TestModelManager::get().getTestModels(filter);
396}
397
398std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info) {
399 const auto& [namedDevice, namedModel] = info.param;
400 return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel));
401}
402
403// Tag for the generated tests
404class GeneratedTest : public GeneratedTestBase {};
405
406// Tag for the dynamic output shape tests
407class DynamicOutputShapeTest : public GeneratedTest {};
408
409TEST_P(GeneratedTest, Test) {
410 Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/false);
411}
412
413TEST_P(DynamicOutputShapeTest, Test) {
414 Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/true);
415}
416
417INSTANTIATE_GENERATED_TEST(GeneratedTest,
418 [](const TestModel& testModel) { return !testModel.expectFailure; });
419
420INSTANTIATE_GENERATED_TEST(DynamicOutputShapeTest,
421 [](const TestModel& testModel) { return !testModel.expectFailure; });
422
Lev Proleev26d1bc82019-08-30 11:57:18 +0100423} // namespace android::hardware::neuralnetworks::V1_3::vts::functional