blob: 8d427b1227a2d906fa1dde303764664219a884a6 [file] [log] [blame]
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Michael Butlercf22a572017-09-22 13:26:12 -070017#include "Callbacks.h"
Michael Butler814d8372019-01-15 11:02:55 -080018#include "ExecutionBurstController.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070019#include "TestHarness.h"
Miao Wanga2d04c82018-02-05 17:26:54 -080020#include "Utils.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070021
22#include <android-base/logging.h>
Miao Wanga2d04c82018-02-05 17:26:54 -080023#include <android/hardware/neuralnetworks/1.0/IDevice.h>
24#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
25#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
26#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
27#include <android/hardware/neuralnetworks/1.0/types.h>
Xusong Wangb5cb8f72018-10-31 08:43:12 -070028#include <android/hardware/neuralnetworks/1.1/IDevice.h>
29#include <android/hardware/neuralnetworks/1.2/IDevice.h>
30#include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h>
31#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
32#include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h>
Miao Wanga2d04c82018-02-05 17:26:54 -080033#include <android/hidl/allocator/1.0/IAllocator.h>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070034#include <android/hidl/memory/1.0/IMemory.h>
35#include <hidlmemory/mapping.h>
Michael Butler0897ab32017-10-04 02:38:42 -070036#include <iostream>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070037
38namespace android {
39namespace hardware {
40namespace neuralnetworks {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070041
42namespace generated_tests {
Xusong Wangb5cb8f72018-10-31 08:43:12 -070043using ::android::hardware::neuralnetworks::V1_2::implementation::ExecutionCallback;
44using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCallback;
Slava Shklyaev9e3fad12018-11-30 17:55:12 +000045using ::test_helper::bool8;
Michael K. Sanders941d61a2018-10-19 14:39:09 +010046using ::test_helper::compare;
47using ::test_helper::expectMultinomialDistributionWithinTolerance;
Mika Raentode166942018-04-17 16:49:50 +010048using ::test_helper::filter;
49using ::test_helper::for_all;
50using ::test_helper::for_each;
Michael K. Sanders941d61a2018-10-19 14:39:09 +010051using ::test_helper::MixedTyped;
52using ::test_helper::MixedTypedExample;
Michael K. Sanders941d61a2018-10-19 14:39:09 +010053using ::test_helper::resize_accordingly;
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070054
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -070055template <typename T>
Xusong Wanga3165812018-11-19 18:26:08 -080056void copy_back_(std::map<int, std::vector<T>>* dst, const std::vector<RequestArgument>& ra,
57 char* src) {
58 for_each<T>(*dst, [&ra, src](int index, std::vector<T>& m) {
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -070059 ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T));
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070060 char* begin = src + ra[index].location.offset;
61 memcpy(m.data(), begin, ra[index].location.length);
62 });
63}
64
65void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
Xusong Wanga3165812018-11-19 18:26:08 -080066 copy_back_(&dst->float32Operands, ra, src);
67 copy_back_(&dst->int32Operands, ra, src);
Xusong Wangd49f6652019-01-16 18:32:24 -080068 copy_back_(&dst->quant8AsymmOperands, ra, src);
69 copy_back_(&dst->quant16SymmOperands, ra, src);
Xusong Wanga3165812018-11-19 18:26:08 -080070 copy_back_(&dst->float16Operands, ra, src);
71 copy_back_(&dst->bool8Operands, ra, src);
72 copy_back_(&dst->quant8ChannelOperands, ra, src);
Xusong Wangd49f6652019-01-16 18:32:24 -080073 copy_back_(&dst->quant16AsymmOperands, ra, src);
74 static_assert(8 == MixedTyped::kNumTypes,
Lev Proleev9b490f42018-11-02 12:44:11 +000075 "Number of types in MixedTyped changed, but copy_back function wasn't updated");
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070076}
77
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070078// Top level driver for models and examples generated by test_generator.py
79// Test driver for those generated from ml/nn/runtime/test/spec
Xusong Wangb5cb8f72018-10-31 08:43:12 -070080static Return<ErrorStatus> ExecutePreparedModel(sp<V1_0::IPreparedModel>& preparedModel,
David Grosse3013492019-01-23 14:01:52 -080081 const Request& request, MeasureTiming,
Xusong Wangb5cb8f72018-10-31 08:43:12 -070082 sp<ExecutionCallback>& callback) {
83 return preparedModel->execute(request, callback);
84}
85static Return<ErrorStatus> ExecutePreparedModel(sp<V1_2::IPreparedModel>& preparedModel,
David Grosse3013492019-01-23 14:01:52 -080086 const Request& request, MeasureTiming measure,
Xusong Wangb5cb8f72018-10-31 08:43:12 -070087 sp<ExecutionCallback>& callback) {
David Grosse3013492019-01-23 14:01:52 -080088 return preparedModel->execute_1_2(request, measure, callback);
Xusong Wangb5cb8f72018-10-31 08:43:12 -070089}
Xusong Wang187c5972018-11-07 09:33:59 -080090static Return<ErrorStatus> ExecutePreparedModel(sp<V1_0::IPreparedModel>&, const Request&,
David Grosse3013492019-01-23 14:01:52 -080091 MeasureTiming, hidl_vec<OutputShape>*, Timing*) {
David Gross49e41672018-12-21 11:20:26 -080092 ADD_FAILURE() << "asking for synchronous execution at V1_0";
93 return ErrorStatus::GENERAL_FAILURE;
94}
95static Return<ErrorStatus> ExecutePreparedModel(sp<V1_2::IPreparedModel>& preparedModel,
David Grosse3013492019-01-23 14:01:52 -080096 const Request& request, MeasureTiming measure,
97 hidl_vec<OutputShape>* outputShapes,
98 Timing* timing) {
Xusong Wang187c5972018-11-07 09:33:59 -080099 ErrorStatus result;
100 Return<void> ret = preparedModel->executeSynchronously(
David Grosse3013492019-01-23 14:01:52 -0800101 request, measure,
102 [&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes,
103 const Timing& time) {
104 result = error;
105 *outputShapes = shapes;
106 *timing = time;
107 });
Xusong Wang187c5972018-11-07 09:33:59 -0800108 if (!ret.isOk()) {
109 return ErrorStatus::GENERAL_FAILURE;
110 }
111 return result;
David Gross49e41672018-12-21 11:20:26 -0800112}
Michael Butler814d8372019-01-15 11:02:55 -0800113static std::unique_ptr<::android::nn::ExecutionBurstController> CreateBurst(
114 const sp<V1_0::IPreparedModel>&) {
115 ADD_FAILURE() << "asking for burst execution at V1_0";
116 return nullptr;
117}
118static std::unique_ptr<::android::nn::ExecutionBurstController> CreateBurst(
119 const sp<V1_2::IPreparedModel>& preparedModel) {
120 return ::android::nn::createExecutionBurstController(preparedModel, /*blocking=*/true);
121}
122enum class Executor { ASYNC, SYNC, BURST };
David Gross49e41672018-12-21 11:20:26 -0800123const float kDefaultAtol = 1e-5f;
124const float kDefaultRtol = 1e-5f;
Xusong Wangb5cb8f72018-10-31 08:43:12 -0700125template <typename T_IPreparedModel>
126void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
Michael K. Sandersefa4c812018-10-30 14:44:48 +0000127 const std::vector<MixedTypedExample>& examples,
David Grosse3013492019-01-23 14:01:52 -0800128 bool hasRelaxedFloat32Model, float fpAtol, float fpRtol,
Michael Butler814d8372019-01-15 11:02:55 -0800129 Executor executor, MeasureTiming measure, bool testDynamicOutputShape) {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700130 const uint32_t INPUT = 0;
131 const uint32_t OUTPUT = 1;
132
133 int example_no = 1;
134 for (auto& example : examples) {
135 SCOPED_TRACE(example_no++);
Michael K. Sanders941d61a2018-10-19 14:39:09 +0100136 const MixedTyped& inputs = example.operands.first;
137 const MixedTyped& golden = example.operands.second;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700138
Xusong Wanga3165812018-11-19 18:26:08 -0800139 const bool hasFloat16Inputs = !inputs.float16Operands.empty();
Michael K. Sandersefa4c812018-10-30 14:44:48 +0000140 if (hasRelaxedFloat32Model || hasFloat16Inputs) {
141 // TODO: Adjust the error limit based on testing.
142 // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
143 fpAtol = 5.0f * 0.0009765625f;
144 // Set the relative tolerance to be 5ULP of the corresponding FP precision.
145 fpRtol = 5.0f * 0.0009765625f;
146 }
147
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700148 std::vector<RequestArgument> inputs_info, outputs_info;
149 uint32_t inputSize = 0, outputSize = 0;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700150 // This function only partially specifies the metadata (vector of RequestArguments).
151 // The contents are copied over below.
152 for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
153 if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
154 RequestArgument arg = {
155 .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
156 .dimensions = {},
157 };
I-Jui (Ray) Sung959cd782017-10-04 20:49:57 -0700158 RequestArgument arg_empty = {
159 .hasNoValue = true,
160 };
161 inputs_info[index] = s ? arg : arg_empty;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700162 inputSize += s;
163 });
164 // Compute offset for inputs 1 and so on
165 {
166 size_t offset = 0;
167 for (auto& i : inputs_info) {
I-Jui (Ray) Sung959cd782017-10-04 20:49:57 -0700168 if (!i.hasNoValue) i.location.offset = offset;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700169 offset += i.location.length;
170 }
171 }
172
173 MixedTyped test; // holding test results
174
175 // Go through all outputs, initialize RequestArgument descriptors
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700176 resize_accordingly(golden, test);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700177 for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
178 if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
179 RequestArgument arg = {
180 .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
181 .dimensions = {},
182 };
183 outputs_info[index] = arg;
184 outputSize += s;
185 });
186 // Compute offset for outputs 1 and so on
187 {
188 size_t offset = 0;
189 for (auto& i : outputs_info) {
190 i.location.offset = offset;
191 offset += i.location.length;
192 }
193 }
Miao Wanga2d04c82018-02-05 17:26:54 -0800194 std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
195 nn::allocateSharedMemory(outputSize)};
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700196 ASSERT_NE(0ull, pools[INPUT].size());
197 ASSERT_NE(0ull, pools[OUTPUT].size());
198
199 // load data
200 sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
201 sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
202 ASSERT_NE(nullptr, inputMemory.get());
203 ASSERT_NE(nullptr, outputMemory.get());
204 char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
205 char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));
206 ASSERT_NE(nullptr, inputPtr);
207 ASSERT_NE(nullptr, outputPtr);
208 inputMemory->update();
209 outputMemory->update();
210
211 // Go through all inputs, copy the values
212 for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
213 char* begin = (char*)p;
214 char* end = begin + s;
215 // TODO: handle more than one input
216 std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
217 });
218
219 inputMemory->commit();
220 outputMemory->commit();
Michael Butlercf22a572017-09-22 13:26:12 -0700221
Michael Butler814d8372019-01-15 11:02:55 -0800222 const Request request = {.inputs = inputs_info, .outputs = outputs_info, .pools = pools};
223
Xusong Wang187c5972018-11-07 09:33:59 -0800224 ErrorStatus executionStatus;
225 hidl_vec<OutputShape> outputShapes;
David Grosse3013492019-01-23 14:01:52 -0800226 Timing timing;
Michael Butler814d8372019-01-15 11:02:55 -0800227 switch (executor) {
228 case Executor::ASYNC: {
229 SCOPED_TRACE("asynchronous");
Michael Butlercf22a572017-09-22 13:26:12 -0700230
Michael Butler814d8372019-01-15 11:02:55 -0800231 // launch execution
232 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
233 ASSERT_NE(nullptr, executionCallback.get());
234 Return<ErrorStatus> executionLaunchStatus =
235 ExecutePreparedModel(preparedModel, request, measure, executionCallback);
236 ASSERT_TRUE(executionLaunchStatus.isOk());
237 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
David Gross49e41672018-12-21 11:20:26 -0800238
Michael Butler814d8372019-01-15 11:02:55 -0800239 // retrieve execution status
240 executionCallback->wait();
241 executionStatus = executionCallback->getStatus();
242 outputShapes = executionCallback->getOutputShapes();
243 timing = executionCallback->getTiming();
David Gross49e41672018-12-21 11:20:26 -0800244
Michael Butler814d8372019-01-15 11:02:55 -0800245 break;
246 }
247 case Executor::SYNC: {
248 SCOPED_TRACE("synchronous");
249
250 // execute
251 Return<ErrorStatus> executionReturnStatus = ExecutePreparedModel(
252 preparedModel, request, measure, &outputShapes, &timing);
253 ASSERT_TRUE(executionReturnStatus.isOk());
254 executionStatus = static_cast<ErrorStatus>(executionReturnStatus);
255
256 break;
257 }
258 case Executor::BURST: {
259 SCOPED_TRACE("burst");
260
261 // create burst
262 const std::unique_ptr<::android::nn::ExecutionBurstController> controller =
263 CreateBurst(preparedModel);
264 ASSERT_NE(nullptr, controller.get());
265
266 // create memory keys
267 std::vector<intptr_t> keys(request.pools.size());
268 for (size_t i = 0; i < keys.size(); ++i) {
269 keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
270 }
271
272 // execute burst
273 std::tie(executionStatus, outputShapes, timing) =
274 controller->compute(request, measure, keys);
275
276 break;
277 }
David Gross49e41672018-12-21 11:20:26 -0800278 }
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700279
Xusong Wanga3165812018-11-19 18:26:08 -0800280 if (testDynamicOutputShape && executionStatus != ErrorStatus::NONE) {
281 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
282 "execute model that it does not support.";
283 std::cout << "[ ] Early termination of test because vendor service cannot "
284 "execute model that it does not support."
285 << std::endl;
286 return;
287 }
Xusong Wang187c5972018-11-07 09:33:59 -0800288 ASSERT_EQ(ErrorStatus::NONE, executionStatus);
David Grosse3013492019-01-23 14:01:52 -0800289 if (measure == MeasureTiming::NO) {
290 EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);
291 EXPECT_EQ(UINT64_MAX, timing.timeInDriver);
292 } else {
293 if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) {
294 EXPECT_LE(timing.timeOnDevice, timing.timeInDriver);
295 }
296 }
Xusong Wanga3165812018-11-19 18:26:08 -0800297
298 // Go through all outputs, overwrite output dimensions with returned output shapes
299 if (testDynamicOutputShape) {
300 ASSERT_NE(outputShapes.size(), 0);
301 for_each<uint32_t>(test.operandDimensions,
302 [&outputShapes](int idx, std::vector<uint32_t>& dim) {
303 dim = outputShapes[idx].dimensions;
304 });
305 }
Xusong Wang187c5972018-11-07 09:33:59 -0800306
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700307 // validate results
308 outputMemory->read();
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700309 copy_back(&test, outputs_info, outputPtr);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700310 outputMemory->commit();
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700311 // Filter out don't cares
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -0700312 MixedTyped filtered_golden = filter(golden, is_ignored);
313 MixedTyped filtered_test = filter(test, is_ignored);
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700314
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700315 // We want "close-enough" results for float
Xusong Wang10d77e42018-08-28 16:50:01 -0700316 compare(filtered_golden, filtered_test, fpAtol, fpRtol);
Michael K. Sanders941d61a2018-10-19 14:39:09 +0100317
318 if (example.expectedMultinomialDistributionTolerance > 0) {
319 expectMultinomialDistributionWithinTolerance(test, example);
320 }
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700321 }
322}
David Gross49e41672018-12-21 11:20:26 -0800323template <typename T_IPreparedModel>
324void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
325 const std::vector<MixedTypedExample>& examples,
Michael Butler814d8372019-01-15 11:02:55 -0800326 bool hasRelaxedFloat32Model, Executor executor, MeasureTiming measure,
Xusong Wanga3165812018-11-19 18:26:08 -0800327 bool testDynamicOutputShape) {
David Gross49e41672018-12-21 11:20:26 -0800328 EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, kDefaultAtol,
Michael Butler814d8372019-01-15 11:02:55 -0800329 kDefaultRtol, executor, measure, testDynamicOutputShape);
David Gross49e41672018-12-21 11:20:26 -0800330}
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700331
Xusong Wangb5cb8f72018-10-31 08:43:12 -0700332static void getPreparedModel(sp<PreparedModelCallback> callback,
333 sp<V1_0::IPreparedModel>* preparedModel) {
334 *preparedModel = callback->getPreparedModel();
335}
336static void getPreparedModel(sp<PreparedModelCallback> callback,
337 sp<V1_2::IPreparedModel>* preparedModel) {
338 sp<V1_0::IPreparedModel> preparedModelV1_0 = callback->getPreparedModel();
339 *preparedModel = V1_2::IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr);
340}
341
Michael Butlerf76acd02018-03-22 16:37:57 -0700342void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,
Michael K. Sanders941d61a2018-10-19 14:39:09 +0100343 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Miao Wanga2d04c82018-02-05 17:26:54 -0800344 V1_0::Model model = create_model();
345
346 // see if service can handle model
347 bool fullySupportsModel = false;
Miao Wanga2d04c82018-02-05 17:26:54 -0800348 Return<void> supportedCall = device->getSupportedOperations(
Michael Butler4d5bb102018-02-26 15:24:46 -0800349 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
350 ASSERT_EQ(ErrorStatus::NONE, status);
Miao Wanga2d04c82018-02-05 17:26:54 -0800351 ASSERT_NE(0ul, supported.size());
352 fullySupportsModel =
353 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
354 });
355 ASSERT_TRUE(supportedCall.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800356
357 // launch prepare model
358 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
359 ASSERT_NE(nullptr, preparedModelCallback.get());
Miao Wanga2d04c82018-02-05 17:26:54 -0800360 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
361 ASSERT_TRUE(prepareLaunchStatus.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800362 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Miao Wanga2d04c82018-02-05 17:26:54 -0800363
364 // retrieve prepared model
365 preparedModelCallback->wait();
366 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
Xusong Wangb5cb8f72018-10-31 08:43:12 -0700367 sp<V1_0::IPreparedModel> preparedModel;
368 getPreparedModel(preparedModelCallback, &preparedModel);
Miao Wanga2d04c82018-02-05 17:26:54 -0800369
370 // early termination if vendor service cannot fully prepare model
Michael Butler4d5bb102018-02-26 15:24:46 -0800371 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
Miao Wanga2d04c82018-02-05 17:26:54 -0800372 ASSERT_EQ(nullptr, preparedModel.get());
373 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
374 "prepare model that it does not support.";
375 std::cout << "[ ] Early termination of test because vendor service cannot "
376 "prepare model that it does not support."
377 << std::endl;
378 return;
379 }
Michael Butler4d5bb102018-02-26 15:24:46 -0800380 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
Miao Wanga2d04c82018-02-05 17:26:54 -0800381 ASSERT_NE(nullptr, preparedModel.get());
382
Xusong Wang10d77e42018-08-28 16:50:01 -0700383 float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f;
Michael K. Sandersefa4c812018-10-30 14:44:48 +0000384 EvaluatePreparedModel(preparedModel, is_ignored, examples,
Michael Butler814d8372019-01-15 11:02:55 -0800385 /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol, Executor::ASYNC,
David Grosse3013492019-01-23 14:01:52 -0800386 MeasureTiming::NO, /*testDynamicOutputShape=*/false);
Miao Wanga2d04c82018-02-05 17:26:54 -0800387}
388
Michael Butlerf76acd02018-03-22 16:37:57 -0700389void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
Michael K. Sanders941d61a2018-10-19 14:39:09 +0100390 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Miao Wanga2d04c82018-02-05 17:26:54 -0800391 V1_1::Model model = create_model();
392
393 // see if service can handle model
394 bool fullySupportsModel = false;
Miao Wanga2d04c82018-02-05 17:26:54 -0800395 Return<void> supportedCall = device->getSupportedOperations_1_1(
Michael Butler4d5bb102018-02-26 15:24:46 -0800396 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
397 ASSERT_EQ(ErrorStatus::NONE, status);
Miao Wanga2d04c82018-02-05 17:26:54 -0800398 ASSERT_NE(0ul, supported.size());
399 fullySupportsModel =
400 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
401 });
402 ASSERT_TRUE(supportedCall.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800403
404 // launch prepare model
405 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
406 ASSERT_NE(nullptr, preparedModelCallback.get());
Michael Butler2504c2f2018-04-11 16:30:09 -0700407 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
408 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
Miao Wanga2d04c82018-02-05 17:26:54 -0800409 ASSERT_TRUE(prepareLaunchStatus.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800410 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Miao Wanga2d04c82018-02-05 17:26:54 -0800411
412 // retrieve prepared model
413 preparedModelCallback->wait();
414 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
Xusong Wangb5cb8f72018-10-31 08:43:12 -0700415 sp<V1_0::IPreparedModel> preparedModel;
416 getPreparedModel(preparedModelCallback, &preparedModel);
Miao Wanga2d04c82018-02-05 17:26:54 -0800417
418 // early termination if vendor service cannot fully prepare model
Michael Butler4d5bb102018-02-26 15:24:46 -0800419 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
Miao Wanga2d04c82018-02-05 17:26:54 -0800420 ASSERT_EQ(nullptr, preparedModel.get());
421 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
422 "prepare model that it does not support.";
423 std::cout << "[ ] Early termination of test because vendor service cannot "
424 "prepare model that it does not support."
425 << std::endl;
426 return;
427 }
Michael Butler4d5bb102018-02-26 15:24:46 -0800428 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
Miao Wanga2d04c82018-02-05 17:26:54 -0800429 ASSERT_NE(nullptr, preparedModel.get());
430
Michael K. Sandersefa4c812018-10-30 14:44:48 +0000431 EvaluatePreparedModel(preparedModel, is_ignored, examples,
Michael Butler814d8372019-01-15 11:02:55 -0800432 model.relaxComputationFloat32toFloat16, 1e-5f, 1e-5f, Executor::ASYNC,
David Grosse3013492019-01-23 14:01:52 -0800433 MeasureTiming::NO, /*testDynamicOutputShape=*/false);
Miao Wanga2d04c82018-02-05 17:26:54 -0800434}
435
Slava Shklyaev871be942018-09-12 14:52:02 +0100436// TODO: Reduce code duplication.
437void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,
Xusong Wanga3165812018-11-19 18:26:08 -0800438 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples,
439 bool testDynamicOutputShape) {
Slava Shklyaev871be942018-09-12 14:52:02 +0100440 V1_2::Model model = create_model();
441
442 // see if service can handle model
443 bool fullySupportsModel = false;
444 Return<void> supportedCall = device->getSupportedOperations_1_2(
445 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
446 ASSERT_EQ(ErrorStatus::NONE, status);
447 ASSERT_NE(0ul, supported.size());
448 fullySupportsModel =
449 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
450 });
451 ASSERT_TRUE(supportedCall.isOk());
452
453 // launch prepare model
454 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
455 ASSERT_NE(nullptr, preparedModelCallback.get());
456 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_2(
457 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
458 ASSERT_TRUE(prepareLaunchStatus.isOk());
459 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
460
461 // retrieve prepared model
462 preparedModelCallback->wait();
463 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
Xusong Wangb5cb8f72018-10-31 08:43:12 -0700464 sp<V1_2::IPreparedModel> preparedModel;
465 getPreparedModel(preparedModelCallback, &preparedModel);
Slava Shklyaev871be942018-09-12 14:52:02 +0100466
467 // early termination if vendor service cannot fully prepare model
468 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
469 ASSERT_EQ(nullptr, preparedModel.get());
470 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
471 "prepare model that it does not support.";
472 std::cout << "[ ] Early termination of test because vendor service cannot "
473 "prepare model that it does not support."
474 << std::endl;
475 return;
476 }
477 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
478 ASSERT_NE(nullptr, preparedModel.get());
479
Michael K. Sandersefa4c812018-10-30 14:44:48 +0000480 EvaluatePreparedModel(preparedModel, is_ignored, examples,
Michael Butler814d8372019-01-15 11:02:55 -0800481 model.relaxComputationFloat32toFloat16, Executor::ASYNC,
David Grosse3013492019-01-23 14:01:52 -0800482 MeasureTiming::NO, testDynamicOutputShape);
David Gross49e41672018-12-21 11:20:26 -0800483 EvaluatePreparedModel(preparedModel, is_ignored, examples,
Michael Butler814d8372019-01-15 11:02:55 -0800484 model.relaxComputationFloat32toFloat16, Executor::SYNC, MeasureTiming::NO,
485 testDynamicOutputShape);
486 EvaluatePreparedModel(preparedModel, is_ignored, examples,
487 model.relaxComputationFloat32toFloat16, Executor::BURST,
David Grosse3013492019-01-23 14:01:52 -0800488 MeasureTiming::NO, testDynamicOutputShape);
489 EvaluatePreparedModel(preparedModel, is_ignored, examples,
Michael Butler814d8372019-01-15 11:02:55 -0800490 model.relaxComputationFloat32toFloat16, Executor::ASYNC,
David Grosse3013492019-01-23 14:01:52 -0800491 MeasureTiming::YES, testDynamicOutputShape);
492 EvaluatePreparedModel(preparedModel, is_ignored, examples,
Michael Butler814d8372019-01-15 11:02:55 -0800493 model.relaxComputationFloat32toFloat16, Executor::SYNC,
494 MeasureTiming::YES, testDynamicOutputShape);
495 EvaluatePreparedModel(preparedModel, is_ignored, examples,
496 model.relaxComputationFloat32toFloat16, Executor::BURST,
David Grosse3013492019-01-23 14:01:52 -0800497 MeasureTiming::YES, testDynamicOutputShape);
Slava Shklyaev871be942018-09-12 14:52:02 +0100498}
499
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700500} // namespace generated_tests
501
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700502} // namespace neuralnetworks
503} // namespace hardware
504} // namespace android