blob: 3b4eb217aa13ff57530fd6a84f4e89a747e18094 [file] [log] [blame]
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Michael Butlercf22a572017-09-22 13:26:12 -070017#include "Callbacks.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070018#include "TestHarness.h"
Miao Wanga2d04c82018-02-05 17:26:54 -080019#include "Utils.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070020
21#include <android-base/logging.h>
Miao Wanga2d04c82018-02-05 17:26:54 -080022#include <android/hardware/neuralnetworks/1.0/IDevice.h>
23#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
24#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
25#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
26#include <android/hardware/neuralnetworks/1.0/types.h>
Xusong Wangb5cb8f72018-10-31 08:43:12 -070027#include <android/hardware/neuralnetworks/1.1/IDevice.h>
28#include <android/hardware/neuralnetworks/1.2/IDevice.h>
29#include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h>
30#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
31#include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h>
Miao Wanga2d04c82018-02-05 17:26:54 -080032#include <android/hidl/allocator/1.0/IAllocator.h>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070033#include <android/hidl/memory/1.0/IMemory.h>
34#include <hidlmemory/mapping.h>
Michael Butler0897ab32017-10-04 02:38:42 -070035#include <iostream>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070036
37namespace android {
38namespace hardware {
39namespace neuralnetworks {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070040
41namespace generated_tests {
Xusong Wangb5cb8f72018-10-31 08:43:12 -070042using ::android::hardware::neuralnetworks::V1_2::implementation::ExecutionCallback;
43using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCallback;
Slava Shklyaev9e3fad12018-11-30 17:55:12 +000044using ::test_helper::bool8;
Michael K. Sanders941d61a2018-10-19 14:39:09 +010045using ::test_helper::compare;
46using ::test_helper::expectMultinomialDistributionWithinTolerance;
Mika Raentode166942018-04-17 16:49:50 +010047using ::test_helper::filter;
Michael K. Sanders941d61a2018-10-19 14:39:09 +010048using ::test_helper::Float32Operands;
Mika Raentode166942018-04-17 16:49:50 +010049using ::test_helper::for_all;
50using ::test_helper::for_each;
Mika Raentode166942018-04-17 16:49:50 +010051using ::test_helper::Int32Operands;
Michael K. Sanders941d61a2018-10-19 14:39:09 +010052using ::test_helper::MixedTyped;
53using ::test_helper::MixedTypedExample;
Michael K. Sandersefa4c812018-10-30 14:44:48 +000054using ::test_helper::MixedTypedIndex;
Mika Raentode166942018-04-17 16:49:50 +010055using ::test_helper::Quant8Operands;
Michael K. Sanders941d61a2018-10-19 14:39:09 +010056using ::test_helper::resize_accordingly;
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070057
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -070058template <typename T>
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070059void copy_back_(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
60 MixedTyped& test = *dst;
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -070061 for_each<T>(test, [&ra, src](int index, std::vector<T>& m) {
62 ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T));
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070063 char* begin = src + ra[index].location.offset;
64 memcpy(m.data(), begin, ra[index].location.length);
65 });
66}
67
68void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
69 copy_back_<float>(dst, ra, src);
70 copy_back_<int32_t>(dst, ra, src);
71 copy_back_<uint8_t>(dst, ra, src);
Lev Proleevca80ff02018-11-05 13:20:06 +000072 copy_back_<int16_t>(dst, ra, src);
Michael K. Sandersefa4c812018-10-30 14:44:48 +000073 copy_back_<_Float16>(dst, ra, src);
Slava Shklyaev9e3fad12018-11-30 17:55:12 +000074 copy_back_<bool8>(dst, ra, src);
Przemyslaw Szczepaniak42909612018-12-12 13:13:32 +000075 copy_back_<int8_t>(dst, ra, src);
76 static_assert(7 == std::tuple_size<MixedTyped>::value,
Lev Proleev9b490f42018-11-02 12:44:11 +000077 "Number of types in MixedTyped changed, but copy_back function wasn't updated");
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070078}
79
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070080// Top level driver for models and examples generated by test_generator.py
81// Test driver for those generated from ml/nn/runtime/test/spec
Xusong Wangb5cb8f72018-10-31 08:43:12 -070082static Return<ErrorStatus> ExecutePreparedModel(sp<V1_0::IPreparedModel>& preparedModel,
83 const Request& request,
84 sp<ExecutionCallback>& callback) {
85 return preparedModel->execute(request, callback);
86}
87static Return<ErrorStatus> ExecutePreparedModel(sp<V1_2::IPreparedModel>& preparedModel,
88 const Request& request,
89 sp<ExecutionCallback>& callback) {
90 return preparedModel->execute_1_2(request, callback);
91}
David Gross49e41672018-12-21 11:20:26 -080092static Return<ErrorStatus> ExecutePreparedModel(sp<V1_0::IPreparedModel>&, const Request&) {
93 ADD_FAILURE() << "asking for synchronous execution at V1_0";
94 return ErrorStatus::GENERAL_FAILURE;
95}
96static Return<ErrorStatus> ExecutePreparedModel(sp<V1_2::IPreparedModel>& preparedModel,
97 const Request& request) {
98 return preparedModel->executeSynchronously(request);
99}
100enum class Synchronously { NO, YES };
101const float kDefaultAtol = 1e-5f;
102const float kDefaultRtol = 1e-5f;
Xusong Wangb5cb8f72018-10-31 08:43:12 -0700103template <typename T_IPreparedModel>
104void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
Michael K. Sandersefa4c812018-10-30 14:44:48 +0000105 const std::vector<MixedTypedExample>& examples,
David Gross49e41672018-12-21 11:20:26 -0800106 bool hasRelaxedFloat32Model = false, float fpAtol = kDefaultAtol,
107 float fpRtol = kDefaultRtol, Synchronously sync = Synchronously::NO) {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700108 const uint32_t INPUT = 0;
109 const uint32_t OUTPUT = 1;
110
111 int example_no = 1;
112 for (auto& example : examples) {
113 SCOPED_TRACE(example_no++);
Michael K. Sanders941d61a2018-10-19 14:39:09 +0100114 const MixedTyped& inputs = example.operands.first;
115 const MixedTyped& golden = example.operands.second;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700116
Michael K. Sandersefa4c812018-10-30 14:44:48 +0000117 const bool hasFloat16Inputs = !std::get<MixedTypedIndex<_Float16>::index>(inputs).empty();
118 if (hasRelaxedFloat32Model || hasFloat16Inputs) {
119 // TODO: Adjust the error limit based on testing.
120 // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
121 fpAtol = 5.0f * 0.0009765625f;
122 // Set the relative tolerance to be 5ULP of the corresponding FP precision.
123 fpRtol = 5.0f * 0.0009765625f;
124 }
125
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700126 std::vector<RequestArgument> inputs_info, outputs_info;
127 uint32_t inputSize = 0, outputSize = 0;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700128 // This function only partially specifies the metadata (vector of RequestArguments).
129 // The contents are copied over below.
130 for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
131 if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
132 RequestArgument arg = {
133 .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
134 .dimensions = {},
135 };
I-Jui (Ray) Sung959cd782017-10-04 20:49:57 -0700136 RequestArgument arg_empty = {
137 .hasNoValue = true,
138 };
139 inputs_info[index] = s ? arg : arg_empty;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700140 inputSize += s;
141 });
142 // Compute offset for inputs 1 and so on
143 {
144 size_t offset = 0;
145 for (auto& i : inputs_info) {
I-Jui (Ray) Sung959cd782017-10-04 20:49:57 -0700146 if (!i.hasNoValue) i.location.offset = offset;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700147 offset += i.location.length;
148 }
149 }
150
151 MixedTyped test; // holding test results
152
153 // Go through all outputs, initialize RequestArgument descriptors
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700154 resize_accordingly(golden, test);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700155 for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
156 if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
157 RequestArgument arg = {
158 .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
159 .dimensions = {},
160 };
161 outputs_info[index] = arg;
162 outputSize += s;
163 });
164 // Compute offset for outputs 1 and so on
165 {
166 size_t offset = 0;
167 for (auto& i : outputs_info) {
168 i.location.offset = offset;
169 offset += i.location.length;
170 }
171 }
Miao Wanga2d04c82018-02-05 17:26:54 -0800172 std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
173 nn::allocateSharedMemory(outputSize)};
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700174 ASSERT_NE(0ull, pools[INPUT].size());
175 ASSERT_NE(0ull, pools[OUTPUT].size());
176
177 // load data
178 sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
179 sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
180 ASSERT_NE(nullptr, inputMemory.get());
181 ASSERT_NE(nullptr, outputMemory.get());
182 char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
183 char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));
184 ASSERT_NE(nullptr, inputPtr);
185 ASSERT_NE(nullptr, outputPtr);
186 inputMemory->update();
187 outputMemory->update();
188
189 // Go through all inputs, copy the values
190 for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
191 char* begin = (char*)p;
192 char* end = begin + s;
193 // TODO: handle more than one input
194 std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
195 });
196
197 inputMemory->commit();
198 outputMemory->commit();
Michael Butlercf22a572017-09-22 13:26:12 -0700199
David Gross49e41672018-12-21 11:20:26 -0800200 if (sync == Synchronously::NO) {
201 SCOPED_TRACE("asynchronous");
Michael Butlercf22a572017-09-22 13:26:12 -0700202
David Gross49e41672018-12-21 11:20:26 -0800203 // launch execution
204 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
205 ASSERT_NE(nullptr, executionCallback.get());
206 Return<ErrorStatus> executionLaunchStatus = ExecutePreparedModel(
207 preparedModel, {.inputs = inputs_info, .outputs = outputs_info, .pools = pools},
208 executionCallback);
209 ASSERT_TRUE(executionLaunchStatus.isOk());
210 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
211
212 // retrieve execution status
213 executionCallback->wait();
214 ErrorStatus executionReturnStatus = executionCallback->getStatus();
215 EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
216 } else {
217 SCOPED_TRACE("synchronous");
218
219 // execute
220 Return<ErrorStatus> executionStatus = ExecutePreparedModel(
221 preparedModel, {.inputs = inputs_info, .outputs = outputs_info, .pools = pools});
222 ASSERT_TRUE(executionStatus.isOk());
223 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionStatus));
224 }
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700225
226 // validate results
227 outputMemory->read();
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700228 copy_back(&test, outputs_info, outputPtr);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700229 outputMemory->commit();
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700230 // Filter out don't cares
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -0700231 MixedTyped filtered_golden = filter(golden, is_ignored);
232 MixedTyped filtered_test = filter(test, is_ignored);
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700233
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700234 // We want "close-enough" results for float
Xusong Wang10d77e42018-08-28 16:50:01 -0700235 compare(filtered_golden, filtered_test, fpAtol, fpRtol);
Michael K. Sanders941d61a2018-10-19 14:39:09 +0100236
237 if (example.expectedMultinomialDistributionTolerance > 0) {
238 expectMultinomialDistributionWithinTolerance(test, example);
239 }
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700240 }
241}
David Gross49e41672018-12-21 11:20:26 -0800242template <typename T_IPreparedModel>
243void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
244 const std::vector<MixedTypedExample>& examples,
245 bool hasRelaxedFloat32Model, Synchronously sync) {
246 EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, kDefaultAtol,
247 kDefaultRtol, sync);
248}
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700249
Xusong Wangb5cb8f72018-10-31 08:43:12 -0700250static void getPreparedModel(sp<PreparedModelCallback> callback,
251 sp<V1_0::IPreparedModel>* preparedModel) {
252 *preparedModel = callback->getPreparedModel();
253}
254static void getPreparedModel(sp<PreparedModelCallback> callback,
255 sp<V1_2::IPreparedModel>* preparedModel) {
256 sp<V1_0::IPreparedModel> preparedModelV1_0 = callback->getPreparedModel();
257 *preparedModel = V1_2::IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr);
258}
259
Michael Butlerf76acd02018-03-22 16:37:57 -0700260void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,
Michael K. Sanders941d61a2018-10-19 14:39:09 +0100261 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Miao Wanga2d04c82018-02-05 17:26:54 -0800262 V1_0::Model model = create_model();
263
264 // see if service can handle model
265 bool fullySupportsModel = false;
Miao Wanga2d04c82018-02-05 17:26:54 -0800266 Return<void> supportedCall = device->getSupportedOperations(
Michael Butler4d5bb102018-02-26 15:24:46 -0800267 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
268 ASSERT_EQ(ErrorStatus::NONE, status);
Miao Wanga2d04c82018-02-05 17:26:54 -0800269 ASSERT_NE(0ul, supported.size());
270 fullySupportsModel =
271 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
272 });
273 ASSERT_TRUE(supportedCall.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800274
275 // launch prepare model
276 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
277 ASSERT_NE(nullptr, preparedModelCallback.get());
Miao Wanga2d04c82018-02-05 17:26:54 -0800278 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
279 ASSERT_TRUE(prepareLaunchStatus.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800280 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Miao Wanga2d04c82018-02-05 17:26:54 -0800281
282 // retrieve prepared model
283 preparedModelCallback->wait();
284 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
Xusong Wangb5cb8f72018-10-31 08:43:12 -0700285 sp<V1_0::IPreparedModel> preparedModel;
286 getPreparedModel(preparedModelCallback, &preparedModel);
Miao Wanga2d04c82018-02-05 17:26:54 -0800287
288 // early termination if vendor service cannot fully prepare model
Michael Butler4d5bb102018-02-26 15:24:46 -0800289 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
Miao Wanga2d04c82018-02-05 17:26:54 -0800290 ASSERT_EQ(nullptr, preparedModel.get());
291 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
292 "prepare model that it does not support.";
293 std::cout << "[ ] Early termination of test because vendor service cannot "
294 "prepare model that it does not support."
295 << std::endl;
296 return;
297 }
Michael Butler4d5bb102018-02-26 15:24:46 -0800298 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
Miao Wanga2d04c82018-02-05 17:26:54 -0800299 ASSERT_NE(nullptr, preparedModel.get());
300
Xusong Wang10d77e42018-08-28 16:50:01 -0700301 float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f;
Michael K. Sandersefa4c812018-10-30 14:44:48 +0000302 EvaluatePreparedModel(preparedModel, is_ignored, examples,
303 /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol);
Miao Wanga2d04c82018-02-05 17:26:54 -0800304}
305
Michael Butlerf76acd02018-03-22 16:37:57 -0700306void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
Michael K. Sanders941d61a2018-10-19 14:39:09 +0100307 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Miao Wanga2d04c82018-02-05 17:26:54 -0800308 V1_1::Model model = create_model();
309
310 // see if service can handle model
311 bool fullySupportsModel = false;
Miao Wanga2d04c82018-02-05 17:26:54 -0800312 Return<void> supportedCall = device->getSupportedOperations_1_1(
Michael Butler4d5bb102018-02-26 15:24:46 -0800313 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
314 ASSERT_EQ(ErrorStatus::NONE, status);
Miao Wanga2d04c82018-02-05 17:26:54 -0800315 ASSERT_NE(0ul, supported.size());
316 fullySupportsModel =
317 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
318 });
319 ASSERT_TRUE(supportedCall.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800320
321 // launch prepare model
322 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
323 ASSERT_NE(nullptr, preparedModelCallback.get());
Michael Butler2504c2f2018-04-11 16:30:09 -0700324 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
325 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
Miao Wanga2d04c82018-02-05 17:26:54 -0800326 ASSERT_TRUE(prepareLaunchStatus.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800327 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Miao Wanga2d04c82018-02-05 17:26:54 -0800328
329 // retrieve prepared model
330 preparedModelCallback->wait();
331 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
Xusong Wangb5cb8f72018-10-31 08:43:12 -0700332 sp<V1_0::IPreparedModel> preparedModel;
333 getPreparedModel(preparedModelCallback, &preparedModel);
Miao Wanga2d04c82018-02-05 17:26:54 -0800334
335 // early termination if vendor service cannot fully prepare model
Michael Butler4d5bb102018-02-26 15:24:46 -0800336 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
Miao Wanga2d04c82018-02-05 17:26:54 -0800337 ASSERT_EQ(nullptr, preparedModel.get());
338 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
339 "prepare model that it does not support.";
340 std::cout << "[ ] Early termination of test because vendor service cannot "
341 "prepare model that it does not support."
342 << std::endl;
343 return;
344 }
Michael Butler4d5bb102018-02-26 15:24:46 -0800345 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
Miao Wanga2d04c82018-02-05 17:26:54 -0800346 ASSERT_NE(nullptr, preparedModel.get());
347
Michael K. Sandersefa4c812018-10-30 14:44:48 +0000348 EvaluatePreparedModel(preparedModel, is_ignored, examples,
349 model.relaxComputationFloat32toFloat16);
Miao Wanga2d04c82018-02-05 17:26:54 -0800350}
351
Slava Shklyaev871be942018-09-12 14:52:02 +0100352// TODO: Reduce code duplication.
353void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,
Michael K. Sanders941d61a2018-10-19 14:39:09 +0100354 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Slava Shklyaev871be942018-09-12 14:52:02 +0100355 V1_2::Model model = create_model();
356
357 // see if service can handle model
358 bool fullySupportsModel = false;
359 Return<void> supportedCall = device->getSupportedOperations_1_2(
360 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
361 ASSERT_EQ(ErrorStatus::NONE, status);
362 ASSERT_NE(0ul, supported.size());
363 fullySupportsModel =
364 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
365 });
366 ASSERT_TRUE(supportedCall.isOk());
367
368 // launch prepare model
369 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
370 ASSERT_NE(nullptr, preparedModelCallback.get());
371 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_2(
372 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
373 ASSERT_TRUE(prepareLaunchStatus.isOk());
374 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
375
376 // retrieve prepared model
377 preparedModelCallback->wait();
378 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
Xusong Wangb5cb8f72018-10-31 08:43:12 -0700379 sp<V1_2::IPreparedModel> preparedModel;
380 getPreparedModel(preparedModelCallback, &preparedModel);
Slava Shklyaev871be942018-09-12 14:52:02 +0100381
382 // early termination if vendor service cannot fully prepare model
383 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
384 ASSERT_EQ(nullptr, preparedModel.get());
385 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
386 "prepare model that it does not support.";
387 std::cout << "[ ] Early termination of test because vendor service cannot "
388 "prepare model that it does not support."
389 << std::endl;
390 return;
391 }
392 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
393 ASSERT_NE(nullptr, preparedModel.get());
394
Michael K. Sandersefa4c812018-10-30 14:44:48 +0000395 EvaluatePreparedModel(preparedModel, is_ignored, examples,
David Gross49e41672018-12-21 11:20:26 -0800396 model.relaxComputationFloat32toFloat16, Synchronously::NO);
397 EvaluatePreparedModel(preparedModel, is_ignored, examples,
398 model.relaxComputationFloat32toFloat16, Synchronously::YES);
Slava Shklyaev871be942018-09-12 14:52:02 +0100399}
400
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700401} // namespace generated_tests
402
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700403} // namespace neuralnetworks
404} // namespace hardware
405} // namespace android