blob: 802d01875cf12d9b2b0eb584dab9acf098c20c61 [file] [log] [blame]
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Michael Butlercf22a572017-09-22 13:26:12 -070017#include "Callbacks.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070018#include "TestHarness.h"
Miao Wang4862d612018-02-05 17:26:54 -080019#include "Utils.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070020
21#include <android-base/logging.h>
Miao Wang4862d612018-02-05 17:26:54 -080022#include <android/hardware/neuralnetworks/1.0/IDevice.h>
23#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
24#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
25#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
26#include <android/hardware/neuralnetworks/1.0/types.h>
27#include <android/hidl/allocator/1.0/IAllocator.h>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070028#include <android/hidl/memory/1.0/IMemory.h>
29#include <hidlmemory/mapping.h>
Michael Butler0897ab32017-10-04 02:38:42 -070030#include <iostream>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070031
32namespace android {
33namespace hardware {
34namespace neuralnetworks {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070035
36namespace generated_tests {
Michael Butlercf22a572017-09-22 13:26:12 -070037using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
38using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010039using ::test_helper::compare;
40using ::test_helper::expectMultinomialDistributionWithinTolerance;
Mika Raentod534d322018-04-17 16:49:50 +010041using ::test_helper::filter;
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010042using ::test_helper::Float32Operands;
Mika Raentod534d322018-04-17 16:49:50 +010043using ::test_helper::for_all;
44using ::test_helper::for_each;
Mika Raentod534d322018-04-17 16:49:50 +010045using ::test_helper::Int32Operands;
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010046using ::test_helper::MixedTyped;
47using ::test_helper::MixedTypedExample;
Michael K. Sanders650fd182018-10-30 14:44:48 +000048using ::test_helper::MixedTypedIndex;
Mika Raentod534d322018-04-17 16:49:50 +010049using ::test_helper::Quant8Operands;
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010050using ::test_helper::resize_accordingly;
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070051
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -070052template <typename T>
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070053void copy_back_(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
54 MixedTyped& test = *dst;
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -070055 for_each<T>(test, [&ra, src](int index, std::vector<T>& m) {
56 ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T));
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070057 char* begin = src + ra[index].location.offset;
58 memcpy(m.data(), begin, ra[index].location.length);
59 });
60}
61
62void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
63 copy_back_<float>(dst, ra, src);
64 copy_back_<int32_t>(dst, ra, src);
65 copy_back_<uint8_t>(dst, ra, src);
Lev Proleeved7ce7a2018-11-05 13:20:06 +000066 copy_back_<int16_t>(dst, ra, src);
Michael K. Sanders650fd182018-10-30 14:44:48 +000067 copy_back_<_Float16>(dst, ra, src);
68 static_assert(5 == std::tuple_size<MixedTyped>::value,
Lev Proleevd36b7a82018-11-02 12:44:11 +000069 "Number of types in MixedTyped changed, but copy_back function wasn't updated");
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070070}
71
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070072// Top level driver for models and examples generated by test_generator.py
73// Test driver for those generated from ml/nn/runtime/test/spec
Miao Wang4862d612018-02-05 17:26:54 -080074void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
Michael K. Sanders650fd182018-10-30 14:44:48 +000075 const std::vector<MixedTypedExample>& examples,
76 bool hasRelaxedFloat32Model = false, float fpAtol = 1e-5f,
Xusong Wangf6235f82018-08-28 16:50:01 -070077 float fpRtol = 1e-5f) {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070078 const uint32_t INPUT = 0;
79 const uint32_t OUTPUT = 1;
80
81 int example_no = 1;
82 for (auto& example : examples) {
83 SCOPED_TRACE(example_no++);
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010084 const MixedTyped& inputs = example.operands.first;
85 const MixedTyped& golden = example.operands.second;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070086
Michael K. Sanders650fd182018-10-30 14:44:48 +000087 const bool hasFloat16Inputs = !std::get<MixedTypedIndex<_Float16>::index>(inputs).empty();
88 if (hasRelaxedFloat32Model || hasFloat16Inputs) {
89 // TODO: Adjust the error limit based on testing.
90 // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
91 fpAtol = 5.0f * 0.0009765625f;
92 // Set the relative tolerance to be 5ULP of the corresponding FP precision.
93 fpRtol = 5.0f * 0.0009765625f;
94 }
95
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070096 std::vector<RequestArgument> inputs_info, outputs_info;
97 uint32_t inputSize = 0, outputSize = 0;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070098 // This function only partially specifies the metadata (vector of RequestArguments).
99 // The contents are copied over below.
100 for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
101 if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
102 RequestArgument arg = {
103 .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
104 .dimensions = {},
105 };
I-Jui (Ray) Sung959cd782017-10-04 20:49:57 -0700106 RequestArgument arg_empty = {
107 .hasNoValue = true,
108 };
109 inputs_info[index] = s ? arg : arg_empty;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700110 inputSize += s;
111 });
112 // Compute offset for inputs 1 and so on
113 {
114 size_t offset = 0;
115 for (auto& i : inputs_info) {
I-Jui (Ray) Sung959cd782017-10-04 20:49:57 -0700116 if (!i.hasNoValue) i.location.offset = offset;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700117 offset += i.location.length;
118 }
119 }
120
121 MixedTyped test; // holding test results
122
123 // Go through all outputs, initialize RequestArgument descriptors
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700124 resize_accordingly(golden, test);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700125 for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
126 if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
127 RequestArgument arg = {
128 .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
129 .dimensions = {},
130 };
131 outputs_info[index] = arg;
132 outputSize += s;
133 });
134 // Compute offset for outputs 1 and so on
135 {
136 size_t offset = 0;
137 for (auto& i : outputs_info) {
138 i.location.offset = offset;
139 offset += i.location.length;
140 }
141 }
Miao Wang4862d612018-02-05 17:26:54 -0800142 std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
143 nn::allocateSharedMemory(outputSize)};
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700144 ASSERT_NE(0ull, pools[INPUT].size());
145 ASSERT_NE(0ull, pools[OUTPUT].size());
146
147 // load data
148 sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
149 sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
150 ASSERT_NE(nullptr, inputMemory.get());
151 ASSERT_NE(nullptr, outputMemory.get());
152 char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
153 char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));
154 ASSERT_NE(nullptr, inputPtr);
155 ASSERT_NE(nullptr, outputPtr);
156 inputMemory->update();
157 outputMemory->update();
158
159 // Go through all inputs, copy the values
160 for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
161 char* begin = (char*)p;
162 char* end = begin + s;
163 // TODO: handle more than one input
164 std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
165 });
166
167 inputMemory->commit();
168 outputMemory->commit();
Michael Butlercf22a572017-09-22 13:26:12 -0700169
170 // launch execution
171 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
172 ASSERT_NE(nullptr, executionCallback.get());
173 Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(
174 {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}, executionCallback);
175 ASSERT_TRUE(executionLaunchStatus.isOk());
176 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
177
178 // retrieve execution status
179 executionCallback->wait();
180 ErrorStatus executionReturnStatus = executionCallback->getStatus();
181 EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700182
183 // validate results
184 outputMemory->read();
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700185 copy_back(&test, outputs_info, outputPtr);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700186 outputMemory->commit();
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700187 // Filter out don't cares
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -0700188 MixedTyped filtered_golden = filter(golden, is_ignored);
189 MixedTyped filtered_test = filter(test, is_ignored);
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700190
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700191 // We want "close-enough" results for float
Xusong Wangf6235f82018-08-28 16:50:01 -0700192 compare(filtered_golden, filtered_test, fpAtol, fpRtol);
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +0100193
194 if (example.expectedMultinomialDistributionTolerance > 0) {
195 expectMultinomialDistributionWithinTolerance(test, example);
196 }
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700197 }
198}
199
Michael Butler7ed61352018-03-22 16:37:57 -0700200void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +0100201 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Miao Wang4862d612018-02-05 17:26:54 -0800202 V1_0::Model model = create_model();
203
204 // see if service can handle model
205 bool fullySupportsModel = false;
Miao Wang4862d612018-02-05 17:26:54 -0800206 Return<void> supportedCall = device->getSupportedOperations(
Michael Butler1ae02d62018-02-26 15:24:46 -0800207 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
208 ASSERT_EQ(ErrorStatus::NONE, status);
Miao Wang4862d612018-02-05 17:26:54 -0800209 ASSERT_NE(0ul, supported.size());
210 fullySupportsModel =
211 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
212 });
213 ASSERT_TRUE(supportedCall.isOk());
Michael Butler1ae02d62018-02-26 15:24:46 -0800214
215 // launch prepare model
216 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
217 ASSERT_NE(nullptr, preparedModelCallback.get());
Miao Wang4862d612018-02-05 17:26:54 -0800218 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
219 ASSERT_TRUE(prepareLaunchStatus.isOk());
Michael Butler1ae02d62018-02-26 15:24:46 -0800220 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Miao Wang4862d612018-02-05 17:26:54 -0800221
222 // retrieve prepared model
223 preparedModelCallback->wait();
224 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
225 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
Miao Wang4862d612018-02-05 17:26:54 -0800226
227 // early termination if vendor service cannot fully prepare model
Michael Butler1ae02d62018-02-26 15:24:46 -0800228 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
Miao Wang4862d612018-02-05 17:26:54 -0800229 ASSERT_EQ(nullptr, preparedModel.get());
230 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
231 "prepare model that it does not support.";
232 std::cout << "[ ] Early termination of test because vendor service cannot "
233 "prepare model that it does not support."
234 << std::endl;
235 return;
236 }
Michael Butler1ae02d62018-02-26 15:24:46 -0800237 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
Miao Wang4862d612018-02-05 17:26:54 -0800238 ASSERT_NE(nullptr, preparedModel.get());
239
Xusong Wangf6235f82018-08-28 16:50:01 -0700240 float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f;
Michael K. Sanders650fd182018-10-30 14:44:48 +0000241 EvaluatePreparedModel(preparedModel, is_ignored, examples,
242 /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol);
Miao Wang4862d612018-02-05 17:26:54 -0800243}
244
Michael Butler7ed61352018-03-22 16:37:57 -0700245void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +0100246 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Miao Wang4862d612018-02-05 17:26:54 -0800247 V1_1::Model model = create_model();
248
249 // see if service can handle model
250 bool fullySupportsModel = false;
Miao Wang4862d612018-02-05 17:26:54 -0800251 Return<void> supportedCall = device->getSupportedOperations_1_1(
Michael Butler1ae02d62018-02-26 15:24:46 -0800252 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
253 ASSERT_EQ(ErrorStatus::NONE, status);
Miao Wang4862d612018-02-05 17:26:54 -0800254 ASSERT_NE(0ul, supported.size());
255 fullySupportsModel =
256 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
257 });
258 ASSERT_TRUE(supportedCall.isOk());
Michael Butler1ae02d62018-02-26 15:24:46 -0800259
260 // launch prepare model
261 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
262 ASSERT_NE(nullptr, preparedModelCallback.get());
Michael Butlerf02692d2018-04-11 16:30:09 -0700263 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
264 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
Miao Wang4862d612018-02-05 17:26:54 -0800265 ASSERT_TRUE(prepareLaunchStatus.isOk());
Michael Butler1ae02d62018-02-26 15:24:46 -0800266 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Miao Wang4862d612018-02-05 17:26:54 -0800267
268 // retrieve prepared model
269 preparedModelCallback->wait();
270 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
271 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
Miao Wang4862d612018-02-05 17:26:54 -0800272
273 // early termination if vendor service cannot fully prepare model
Michael Butler1ae02d62018-02-26 15:24:46 -0800274 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
Miao Wang4862d612018-02-05 17:26:54 -0800275 ASSERT_EQ(nullptr, preparedModel.get());
276 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
277 "prepare model that it does not support.";
278 std::cout << "[ ] Early termination of test because vendor service cannot "
279 "prepare model that it does not support."
280 << std::endl;
281 return;
282 }
Michael Butler1ae02d62018-02-26 15:24:46 -0800283 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
Miao Wang4862d612018-02-05 17:26:54 -0800284 ASSERT_NE(nullptr, preparedModel.get());
285
Michael K. Sanders650fd182018-10-30 14:44:48 +0000286 EvaluatePreparedModel(preparedModel, is_ignored, examples,
287 model.relaxComputationFloat32toFloat16);
Miao Wang4862d612018-02-05 17:26:54 -0800288}
289
Slava Shklyaevfeb87a92018-09-12 14:52:02 +0100290// TODO: Reduce code duplication.
291void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +0100292 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Slava Shklyaevfeb87a92018-09-12 14:52:02 +0100293 V1_2::Model model = create_model();
294
295 // see if service can handle model
296 bool fullySupportsModel = false;
297 Return<void> supportedCall = device->getSupportedOperations_1_2(
298 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
299 ASSERT_EQ(ErrorStatus::NONE, status);
300 ASSERT_NE(0ul, supported.size());
301 fullySupportsModel =
302 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
303 });
304 ASSERT_TRUE(supportedCall.isOk());
305
306 // launch prepare model
307 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
308 ASSERT_NE(nullptr, preparedModelCallback.get());
309 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_2(
310 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
311 ASSERT_TRUE(prepareLaunchStatus.isOk());
312 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
313
314 // retrieve prepared model
315 preparedModelCallback->wait();
316 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
317 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
318
319 // early termination if vendor service cannot fully prepare model
320 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
321 ASSERT_EQ(nullptr, preparedModel.get());
322 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
323 "prepare model that it does not support.";
324 std::cout << "[ ] Early termination of test because vendor service cannot "
325 "prepare model that it does not support."
326 << std::endl;
327 return;
328 }
329 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
330 ASSERT_NE(nullptr, preparedModel.get());
331
Michael K. Sanders650fd182018-10-30 14:44:48 +0000332 EvaluatePreparedModel(preparedModel, is_ignored, examples,
333 model.relaxComputationFloat32toFloat16);
Slava Shklyaevfeb87a92018-09-12 14:52:02 +0100334}
335
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700336} // namespace generated_tests
337
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700338} // namespace neuralnetworks
339} // namespace hardware
340} // namespace android