blob: 1f66c43bf92af37bddc11ee3d8034c9b8ae996ae [file] [log] [blame]
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Michael Butlercf22a572017-09-22 13:26:12 -070017#include "Callbacks.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070018#include "TestHarness.h"
Miao Wang4862d612018-02-05 17:26:54 -080019#include "Utils.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070020
21#include <android-base/logging.h>
Miao Wang4862d612018-02-05 17:26:54 -080022#include <android/hardware/neuralnetworks/1.0/IDevice.h>
23#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
24#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
25#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
26#include <android/hardware/neuralnetworks/1.0/types.h>
27#include <android/hidl/allocator/1.0/IAllocator.h>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070028#include <android/hidl/memory/1.0/IMemory.h>
29#include <hidlmemory/mapping.h>
Michael Butler0897ab32017-10-04 02:38:42 -070030#include <iostream>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070031
32namespace android {
33namespace hardware {
34namespace neuralnetworks {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070035
36namespace generated_tests {
Michael Butlercf22a572017-09-22 13:26:12 -070037using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
38using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010039using ::test_helper::compare;
40using ::test_helper::expectMultinomialDistributionWithinTolerance;
Mika Raentod534d322018-04-17 16:49:50 +010041using ::test_helper::filter;
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010042using ::test_helper::Float32Operands;
Mika Raentod534d322018-04-17 16:49:50 +010043using ::test_helper::for_all;
44using ::test_helper::for_each;
Mika Raentod534d322018-04-17 16:49:50 +010045using ::test_helper::Int32Operands;
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010046using ::test_helper::MixedTyped;
47using ::test_helper::MixedTypedExample;
Mika Raentod534d322018-04-17 16:49:50 +010048using ::test_helper::Quant8Operands;
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010049using ::test_helper::resize_accordingly;
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070050
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -070051template <typename T>
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070052void copy_back_(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
53 MixedTyped& test = *dst;
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -070054 for_each<T>(test, [&ra, src](int index, std::vector<T>& m) {
55 ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T));
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070056 char* begin = src + ra[index].location.offset;
57 memcpy(m.data(), begin, ra[index].location.length);
58 });
59}
60
61void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
62 copy_back_<float>(dst, ra, src);
63 copy_back_<int32_t>(dst, ra, src);
64 copy_back_<uint8_t>(dst, ra, src);
Lev Proleeved7ce7a2018-11-05 13:20:06 +000065 copy_back_<int16_t>(dst, ra, src);
66 static_assert(4 == std::tuple_size<MixedTyped>::value,
Lev Proleevd36b7a82018-11-02 12:44:11 +000067 "Number of types in MixedTyped changed, but copy_back function wasn't updated");
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070068}
69
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070070// Top level driver for models and examples generated by test_generator.py
71// Test driver for those generated from ml/nn/runtime/test/spec
Miao Wang4862d612018-02-05 17:26:54 -080072void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010073 const std::vector<MixedTypedExample>& examples, float fpAtol = 1e-5f,
Xusong Wangf6235f82018-08-28 16:50:01 -070074 float fpRtol = 1e-5f) {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070075 const uint32_t INPUT = 0;
76 const uint32_t OUTPUT = 1;
77
78 int example_no = 1;
79 for (auto& example : examples) {
80 SCOPED_TRACE(example_no++);
81
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +010082 const MixedTyped& inputs = example.operands.first;
83 const MixedTyped& golden = example.operands.second;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070084
85 std::vector<RequestArgument> inputs_info, outputs_info;
86 uint32_t inputSize = 0, outputSize = 0;
87
88 // This function only partially specifies the metadata (vector of RequestArguments).
89 // The contents are copied over below.
90 for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
91 if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
92 RequestArgument arg = {
93 .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
94 .dimensions = {},
95 };
I-Jui (Ray) Sung959cd782017-10-04 20:49:57 -070096 RequestArgument arg_empty = {
97 .hasNoValue = true,
98 };
99 inputs_info[index] = s ? arg : arg_empty;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700100 inputSize += s;
101 });
102 // Compute offset for inputs 1 and so on
103 {
104 size_t offset = 0;
105 for (auto& i : inputs_info) {
I-Jui (Ray) Sung959cd782017-10-04 20:49:57 -0700106 if (!i.hasNoValue) i.location.offset = offset;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700107 offset += i.location.length;
108 }
109 }
110
111 MixedTyped test; // holding test results
112
113 // Go through all outputs, initialize RequestArgument descriptors
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700114 resize_accordingly(golden, test);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700115 for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
116 if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
117 RequestArgument arg = {
118 .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
119 .dimensions = {},
120 };
121 outputs_info[index] = arg;
122 outputSize += s;
123 });
124 // Compute offset for outputs 1 and so on
125 {
126 size_t offset = 0;
127 for (auto& i : outputs_info) {
128 i.location.offset = offset;
129 offset += i.location.length;
130 }
131 }
Miao Wang4862d612018-02-05 17:26:54 -0800132 std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
133 nn::allocateSharedMemory(outputSize)};
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700134 ASSERT_NE(0ull, pools[INPUT].size());
135 ASSERT_NE(0ull, pools[OUTPUT].size());
136
137 // load data
138 sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
139 sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
140 ASSERT_NE(nullptr, inputMemory.get());
141 ASSERT_NE(nullptr, outputMemory.get());
142 char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
143 char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));
144 ASSERT_NE(nullptr, inputPtr);
145 ASSERT_NE(nullptr, outputPtr);
146 inputMemory->update();
147 outputMemory->update();
148
149 // Go through all inputs, copy the values
150 for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
151 char* begin = (char*)p;
152 char* end = begin + s;
153 // TODO: handle more than one input
154 std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
155 });
156
157 inputMemory->commit();
158 outputMemory->commit();
Michael Butlercf22a572017-09-22 13:26:12 -0700159
160 // launch execution
161 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
162 ASSERT_NE(nullptr, executionCallback.get());
163 Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(
164 {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}, executionCallback);
165 ASSERT_TRUE(executionLaunchStatus.isOk());
166 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
167
168 // retrieve execution status
169 executionCallback->wait();
170 ErrorStatus executionReturnStatus = executionCallback->getStatus();
171 EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700172
173 // validate results
174 outputMemory->read();
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700175 copy_back(&test, outputs_info, outputPtr);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700176 outputMemory->commit();
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700177 // Filter out don't cares
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -0700178 MixedTyped filtered_golden = filter(golden, is_ignored);
179 MixedTyped filtered_test = filter(test, is_ignored);
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700180
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700181 // We want "close-enough" results for float
Xusong Wangf6235f82018-08-28 16:50:01 -0700182 compare(filtered_golden, filtered_test, fpAtol, fpRtol);
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +0100183
184 if (example.expectedMultinomialDistributionTolerance > 0) {
185 expectMultinomialDistributionWithinTolerance(test, example);
186 }
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700187 }
188}
189
Michael Butler7ed61352018-03-22 16:37:57 -0700190void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +0100191 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Miao Wang4862d612018-02-05 17:26:54 -0800192 V1_0::Model model = create_model();
193
194 // see if service can handle model
195 bool fullySupportsModel = false;
Miao Wang4862d612018-02-05 17:26:54 -0800196 Return<void> supportedCall = device->getSupportedOperations(
Michael Butler1ae02d62018-02-26 15:24:46 -0800197 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
198 ASSERT_EQ(ErrorStatus::NONE, status);
Miao Wang4862d612018-02-05 17:26:54 -0800199 ASSERT_NE(0ul, supported.size());
200 fullySupportsModel =
201 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
202 });
203 ASSERT_TRUE(supportedCall.isOk());
Michael Butler1ae02d62018-02-26 15:24:46 -0800204
205 // launch prepare model
206 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
207 ASSERT_NE(nullptr, preparedModelCallback.get());
Miao Wang4862d612018-02-05 17:26:54 -0800208 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
209 ASSERT_TRUE(prepareLaunchStatus.isOk());
Michael Butler1ae02d62018-02-26 15:24:46 -0800210 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Miao Wang4862d612018-02-05 17:26:54 -0800211
212 // retrieve prepared model
213 preparedModelCallback->wait();
214 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
215 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
Miao Wang4862d612018-02-05 17:26:54 -0800216
217 // early termination if vendor service cannot fully prepare model
Michael Butler1ae02d62018-02-26 15:24:46 -0800218 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
Miao Wang4862d612018-02-05 17:26:54 -0800219 ASSERT_EQ(nullptr, preparedModel.get());
220 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
221 "prepare model that it does not support.";
222 std::cout << "[ ] Early termination of test because vendor service cannot "
223 "prepare model that it does not support."
224 << std::endl;
225 return;
226 }
Michael Butler1ae02d62018-02-26 15:24:46 -0800227 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
Miao Wang4862d612018-02-05 17:26:54 -0800228 ASSERT_NE(nullptr, preparedModel.get());
229
Xusong Wangf6235f82018-08-28 16:50:01 -0700230 float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f;
231 EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol);
Miao Wang4862d612018-02-05 17:26:54 -0800232}
233
Michael Butler7ed61352018-03-22 16:37:57 -0700234void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +0100235 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Miao Wang4862d612018-02-05 17:26:54 -0800236 V1_1::Model model = create_model();
237
238 // see if service can handle model
239 bool fullySupportsModel = false;
Miao Wang4862d612018-02-05 17:26:54 -0800240 Return<void> supportedCall = device->getSupportedOperations_1_1(
Michael Butler1ae02d62018-02-26 15:24:46 -0800241 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
242 ASSERT_EQ(ErrorStatus::NONE, status);
Miao Wang4862d612018-02-05 17:26:54 -0800243 ASSERT_NE(0ul, supported.size());
244 fullySupportsModel =
245 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
246 });
247 ASSERT_TRUE(supportedCall.isOk());
Michael Butler1ae02d62018-02-26 15:24:46 -0800248
249 // launch prepare model
250 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
251 ASSERT_NE(nullptr, preparedModelCallback.get());
Michael Butlerf02692d2018-04-11 16:30:09 -0700252 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
253 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
Miao Wang4862d612018-02-05 17:26:54 -0800254 ASSERT_TRUE(prepareLaunchStatus.isOk());
Michael Butler1ae02d62018-02-26 15:24:46 -0800255 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Miao Wang4862d612018-02-05 17:26:54 -0800256
257 // retrieve prepared model
258 preparedModelCallback->wait();
259 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
260 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
Miao Wang4862d612018-02-05 17:26:54 -0800261
262 // early termination if vendor service cannot fully prepare model
Michael Butler1ae02d62018-02-26 15:24:46 -0800263 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
Miao Wang4862d612018-02-05 17:26:54 -0800264 ASSERT_EQ(nullptr, preparedModel.get());
265 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
266 "prepare model that it does not support.";
267 std::cout << "[ ] Early termination of test because vendor service cannot "
268 "prepare model that it does not support."
269 << std::endl;
270 return;
271 }
Michael Butler1ae02d62018-02-26 15:24:46 -0800272 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
Miao Wang4862d612018-02-05 17:26:54 -0800273 ASSERT_NE(nullptr, preparedModel.get());
274
Xusong Wangf6235f82018-08-28 16:50:01 -0700275 // TODO: Adjust the error limit based on testing.
276 // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
277 float fpAtol = !model.relaxComputationFloat32toFloat16 ? 1e-5f : 5.0f * 0.0009765625f;
278 // Set the relative tolerance to be 5ULP of the corresponding FP precision.
279 float fpRtol = !model.relaxComputationFloat32toFloat16 ? 5.0f * 1.1920928955078125e-7f
280 : 5.0f * 0.0009765625f;
281 EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol);
Miao Wang4862d612018-02-05 17:26:54 -0800282}
283
Slava Shklyaevfeb87a92018-09-12 14:52:02 +0100284// TODO: Reduce code duplication.
285void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,
Michael K. Sandersda3bdbc2018-10-19 14:39:09 +0100286 std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
Slava Shklyaevfeb87a92018-09-12 14:52:02 +0100287 V1_2::Model model = create_model();
288
289 // see if service can handle model
290 bool fullySupportsModel = false;
291 Return<void> supportedCall = device->getSupportedOperations_1_2(
292 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
293 ASSERT_EQ(ErrorStatus::NONE, status);
294 ASSERT_NE(0ul, supported.size());
295 fullySupportsModel =
296 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
297 });
298 ASSERT_TRUE(supportedCall.isOk());
299
300 // launch prepare model
301 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
302 ASSERT_NE(nullptr, preparedModelCallback.get());
303 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_2(
304 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
305 ASSERT_TRUE(prepareLaunchStatus.isOk());
306 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
307
308 // retrieve prepared model
309 preparedModelCallback->wait();
310 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
311 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
312
313 // early termination if vendor service cannot fully prepare model
314 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
315 ASSERT_EQ(nullptr, preparedModel.get());
316 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
317 "prepare model that it does not support.";
318 std::cout << "[ ] Early termination of test because vendor service cannot "
319 "prepare model that it does not support."
320 << std::endl;
321 return;
322 }
323 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
324 ASSERT_NE(nullptr, preparedModel.get());
325
326 // TODO: Adjust the error limit based on testing.
327 // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
328 float fpAtol = !model.relaxComputationFloat32toFloat16 ? 1e-5f : 5.0f * 0.0009765625f;
329 // Set the relative tolerance to be 5ULP of the corresponding FP precision.
330 float fpRtol = !model.relaxComputationFloat32toFloat16 ? 5.0f * 1.1920928955078125e-7f
331 : 5.0f * 0.0009765625f;
332 EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol);
333}
334
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700335} // namespace generated_tests
336
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700337} // namespace neuralnetworks
338} // namespace hardware
339} // namespace android