blob: b8046c79b2365499300ea684e6abe1ef778fd59f [file] [log] [blame]
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Michael Butlercf22a572017-09-22 13:26:12 -070017#include "Callbacks.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070018#include "TestHarness.h"
Miao Wanga2d04c82018-02-05 17:26:54 -080019#include "Utils.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070020
21#include <android-base/logging.h>
Miao Wanga2d04c82018-02-05 17:26:54 -080022#include <android/hardware/neuralnetworks/1.0/IDevice.h>
23#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
24#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
25#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
26#include <android/hardware/neuralnetworks/1.0/types.h>
27#include <android/hidl/allocator/1.0/IAllocator.h>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070028#include <android/hidl/memory/1.0/IMemory.h>
29#include <hidlmemory/mapping.h>
Michael Butler0897ab32017-10-04 02:38:42 -070030#include <iostream>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070031
32namespace android {
33namespace hardware {
34namespace neuralnetworks {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070035
36namespace generated_tests {
Michael Butlercf22a572017-09-22 13:26:12 -070037using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
38using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
Mika Raentode166942018-04-17 16:49:50 +010039using ::test_helper::filter;
40using ::test_helper::for_all;
41using ::test_helper::for_each;
42using ::test_helper::resize_accordingly;
43using ::test_helper::MixedTyped;
44using ::test_helper::MixedTypedExampleType;
45using ::test_helper::Float32Operands;
46using ::test_helper::Int32Operands;
47using ::test_helper::Quant8Operands;
48using ::test_helper::compare;
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070049
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -070050template <typename T>
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070051void copy_back_(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
52 MixedTyped& test = *dst;
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -070053 for_each<T>(test, [&ra, src](int index, std::vector<T>& m) {
54 ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T));
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070055 char* begin = src + ra[index].location.offset;
56 memcpy(m.data(), begin, ra[index].location.length);
57 });
58}
59
60void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
61 copy_back_<float>(dst, ra, src);
62 copy_back_<int32_t>(dst, ra, src);
63 copy_back_<uint8_t>(dst, ra, src);
64}
65
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070066// Top level driver for models and examples generated by test_generator.py
67// Test driver for those generated from ml/nn/runtime/test/spec
Miao Wanga2d04c82018-02-05 17:26:54 -080068void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
Xusong Wang10d77e42018-08-28 16:50:01 -070069 const std::vector<MixedTypedExampleType>& examples, float fpAtol = 1e-5f,
70 float fpRtol = 1e-5f) {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070071 const uint32_t INPUT = 0;
72 const uint32_t OUTPUT = 1;
73
74 int example_no = 1;
75 for (auto& example : examples) {
76 SCOPED_TRACE(example_no++);
77
78 const MixedTyped& inputs = example.first;
79 const MixedTyped& golden = example.second;
80
81 std::vector<RequestArgument> inputs_info, outputs_info;
82 uint32_t inputSize = 0, outputSize = 0;
83
84 // This function only partially specifies the metadata (vector of RequestArguments).
85 // The contents are copied over below.
86 for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
87 if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
88 RequestArgument arg = {
89 .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
90 .dimensions = {},
91 };
I-Jui (Ray) Sung959cd782017-10-04 20:49:57 -070092 RequestArgument arg_empty = {
93 .hasNoValue = true,
94 };
95 inputs_info[index] = s ? arg : arg_empty;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070096 inputSize += s;
97 });
98 // Compute offset for inputs 1 and so on
99 {
100 size_t offset = 0;
101 for (auto& i : inputs_info) {
I-Jui (Ray) Sung959cd782017-10-04 20:49:57 -0700102 if (!i.hasNoValue) i.location.offset = offset;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700103 offset += i.location.length;
104 }
105 }
106
107 MixedTyped test; // holding test results
108
109 // Go through all outputs, initialize RequestArgument descriptors
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700110 resize_accordingly(golden, test);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700111 for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
112 if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
113 RequestArgument arg = {
114 .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
115 .dimensions = {},
116 };
117 outputs_info[index] = arg;
118 outputSize += s;
119 });
120 // Compute offset for outputs 1 and so on
121 {
122 size_t offset = 0;
123 for (auto& i : outputs_info) {
124 i.location.offset = offset;
125 offset += i.location.length;
126 }
127 }
Miao Wanga2d04c82018-02-05 17:26:54 -0800128 std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
129 nn::allocateSharedMemory(outputSize)};
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700130 ASSERT_NE(0ull, pools[INPUT].size());
131 ASSERT_NE(0ull, pools[OUTPUT].size());
132
133 // load data
134 sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
135 sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
136 ASSERT_NE(nullptr, inputMemory.get());
137 ASSERT_NE(nullptr, outputMemory.get());
138 char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
139 char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));
140 ASSERT_NE(nullptr, inputPtr);
141 ASSERT_NE(nullptr, outputPtr);
142 inputMemory->update();
143 outputMemory->update();
144
145 // Go through all inputs, copy the values
146 for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
147 char* begin = (char*)p;
148 char* end = begin + s;
149 // TODO: handle more than one input
150 std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
151 });
152
153 inputMemory->commit();
154 outputMemory->commit();
Michael Butlercf22a572017-09-22 13:26:12 -0700155
156 // launch execution
157 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
158 ASSERT_NE(nullptr, executionCallback.get());
159 Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(
160 {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}, executionCallback);
161 ASSERT_TRUE(executionLaunchStatus.isOk());
162 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
163
164 // retrieve execution status
165 executionCallback->wait();
166 ErrorStatus executionReturnStatus = executionCallback->getStatus();
167 EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700168
169 // validate results
170 outputMemory->read();
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700171 copy_back(&test, outputs_info, outputPtr);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700172 outputMemory->commit();
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700173 // Filter out don't cares
I-Jui (Ray) Sung5bf4edf2017-10-06 13:22:39 -0700174 MixedTyped filtered_golden = filter(golden, is_ignored);
175 MixedTyped filtered_test = filter(test, is_ignored);
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700176
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700177 // We want "close-enough" results for float
Xusong Wang10d77e42018-08-28 16:50:01 -0700178 compare(filtered_golden, filtered_test, fpAtol, fpRtol);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700179 }
180}
181
Michael Butlerf76acd02018-03-22 16:37:57 -0700182void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,
Miao Wanga2d04c82018-02-05 17:26:54 -0800183 std::function<bool(int)> is_ignored,
184 const std::vector<MixedTypedExampleType>& examples) {
185 V1_0::Model model = create_model();
186
187 // see if service can handle model
188 bool fullySupportsModel = false;
Miao Wanga2d04c82018-02-05 17:26:54 -0800189 Return<void> supportedCall = device->getSupportedOperations(
Michael Butler4d5bb102018-02-26 15:24:46 -0800190 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
191 ASSERT_EQ(ErrorStatus::NONE, status);
Miao Wanga2d04c82018-02-05 17:26:54 -0800192 ASSERT_NE(0ul, supported.size());
193 fullySupportsModel =
194 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
195 });
196 ASSERT_TRUE(supportedCall.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800197
198 // launch prepare model
199 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
200 ASSERT_NE(nullptr, preparedModelCallback.get());
Miao Wanga2d04c82018-02-05 17:26:54 -0800201 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
202 ASSERT_TRUE(prepareLaunchStatus.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800203 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Miao Wanga2d04c82018-02-05 17:26:54 -0800204
205 // retrieve prepared model
206 preparedModelCallback->wait();
207 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
208 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
Miao Wanga2d04c82018-02-05 17:26:54 -0800209
210 // early termination if vendor service cannot fully prepare model
Michael Butler4d5bb102018-02-26 15:24:46 -0800211 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
Miao Wanga2d04c82018-02-05 17:26:54 -0800212 ASSERT_EQ(nullptr, preparedModel.get());
213 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
214 "prepare model that it does not support.";
215 std::cout << "[ ] Early termination of test because vendor service cannot "
216 "prepare model that it does not support."
217 << std::endl;
218 return;
219 }
Michael Butler4d5bb102018-02-26 15:24:46 -0800220 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
Miao Wanga2d04c82018-02-05 17:26:54 -0800221 ASSERT_NE(nullptr, preparedModel.get());
222
Xusong Wang10d77e42018-08-28 16:50:01 -0700223 float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f;
224 EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol);
Miao Wanga2d04c82018-02-05 17:26:54 -0800225}
226
Michael Butlerf76acd02018-03-22 16:37:57 -0700227void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
Miao Wanga2d04c82018-02-05 17:26:54 -0800228 std::function<bool(int)> is_ignored,
229 const std::vector<MixedTypedExampleType>& examples) {
230 V1_1::Model model = create_model();
231
232 // see if service can handle model
233 bool fullySupportsModel = false;
Miao Wanga2d04c82018-02-05 17:26:54 -0800234 Return<void> supportedCall = device->getSupportedOperations_1_1(
Michael Butler4d5bb102018-02-26 15:24:46 -0800235 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
236 ASSERT_EQ(ErrorStatus::NONE, status);
Miao Wanga2d04c82018-02-05 17:26:54 -0800237 ASSERT_NE(0ul, supported.size());
238 fullySupportsModel =
239 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
240 });
241 ASSERT_TRUE(supportedCall.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800242
243 // launch prepare model
244 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
245 ASSERT_NE(nullptr, preparedModelCallback.get());
Michael Butler2504c2f2018-04-11 16:30:09 -0700246 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
247 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
Miao Wanga2d04c82018-02-05 17:26:54 -0800248 ASSERT_TRUE(prepareLaunchStatus.isOk());
Michael Butler4d5bb102018-02-26 15:24:46 -0800249 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Miao Wanga2d04c82018-02-05 17:26:54 -0800250
251 // retrieve prepared model
252 preparedModelCallback->wait();
253 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
254 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
Miao Wanga2d04c82018-02-05 17:26:54 -0800255
256 // early termination if vendor service cannot fully prepare model
Michael Butler4d5bb102018-02-26 15:24:46 -0800257 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
Miao Wanga2d04c82018-02-05 17:26:54 -0800258 ASSERT_EQ(nullptr, preparedModel.get());
259 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
260 "prepare model that it does not support.";
261 std::cout << "[ ] Early termination of test because vendor service cannot "
262 "prepare model that it does not support."
263 << std::endl;
264 return;
265 }
Michael Butler4d5bb102018-02-26 15:24:46 -0800266 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
Miao Wanga2d04c82018-02-05 17:26:54 -0800267 ASSERT_NE(nullptr, preparedModel.get());
268
Xusong Wang10d77e42018-08-28 16:50:01 -0700269 // TODO: Adjust the error limit based on testing.
270 // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
271 float fpAtol = !model.relaxComputationFloat32toFloat16 ? 1e-5f : 5.0f * 0.0009765625f;
272 // Set the relative tolerance to be 5ULP of the corresponding FP precision.
273 float fpRtol = !model.relaxComputationFloat32toFloat16 ? 5.0f * 1.1920928955078125e-7f
274 : 5.0f * 0.0009765625f;
275 EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol);
Miao Wanga2d04c82018-02-05 17:26:54 -0800276}
277
Slava Shklyaev871be942018-09-12 14:52:02 +0100278// TODO: Reduce code duplication.
279void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,
280 std::function<bool(int)> is_ignored,
281 const std::vector<MixedTypedExampleType>& examples) {
282 V1_2::Model model = create_model();
283
284 // see if service can handle model
285 bool fullySupportsModel = false;
286 Return<void> supportedCall = device->getSupportedOperations_1_2(
287 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
288 ASSERT_EQ(ErrorStatus::NONE, status);
289 ASSERT_NE(0ul, supported.size());
290 fullySupportsModel =
291 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
292 });
293 ASSERT_TRUE(supportedCall.isOk());
294
295 // launch prepare model
296 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
297 ASSERT_NE(nullptr, preparedModelCallback.get());
298 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_2(
299 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
300 ASSERT_TRUE(prepareLaunchStatus.isOk());
301 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
302
303 // retrieve prepared model
304 preparedModelCallback->wait();
305 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
306 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
307
308 // early termination if vendor service cannot fully prepare model
309 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
310 ASSERT_EQ(nullptr, preparedModel.get());
311 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
312 "prepare model that it does not support.";
313 std::cout << "[ ] Early termination of test because vendor service cannot "
314 "prepare model that it does not support."
315 << std::endl;
316 return;
317 }
318 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
319 ASSERT_NE(nullptr, preparedModel.get());
320
321 // TODO: Adjust the error limit based on testing.
322 // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
323 float fpAtol = !model.relaxComputationFloat32toFloat16 ? 1e-5f : 5.0f * 0.0009765625f;
324 // Set the relative tolerance to be 5ULP of the corresponding FP precision.
325 float fpRtol = !model.relaxComputationFloat32toFloat16 ? 5.0f * 1.1920928955078125e-7f
326 : 5.0f * 0.0009765625f;
327 EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol);
328}
329
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700330} // namespace generated_tests
331
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700332} // namespace neuralnetworks
333} // namespace hardware
334} // namespace android