blob: 8c9a0a369c88c2e81573c554129f3b82a8b1d8b6 [file] [log] [blame]
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Michael Butlercf22a572017-09-22 13:26:12 -070017#include "Callbacks.h"
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070018#include "TestHarness.h"
19#include "VtsHalNeuralnetworksV1_0TargetTest.h"
20
21#include <android-base/logging.h>
22#include <android/hidl/memory/1.0/IMemory.h>
23#include <hidlmemory/mapping.h>
Michael Butler0897ab32017-10-04 02:38:42 -070024#include <iostream>
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070025
26namespace android {
27namespace hardware {
28namespace neuralnetworks {
29namespace V1_0 {
30namespace vts {
31namespace functional {
32// allocator helper
33hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem");
34
35namespace generated_tests {
Michael Butlercf22a572017-09-22 13:26:12 -070036using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
37using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -070038using ::generated_tests::filter;
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070039using ::generated_tests::for_all;
40using ::generated_tests::for_each;
41using ::generated_tests::resize_accordingly;
42using ::generated_tests::MixedTyped;
43using ::generated_tests::MixedTypedExampleType;
44using ::generated_tests::Float32Operands;
45using ::generated_tests::Int32Operands;
46using ::generated_tests::Quant8Operands;
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -070047using ::generated_tests::compare;
48
49template <typename ty>
50void copy_back_(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
51 MixedTyped& test = *dst;
52 for_each(test, [&ra, src](int index, std::vector<ty>& m) {
53 ASSERT_EQ(m.size(), ra[index].location.length / sizeof(ty));
54 char* begin = src + ra[index].location.offset;
55 memcpy(m.data(), begin, ra[index].location.length);
56 });
57}
58
59void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
60 copy_back_<float>(dst, ra, src);
61 copy_back_<int32_t>(dst, ra, src);
62 copy_back_<uint8_t>(dst, ra, src);
63}
64
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070065// Top level driver for models and examples generated by test_generator.py
66// Test driver for those generated from ml/nn/runtime/test/spec
67void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -070068 std::function<bool(int)> is_ignored,
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070069 const std::vector<MixedTypedExampleType>& examples) {
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -070070 const uint32_t INPUT = 0;
71 const uint32_t OUTPUT = 1;
Michael Butlercf22a572017-09-22 13:26:12 -070072 Model model = create_model();
73
Michael Butler0897ab32017-10-04 02:38:42 -070074 // see if service can handle model
75 ErrorStatus supportedStatus;
76 bool fullySupportsModel = false;
77 Return<void> supportedCall = device->getSupportedOperations(
78 model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
79 supportedStatus = status;
80 ASSERT_NE(0ul, supported.size());
81 fullySupportsModel =
82 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
83 });
84 ASSERT_TRUE(supportedCall.isOk());
85 ASSERT_EQ(ErrorStatus::NONE, supportedStatus);
86
Michael Butlercf22a572017-09-22 13:26:12 -070087 // launch prepare model
88 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
89 ASSERT_NE(nullptr, preparedModelCallback.get());
90 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
91 ASSERT_TRUE(prepareLaunchStatus.isOk());
92
93 // retrieve prepared model
94 preparedModelCallback->wait();
95 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
Michael Butlercf22a572017-09-22 13:26:12 -070096 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
Michael Butler0897ab32017-10-04 02:38:42 -070097 if (fullySupportsModel) {
98 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
99 } else {
100 EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||
101 prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);
102 }
Michael Butlercf22a572017-09-22 13:26:12 -0700103 ASSERT_NE(nullptr, preparedModel.get());
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700104
105 int example_no = 1;
106 for (auto& example : examples) {
107 SCOPED_TRACE(example_no++);
108
109 const MixedTyped& inputs = example.first;
110 const MixedTyped& golden = example.second;
111
112 std::vector<RequestArgument> inputs_info, outputs_info;
113 uint32_t inputSize = 0, outputSize = 0;
114
115 // This function only partially specifies the metadata (vector of RequestArguments).
116 // The contents are copied over below.
117 for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
118 if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
119 RequestArgument arg = {
120 .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
121 .dimensions = {},
122 };
123 inputs_info[index] = arg;
124 inputSize += s;
125 });
126 // Compute offset for inputs 1 and so on
127 {
128 size_t offset = 0;
129 for (auto& i : inputs_info) {
130 i.location.offset = offset;
131 offset += i.location.length;
132 }
133 }
134
135 MixedTyped test; // holding test results
136
137 // Go through all outputs, initialize RequestArgument descriptors
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700138 resize_accordingly(golden, test);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700139 for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
140 if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
141 RequestArgument arg = {
142 .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
143 .dimensions = {},
144 };
145 outputs_info[index] = arg;
146 outputSize += s;
147 });
148 // Compute offset for outputs 1 and so on
149 {
150 size_t offset = 0;
151 for (auto& i : outputs_info) {
152 i.location.offset = offset;
153 offset += i.location.length;
154 }
155 }
156 std::vector<hidl_memory> pools = {allocateSharedMemory(inputSize),
157 allocateSharedMemory(outputSize)};
158 ASSERT_NE(0ull, pools[INPUT].size());
159 ASSERT_NE(0ull, pools[OUTPUT].size());
160
161 // load data
162 sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
163 sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
164 ASSERT_NE(nullptr, inputMemory.get());
165 ASSERT_NE(nullptr, outputMemory.get());
166 char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
167 char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));
168 ASSERT_NE(nullptr, inputPtr);
169 ASSERT_NE(nullptr, outputPtr);
170 inputMemory->update();
171 outputMemory->update();
172
173 // Go through all inputs, copy the values
174 for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
175 char* begin = (char*)p;
176 char* end = begin + s;
177 // TODO: handle more than one input
178 std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
179 });
180
181 inputMemory->commit();
182 outputMemory->commit();
Michael Butlercf22a572017-09-22 13:26:12 -0700183
184 // launch execution
185 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
186 ASSERT_NE(nullptr, executionCallback.get());
187 Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(
188 {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}, executionCallback);
189 ASSERT_TRUE(executionLaunchStatus.isOk());
190 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
191
192 // retrieve execution status
193 executionCallback->wait();
194 ErrorStatus executionReturnStatus = executionCallback->getStatus();
Michael Butler0897ab32017-10-04 02:38:42 -0700195 if (!fullySupportsModel &&
196 static_cast<ErrorStatus>(executionReturnStatus) == ErrorStatus::GENERAL_FAILURE) {
197 LOG(INFO) << "Ignoring execution results from model that is not supported by the "
198 "vendor service driver";
199 std::cout << "[ ] Ignoring execution results from model that is not "
200 "supported by the vendor service driver"
201 << std::endl;
202 continue;
203 }
204 LOG(INFO) << "CONTINUING TO CHECK VALUE";
205 std::cout << "[ ] CONTINUING TO CHECK VALUE" << std::endl;
Michael Butlercf22a572017-09-22 13:26:12 -0700206 EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700207
208 // validate results
209 outputMemory->read();
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700210 copy_back(&test, outputs_info, outputPtr);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700211 outputMemory->commit();
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700212 // Filter out don't cares
213 MixedTyped filtered_golden;
214 MixedTyped filtered_test;
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700215 filter(golden, &filtered_golden, is_ignored);
216 filter(test, &filtered_test, is_ignored);
I-Jui (Ray) Sung7d765bd2017-09-13 18:47:12 -0700217
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700218 // We want "close-enough" results for float
I-Jui (Ray) Sungf6b85502017-09-20 13:45:50 -0700219 compare(filtered_golden, filtered_test);
I-Jui (Ray) Sung2c4e1362017-09-06 02:15:54 -0700220 }
221}
222
223} // namespace generated_tests
224
225} // namespace functional
226} // namespace vts
227} // namespace V1_0
228} // namespace neuralnetworks
229} // namespace hardware
230} // namespace android