I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2017 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Michael Butler | cf22a57 | 2017-09-22 13:26:12 -0700 | [diff] [blame] | 17 | #include "Callbacks.h" |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 18 | #include "ExecutionBurstController.h" |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 19 | #include "TestHarness.h" |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 20 | #include "Utils.h" |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 21 | |
| 22 | #include <android-base/logging.h> |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 23 | #include <android/hardware/neuralnetworks/1.0/IDevice.h> |
| 24 | #include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h> |
| 25 | #include <android/hardware/neuralnetworks/1.0/IPreparedModel.h> |
| 26 | #include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h> |
| 27 | #include <android/hardware/neuralnetworks/1.0/types.h> |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 28 | #include <android/hardware/neuralnetworks/1.1/IDevice.h> |
| 29 | #include <android/hardware/neuralnetworks/1.2/IDevice.h> |
| 30 | #include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h> |
| 31 | #include <android/hardware/neuralnetworks/1.2/IPreparedModel.h> |
| 32 | #include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h> |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 33 | #include <android/hidl/allocator/1.0/IAllocator.h> |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 34 | #include <android/hidl/memory/1.0/IMemory.h> |
| 35 | #include <hidlmemory/mapping.h> |
Michael Butler | 0897ab3 | 2017-10-04 02:38:42 -0700 | [diff] [blame] | 36 | #include <iostream> |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 37 | |
| 38 | namespace android { |
| 39 | namespace hardware { |
| 40 | namespace neuralnetworks { |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 41 | |
| 42 | namespace generated_tests { |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 43 | using ::android::hardware::neuralnetworks::V1_2::implementation::ExecutionCallback; |
| 44 | using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCallback; |
Slava Shklyaev | 9e3fad1 | 2018-11-30 17:55:12 +0000 | [diff] [blame] | 45 | using ::test_helper::bool8; |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 46 | using ::test_helper::compare; |
| 47 | using ::test_helper::expectMultinomialDistributionWithinTolerance; |
Mika Raento | de16694 | 2018-04-17 16:49:50 +0100 | [diff] [blame] | 48 | using ::test_helper::filter; |
| 49 | using ::test_helper::for_all; |
| 50 | using ::test_helper::for_each; |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 51 | using ::test_helper::MixedTyped; |
| 52 | using ::test_helper::MixedTypedExample; |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 53 | using ::test_helper::resize_accordingly; |
I-Jui (Ray) Sung | f6b8550 | 2017-09-20 13:45:50 -0700 | [diff] [blame] | 54 | |
I-Jui (Ray) Sung | 5bf4edf | 2017-10-06 13:22:39 -0700 | [diff] [blame] | 55 | template <typename T> |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 56 | void copy_back_(std::map<int, std::vector<T>>* dst, const std::vector<RequestArgument>& ra, |
| 57 | char* src) { |
| 58 | for_each<T>(*dst, [&ra, src](int index, std::vector<T>& m) { |
I-Jui (Ray) Sung | 5bf4edf | 2017-10-06 13:22:39 -0700 | [diff] [blame] | 59 | ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T)); |
I-Jui (Ray) Sung | f6b8550 | 2017-09-20 13:45:50 -0700 | [diff] [blame] | 60 | char* begin = src + ra[index].location.offset; |
| 61 | memcpy(m.data(), begin, ra[index].location.length); |
| 62 | }); |
| 63 | } |
| 64 | |
| 65 | void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) { |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 66 | copy_back_(&dst->float32Operands, ra, src); |
| 67 | copy_back_(&dst->int32Operands, ra, src); |
Xusong Wang | d49f665 | 2019-01-16 18:32:24 -0800 | [diff] [blame] | 68 | copy_back_(&dst->quant8AsymmOperands, ra, src); |
| 69 | copy_back_(&dst->quant16SymmOperands, ra, src); |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 70 | copy_back_(&dst->float16Operands, ra, src); |
| 71 | copy_back_(&dst->bool8Operands, ra, src); |
| 72 | copy_back_(&dst->quant8ChannelOperands, ra, src); |
Xusong Wang | d49f665 | 2019-01-16 18:32:24 -0800 | [diff] [blame] | 73 | copy_back_(&dst->quant16AsymmOperands, ra, src); |
| 74 | static_assert(8 == MixedTyped::kNumTypes, |
Lev Proleev | 9b490f4 | 2018-11-02 12:44:11 +0000 | [diff] [blame] | 75 | "Number of types in MixedTyped changed, but copy_back function wasn't updated"); |
I-Jui (Ray) Sung | f6b8550 | 2017-09-20 13:45:50 -0700 | [diff] [blame] | 76 | } |
| 77 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 78 | // Top level driver for models and examples generated by test_generator.py |
| 79 | // Test driver for those generated from ml/nn/runtime/test/spec |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 80 | static Return<ErrorStatus> ExecutePreparedModel(sp<V1_0::IPreparedModel>& preparedModel, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 81 | const Request& request, MeasureTiming, |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 82 | sp<ExecutionCallback>& callback) { |
| 83 | return preparedModel->execute(request, callback); |
| 84 | } |
| 85 | static Return<ErrorStatus> ExecutePreparedModel(sp<V1_2::IPreparedModel>& preparedModel, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 86 | const Request& request, MeasureTiming measure, |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 87 | sp<ExecutionCallback>& callback) { |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 88 | return preparedModel->execute_1_2(request, measure, callback); |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 89 | } |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 90 | static Return<ErrorStatus> ExecutePreparedModel(sp<V1_0::IPreparedModel>&, const Request&, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 91 | MeasureTiming, hidl_vec<OutputShape>*, Timing*) { |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 92 | ADD_FAILURE() << "asking for synchronous execution at V1_0"; |
| 93 | return ErrorStatus::GENERAL_FAILURE; |
| 94 | } |
| 95 | static Return<ErrorStatus> ExecutePreparedModel(sp<V1_2::IPreparedModel>& preparedModel, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 96 | const Request& request, MeasureTiming measure, |
| 97 | hidl_vec<OutputShape>* outputShapes, |
| 98 | Timing* timing) { |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 99 | ErrorStatus result; |
| 100 | Return<void> ret = preparedModel->executeSynchronously( |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 101 | request, measure, |
| 102 | [&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes, |
| 103 | const Timing& time) { |
| 104 | result = error; |
| 105 | *outputShapes = shapes; |
| 106 | *timing = time; |
| 107 | }); |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 108 | if (!ret.isOk()) { |
| 109 | return ErrorStatus::GENERAL_FAILURE; |
| 110 | } |
| 111 | return result; |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 112 | } |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 113 | static std::unique_ptr<::android::nn::ExecutionBurstController> CreateBurst( |
| 114 | const sp<V1_0::IPreparedModel>&) { |
| 115 | ADD_FAILURE() << "asking for burst execution at V1_0"; |
| 116 | return nullptr; |
| 117 | } |
| 118 | static std::unique_ptr<::android::nn::ExecutionBurstController> CreateBurst( |
| 119 | const sp<V1_2::IPreparedModel>& preparedModel) { |
| 120 | return ::android::nn::createExecutionBurstController(preparedModel, /*blocking=*/true); |
| 121 | } |
| 122 | enum class Executor { ASYNC, SYNC, BURST }; |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 123 | const float kDefaultAtol = 1e-5f; |
| 124 | const float kDefaultRtol = 1e-5f; |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 125 | template <typename T_IPreparedModel> |
| 126 | void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored, |
Michael K. Sanders | efa4c81 | 2018-10-30 14:44:48 +0000 | [diff] [blame] | 127 | const std::vector<MixedTypedExample>& examples, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 128 | bool hasRelaxedFloat32Model, float fpAtol, float fpRtol, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 129 | Executor executor, MeasureTiming measure, bool testDynamicOutputShape) { |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 130 | const uint32_t INPUT = 0; |
| 131 | const uint32_t OUTPUT = 1; |
| 132 | |
| 133 | int example_no = 1; |
| 134 | for (auto& example : examples) { |
| 135 | SCOPED_TRACE(example_no++); |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 136 | const MixedTyped& inputs = example.operands.first; |
| 137 | const MixedTyped& golden = example.operands.second; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 138 | |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 139 | const bool hasFloat16Inputs = !inputs.float16Operands.empty(); |
Michael K. Sanders | efa4c81 | 2018-10-30 14:44:48 +0000 | [diff] [blame] | 140 | if (hasRelaxedFloat32Model || hasFloat16Inputs) { |
| 141 | // TODO: Adjust the error limit based on testing. |
| 142 | // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16. |
| 143 | fpAtol = 5.0f * 0.0009765625f; |
| 144 | // Set the relative tolerance to be 5ULP of the corresponding FP precision. |
| 145 | fpRtol = 5.0f * 0.0009765625f; |
| 146 | } |
| 147 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 148 | std::vector<RequestArgument> inputs_info, outputs_info; |
| 149 | uint32_t inputSize = 0, outputSize = 0; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 150 | // This function only partially specifies the metadata (vector of RequestArguments). |
| 151 | // The contents are copied over below. |
| 152 | for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) { |
| 153 | if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1); |
| 154 | RequestArgument arg = { |
| 155 | .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)}, |
| 156 | .dimensions = {}, |
| 157 | }; |
I-Jui (Ray) Sung | 959cd78 | 2017-10-04 20:49:57 -0700 | [diff] [blame] | 158 | RequestArgument arg_empty = { |
| 159 | .hasNoValue = true, |
| 160 | }; |
| 161 | inputs_info[index] = s ? arg : arg_empty; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 162 | inputSize += s; |
| 163 | }); |
| 164 | // Compute offset for inputs 1 and so on |
| 165 | { |
| 166 | size_t offset = 0; |
| 167 | for (auto& i : inputs_info) { |
I-Jui (Ray) Sung | 959cd78 | 2017-10-04 20:49:57 -0700 | [diff] [blame] | 168 | if (!i.hasNoValue) i.location.offset = offset; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 169 | offset += i.location.length; |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | MixedTyped test; // holding test results |
| 174 | |
| 175 | // Go through all outputs, initialize RequestArgument descriptors |
I-Jui (Ray) Sung | f6b8550 | 2017-09-20 13:45:50 -0700 | [diff] [blame] | 176 | resize_accordingly(golden, test); |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 177 | for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) { |
| 178 | if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1); |
| 179 | RequestArgument arg = { |
| 180 | .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)}, |
| 181 | .dimensions = {}, |
| 182 | }; |
| 183 | outputs_info[index] = arg; |
| 184 | outputSize += s; |
| 185 | }); |
| 186 | // Compute offset for outputs 1 and so on |
| 187 | { |
| 188 | size_t offset = 0; |
| 189 | for (auto& i : outputs_info) { |
| 190 | i.location.offset = offset; |
| 191 | offset += i.location.length; |
| 192 | } |
| 193 | } |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 194 | std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize), |
| 195 | nn::allocateSharedMemory(outputSize)}; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 196 | ASSERT_NE(0ull, pools[INPUT].size()); |
| 197 | ASSERT_NE(0ull, pools[OUTPUT].size()); |
| 198 | |
| 199 | // load data |
| 200 | sp<IMemory> inputMemory = mapMemory(pools[INPUT]); |
| 201 | sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]); |
| 202 | ASSERT_NE(nullptr, inputMemory.get()); |
| 203 | ASSERT_NE(nullptr, outputMemory.get()); |
| 204 | char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer())); |
| 205 | char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer())); |
| 206 | ASSERT_NE(nullptr, inputPtr); |
| 207 | ASSERT_NE(nullptr, outputPtr); |
| 208 | inputMemory->update(); |
| 209 | outputMemory->update(); |
| 210 | |
| 211 | // Go through all inputs, copy the values |
| 212 | for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) { |
| 213 | char* begin = (char*)p; |
| 214 | char* end = begin + s; |
| 215 | // TODO: handle more than one input |
| 216 | std::copy(begin, end, inputPtr + inputs_info[index].location.offset); |
| 217 | }); |
| 218 | |
| 219 | inputMemory->commit(); |
| 220 | outputMemory->commit(); |
Michael Butler | cf22a57 | 2017-09-22 13:26:12 -0700 | [diff] [blame] | 221 | |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 222 | const Request request = {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}; |
| 223 | |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 224 | ErrorStatus executionStatus; |
| 225 | hidl_vec<OutputShape> outputShapes; |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 226 | Timing timing; |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 227 | switch (executor) { |
| 228 | case Executor::ASYNC: { |
| 229 | SCOPED_TRACE("asynchronous"); |
Michael Butler | cf22a57 | 2017-09-22 13:26:12 -0700 | [diff] [blame] | 230 | |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 231 | // launch execution |
| 232 | sp<ExecutionCallback> executionCallback = new ExecutionCallback(); |
| 233 | ASSERT_NE(nullptr, executionCallback.get()); |
| 234 | Return<ErrorStatus> executionLaunchStatus = |
| 235 | ExecutePreparedModel(preparedModel, request, measure, executionCallback); |
| 236 | ASSERT_TRUE(executionLaunchStatus.isOk()); |
| 237 | EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus)); |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 238 | |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 239 | // retrieve execution status |
| 240 | executionCallback->wait(); |
| 241 | executionStatus = executionCallback->getStatus(); |
| 242 | outputShapes = executionCallback->getOutputShapes(); |
| 243 | timing = executionCallback->getTiming(); |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 244 | |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 245 | break; |
| 246 | } |
| 247 | case Executor::SYNC: { |
| 248 | SCOPED_TRACE("synchronous"); |
| 249 | |
| 250 | // execute |
| 251 | Return<ErrorStatus> executionReturnStatus = ExecutePreparedModel( |
| 252 | preparedModel, request, measure, &outputShapes, &timing); |
| 253 | ASSERT_TRUE(executionReturnStatus.isOk()); |
| 254 | executionStatus = static_cast<ErrorStatus>(executionReturnStatus); |
| 255 | |
| 256 | break; |
| 257 | } |
| 258 | case Executor::BURST: { |
| 259 | SCOPED_TRACE("burst"); |
| 260 | |
| 261 | // create burst |
| 262 | const std::unique_ptr<::android::nn::ExecutionBurstController> controller = |
| 263 | CreateBurst(preparedModel); |
| 264 | ASSERT_NE(nullptr, controller.get()); |
| 265 | |
| 266 | // create memory keys |
| 267 | std::vector<intptr_t> keys(request.pools.size()); |
| 268 | for (size_t i = 0; i < keys.size(); ++i) { |
| 269 | keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]); |
| 270 | } |
| 271 | |
| 272 | // execute burst |
| 273 | std::tie(executionStatus, outputShapes, timing) = |
| 274 | controller->compute(request, measure, keys); |
| 275 | |
| 276 | break; |
| 277 | } |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 278 | } |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 279 | |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 280 | if (testDynamicOutputShape && executionStatus != ErrorStatus::NONE) { |
| 281 | LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " |
| 282 | "execute model that it does not support."; |
| 283 | std::cout << "[ ] Early termination of test because vendor service cannot " |
| 284 | "execute model that it does not support." |
| 285 | << std::endl; |
| 286 | return; |
| 287 | } |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 288 | ASSERT_EQ(ErrorStatus::NONE, executionStatus); |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 289 | if (measure == MeasureTiming::NO) { |
| 290 | EXPECT_EQ(UINT64_MAX, timing.timeOnDevice); |
| 291 | EXPECT_EQ(UINT64_MAX, timing.timeInDriver); |
| 292 | } else { |
| 293 | if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) { |
| 294 | EXPECT_LE(timing.timeOnDevice, timing.timeInDriver); |
| 295 | } |
| 296 | } |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 297 | |
| 298 | // Go through all outputs, overwrite output dimensions with returned output shapes |
| 299 | if (testDynamicOutputShape) { |
| 300 | ASSERT_NE(outputShapes.size(), 0); |
| 301 | for_each<uint32_t>(test.operandDimensions, |
| 302 | [&outputShapes](int idx, std::vector<uint32_t>& dim) { |
| 303 | dim = outputShapes[idx].dimensions; |
| 304 | }); |
| 305 | } |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 306 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 307 | // validate results |
| 308 | outputMemory->read(); |
I-Jui (Ray) Sung | f6b8550 | 2017-09-20 13:45:50 -0700 | [diff] [blame] | 309 | copy_back(&test, outputs_info, outputPtr); |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 310 | outputMemory->commit(); |
I-Jui (Ray) Sung | 7d765bd | 2017-09-13 18:47:12 -0700 | [diff] [blame] | 311 | // Filter out don't cares |
I-Jui (Ray) Sung | 5bf4edf | 2017-10-06 13:22:39 -0700 | [diff] [blame] | 312 | MixedTyped filtered_golden = filter(golden, is_ignored); |
| 313 | MixedTyped filtered_test = filter(test, is_ignored); |
I-Jui (Ray) Sung | 7d765bd | 2017-09-13 18:47:12 -0700 | [diff] [blame] | 314 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 315 | // We want "close-enough" results for float |
Xusong Wang | 10d77e4 | 2018-08-28 16:50:01 -0700 | [diff] [blame] | 316 | compare(filtered_golden, filtered_test, fpAtol, fpRtol); |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 317 | |
| 318 | if (example.expectedMultinomialDistributionTolerance > 0) { |
| 319 | expectMultinomialDistributionWithinTolerance(test, example); |
| 320 | } |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 321 | } |
| 322 | } |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 323 | template <typename T_IPreparedModel> |
| 324 | void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored, |
| 325 | const std::vector<MixedTypedExample>& examples, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 326 | bool hasRelaxedFloat32Model, Executor executor, MeasureTiming measure, |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 327 | bool testDynamicOutputShape) { |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 328 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, kDefaultAtol, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 329 | kDefaultRtol, executor, measure, testDynamicOutputShape); |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 330 | } |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 331 | |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 332 | static void getPreparedModel(sp<PreparedModelCallback> callback, |
| 333 | sp<V1_0::IPreparedModel>* preparedModel) { |
| 334 | *preparedModel = callback->getPreparedModel(); |
| 335 | } |
| 336 | static void getPreparedModel(sp<PreparedModelCallback> callback, |
| 337 | sp<V1_2::IPreparedModel>* preparedModel) { |
| 338 | sp<V1_0::IPreparedModel> preparedModelV1_0 = callback->getPreparedModel(); |
| 339 | *preparedModel = V1_2::IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr); |
| 340 | } |
| 341 | |
Michael Butler | f76acd0 | 2018-03-22 16:37:57 -0700 | [diff] [blame] | 342 | void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model, |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 343 | std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) { |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 344 | V1_0::Model model = create_model(); |
| 345 | |
| 346 | // see if service can handle model |
| 347 | bool fullySupportsModel = false; |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 348 | Return<void> supportedCall = device->getSupportedOperations( |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 349 | model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) { |
| 350 | ASSERT_EQ(ErrorStatus::NONE, status); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 351 | ASSERT_NE(0ul, supported.size()); |
| 352 | fullySupportsModel = |
| 353 | std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); |
| 354 | }); |
| 355 | ASSERT_TRUE(supportedCall.isOk()); |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 356 | |
| 357 | // launch prepare model |
| 358 | sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); |
| 359 | ASSERT_NE(nullptr, preparedModelCallback.get()); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 360 | Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); |
| 361 | ASSERT_TRUE(prepareLaunchStatus.isOk()); |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 362 | ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus)); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 363 | |
| 364 | // retrieve prepared model |
| 365 | preparedModelCallback->wait(); |
| 366 | ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 367 | sp<V1_0::IPreparedModel> preparedModel; |
| 368 | getPreparedModel(preparedModelCallback, &preparedModel); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 369 | |
| 370 | // early termination if vendor service cannot fully prepare model |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 371 | if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 372 | ASSERT_EQ(nullptr, preparedModel.get()); |
| 373 | LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " |
| 374 | "prepare model that it does not support."; |
| 375 | std::cout << "[ ] Early termination of test because vendor service cannot " |
| 376 | "prepare model that it does not support." |
| 377 | << std::endl; |
| 378 | return; |
| 379 | } |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 380 | EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 381 | ASSERT_NE(nullptr, preparedModel.get()); |
| 382 | |
Xusong Wang | 10d77e4 | 2018-08-28 16:50:01 -0700 | [diff] [blame] | 383 | float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f; |
Michael K. Sanders | efa4c81 | 2018-10-30 14:44:48 +0000 | [diff] [blame] | 384 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 385 | /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol, Executor::ASYNC, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 386 | MeasureTiming::NO, /*testDynamicOutputShape=*/false); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 387 | } |
| 388 | |
Michael Butler | f76acd0 | 2018-03-22 16:37:57 -0700 | [diff] [blame] | 389 | void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model, |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 390 | std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) { |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 391 | V1_1::Model model = create_model(); |
| 392 | |
| 393 | // see if service can handle model |
| 394 | bool fullySupportsModel = false; |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 395 | Return<void> supportedCall = device->getSupportedOperations_1_1( |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 396 | model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) { |
| 397 | ASSERT_EQ(ErrorStatus::NONE, status); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 398 | ASSERT_NE(0ul, supported.size()); |
| 399 | fullySupportsModel = |
| 400 | std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); |
| 401 | }); |
| 402 | ASSERT_TRUE(supportedCall.isOk()); |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 403 | |
| 404 | // launch prepare model |
| 405 | sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); |
| 406 | ASSERT_NE(nullptr, preparedModelCallback.get()); |
Michael Butler | 2504c2f | 2018-04-11 16:30:09 -0700 | [diff] [blame] | 407 | Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1( |
| 408 | model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 409 | ASSERT_TRUE(prepareLaunchStatus.isOk()); |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 410 | ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus)); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 411 | |
| 412 | // retrieve prepared model |
| 413 | preparedModelCallback->wait(); |
| 414 | ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 415 | sp<V1_0::IPreparedModel> preparedModel; |
| 416 | getPreparedModel(preparedModelCallback, &preparedModel); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 417 | |
| 418 | // early termination if vendor service cannot fully prepare model |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 419 | if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 420 | ASSERT_EQ(nullptr, preparedModel.get()); |
| 421 | LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " |
| 422 | "prepare model that it does not support."; |
| 423 | std::cout << "[ ] Early termination of test because vendor service cannot " |
| 424 | "prepare model that it does not support." |
| 425 | << std::endl; |
| 426 | return; |
| 427 | } |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 428 | EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 429 | ASSERT_NE(nullptr, preparedModel.get()); |
| 430 | |
Michael K. Sanders | efa4c81 | 2018-10-30 14:44:48 +0000 | [diff] [blame] | 431 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 432 | model.relaxComputationFloat32toFloat16, 1e-5f, 1e-5f, Executor::ASYNC, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 433 | MeasureTiming::NO, /*testDynamicOutputShape=*/false); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 434 | } |
| 435 | |
Slava Shklyaev | 871be94 | 2018-09-12 14:52:02 +0100 | [diff] [blame] | 436 | // TODO: Reduce code duplication. |
| 437 | void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model, |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 438 | std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples, |
| 439 | bool testDynamicOutputShape) { |
Slava Shklyaev | 871be94 | 2018-09-12 14:52:02 +0100 | [diff] [blame] | 440 | V1_2::Model model = create_model(); |
| 441 | |
| 442 | // see if service can handle model |
| 443 | bool fullySupportsModel = false; |
| 444 | Return<void> supportedCall = device->getSupportedOperations_1_2( |
| 445 | model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) { |
| 446 | ASSERT_EQ(ErrorStatus::NONE, status); |
| 447 | ASSERT_NE(0ul, supported.size()); |
| 448 | fullySupportsModel = |
| 449 | std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); |
| 450 | }); |
| 451 | ASSERT_TRUE(supportedCall.isOk()); |
| 452 | |
| 453 | // launch prepare model |
| 454 | sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); |
| 455 | ASSERT_NE(nullptr, preparedModelCallback.get()); |
| 456 | Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_2( |
| 457 | model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback); |
| 458 | ASSERT_TRUE(prepareLaunchStatus.isOk()); |
| 459 | ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus)); |
| 460 | |
| 461 | // retrieve prepared model |
| 462 | preparedModelCallback->wait(); |
| 463 | ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 464 | sp<V1_2::IPreparedModel> preparedModel; |
| 465 | getPreparedModel(preparedModelCallback, &preparedModel); |
Slava Shklyaev | 871be94 | 2018-09-12 14:52:02 +0100 | [diff] [blame] | 466 | |
| 467 | // early termination if vendor service cannot fully prepare model |
| 468 | if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { |
| 469 | ASSERT_EQ(nullptr, preparedModel.get()); |
| 470 | LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " |
| 471 | "prepare model that it does not support."; |
| 472 | std::cout << "[ ] Early termination of test because vendor service cannot " |
| 473 | "prepare model that it does not support." |
| 474 | << std::endl; |
| 475 | return; |
| 476 | } |
| 477 | EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); |
| 478 | ASSERT_NE(nullptr, preparedModel.get()); |
| 479 | |
Michael K. Sanders | efa4c81 | 2018-10-30 14:44:48 +0000 | [diff] [blame] | 480 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 481 | model.relaxComputationFloat32toFloat16, Executor::ASYNC, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 482 | MeasureTiming::NO, testDynamicOutputShape); |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 483 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 484 | model.relaxComputationFloat32toFloat16, Executor::SYNC, MeasureTiming::NO, |
| 485 | testDynamicOutputShape); |
| 486 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
| 487 | model.relaxComputationFloat32toFloat16, Executor::BURST, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 488 | MeasureTiming::NO, testDynamicOutputShape); |
| 489 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 490 | model.relaxComputationFloat32toFloat16, Executor::ASYNC, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 491 | MeasureTiming::YES, testDynamicOutputShape); |
| 492 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame^] | 493 | model.relaxComputationFloat32toFloat16, Executor::SYNC, |
| 494 | MeasureTiming::YES, testDynamicOutputShape); |
| 495 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
| 496 | model.relaxComputationFloat32toFloat16, Executor::BURST, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 497 | MeasureTiming::YES, testDynamicOutputShape); |
Slava Shklyaev | 871be94 | 2018-09-12 14:52:02 +0100 | [diff] [blame] | 498 | } |
| 499 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 500 | } // namespace generated_tests |
| 501 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 502 | } // namespace neuralnetworks |
| 503 | } // namespace hardware |
| 504 | } // namespace android |