I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2017 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Xusong Wang | 3405878 | 2019-01-18 17:28:26 -0800 | [diff] [blame] | 17 | #include "GeneratedTestHarness.h" |
Michael Butler | cf22a57 | 2017-09-22 13:26:12 -0700 | [diff] [blame] | 18 | #include "Callbacks.h" |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame] | 19 | #include "ExecutionBurstController.h" |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 20 | #include "TestHarness.h" |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 21 | #include "Utils.h" |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 22 | |
| 23 | #include <android-base/logging.h> |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 24 | #include <android/hardware/neuralnetworks/1.0/IDevice.h> |
| 25 | #include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h> |
| 26 | #include <android/hardware/neuralnetworks/1.0/IPreparedModel.h> |
| 27 | #include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h> |
| 28 | #include <android/hardware/neuralnetworks/1.0/types.h> |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 29 | #include <android/hardware/neuralnetworks/1.1/IDevice.h> |
| 30 | #include <android/hardware/neuralnetworks/1.2/IDevice.h> |
| 31 | #include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h> |
| 32 | #include <android/hardware/neuralnetworks/1.2/IPreparedModel.h> |
| 33 | #include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h> |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 34 | #include <android/hidl/allocator/1.0/IAllocator.h> |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 35 | #include <android/hidl/memory/1.0/IMemory.h> |
| 36 | #include <hidlmemory/mapping.h> |
Michael Butler | 0897ab3 | 2017-10-04 02:38:42 -0700 | [diff] [blame] | 37 | #include <iostream> |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 38 | |
| 39 | namespace android { |
| 40 | namespace hardware { |
| 41 | namespace neuralnetworks { |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 42 | |
| 43 | namespace generated_tests { |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 44 | using ::android::hardware::neuralnetworks::V1_2::implementation::ExecutionCallback; |
| 45 | using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCallback; |
Slava Shklyaev | 9e3fad1 | 2018-11-30 17:55:12 +0000 | [diff] [blame] | 46 | using ::test_helper::bool8; |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 47 | using ::test_helper::compare; |
| 48 | using ::test_helper::expectMultinomialDistributionWithinTolerance; |
Mika Raento | de16694 | 2018-04-17 16:49:50 +0100 | [diff] [blame] | 49 | using ::test_helper::filter; |
| 50 | using ::test_helper::for_all; |
| 51 | using ::test_helper::for_each; |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 52 | using ::test_helper::MixedTyped; |
| 53 | using ::test_helper::MixedTypedExample; |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 54 | using ::test_helper::resize_accordingly; |
I-Jui (Ray) Sung | f6b8550 | 2017-09-20 13:45:50 -0700 | [diff] [blame] | 55 | |
I-Jui (Ray) Sung | 5bf4edf | 2017-10-06 13:22:39 -0700 | [diff] [blame] | 56 | template <typename T> |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 57 | void copy_back_(std::map<int, std::vector<T>>* dst, const std::vector<RequestArgument>& ra, |
| 58 | char* src) { |
| 59 | for_each<T>(*dst, [&ra, src](int index, std::vector<T>& m) { |
I-Jui (Ray) Sung | 5bf4edf | 2017-10-06 13:22:39 -0700 | [diff] [blame] | 60 | ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T)); |
I-Jui (Ray) Sung | f6b8550 | 2017-09-20 13:45:50 -0700 | [diff] [blame] | 61 | char* begin = src + ra[index].location.offset; |
| 62 | memcpy(m.data(), begin, ra[index].location.length); |
| 63 | }); |
| 64 | } |
| 65 | |
| 66 | void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) { |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 67 | copy_back_(&dst->float32Operands, ra, src); |
| 68 | copy_back_(&dst->int32Operands, ra, src); |
Xusong Wang | d49f665 | 2019-01-16 18:32:24 -0800 | [diff] [blame] | 69 | copy_back_(&dst->quant8AsymmOperands, ra, src); |
| 70 | copy_back_(&dst->quant16SymmOperands, ra, src); |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 71 | copy_back_(&dst->float16Operands, ra, src); |
| 72 | copy_back_(&dst->bool8Operands, ra, src); |
| 73 | copy_back_(&dst->quant8ChannelOperands, ra, src); |
Xusong Wang | d49f665 | 2019-01-16 18:32:24 -0800 | [diff] [blame] | 74 | copy_back_(&dst->quant16AsymmOperands, ra, src); |
Lev Proleev | bf26a9e | 2019-02-20 12:49:14 +0000 | [diff] [blame^] | 75 | copy_back_(&dst->quant8SymmOperands, ra, src); |
| 76 | static_assert(9 == MixedTyped::kNumTypes, |
Lev Proleev | 9b490f4 | 2018-11-02 12:44:11 +0000 | [diff] [blame] | 77 | "Number of types in MixedTyped changed, but copy_back function wasn't updated"); |
I-Jui (Ray) Sung | f6b8550 | 2017-09-20 13:45:50 -0700 | [diff] [blame] | 78 | } |
| 79 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 80 | // Top level driver for models and examples generated by test_generator.py |
| 81 | // Test driver for those generated from ml/nn/runtime/test/spec |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 82 | static Return<ErrorStatus> ExecutePreparedModel(sp<V1_0::IPreparedModel>& preparedModel, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 83 | const Request& request, MeasureTiming, |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 84 | sp<ExecutionCallback>& callback) { |
| 85 | return preparedModel->execute(request, callback); |
| 86 | } |
| 87 | static Return<ErrorStatus> ExecutePreparedModel(sp<V1_2::IPreparedModel>& preparedModel, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 88 | const Request& request, MeasureTiming measure, |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 89 | sp<ExecutionCallback>& callback) { |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 90 | return preparedModel->execute_1_2(request, measure, callback); |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 91 | } |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 92 | static Return<ErrorStatus> ExecutePreparedModel(sp<V1_0::IPreparedModel>&, const Request&, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 93 | MeasureTiming, hidl_vec<OutputShape>*, Timing*) { |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 94 | ADD_FAILURE() << "asking for synchronous execution at V1_0"; |
| 95 | return ErrorStatus::GENERAL_FAILURE; |
| 96 | } |
| 97 | static Return<ErrorStatus> ExecutePreparedModel(sp<V1_2::IPreparedModel>& preparedModel, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 98 | const Request& request, MeasureTiming measure, |
| 99 | hidl_vec<OutputShape>* outputShapes, |
| 100 | Timing* timing) { |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 101 | ErrorStatus result; |
| 102 | Return<void> ret = preparedModel->executeSynchronously( |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 103 | request, measure, |
| 104 | [&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes, |
| 105 | const Timing& time) { |
| 106 | result = error; |
| 107 | *outputShapes = shapes; |
| 108 | *timing = time; |
| 109 | }); |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 110 | if (!ret.isOk()) { |
| 111 | return ErrorStatus::GENERAL_FAILURE; |
| 112 | } |
| 113 | return result; |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 114 | } |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame] | 115 | static std::unique_ptr<::android::nn::ExecutionBurstController> CreateBurst( |
| 116 | const sp<V1_0::IPreparedModel>&) { |
| 117 | ADD_FAILURE() << "asking for burst execution at V1_0"; |
| 118 | return nullptr; |
| 119 | } |
| 120 | static std::unique_ptr<::android::nn::ExecutionBurstController> CreateBurst( |
| 121 | const sp<V1_2::IPreparedModel>& preparedModel) { |
| 122 | return ::android::nn::createExecutionBurstController(preparedModel, /*blocking=*/true); |
| 123 | } |
| 124 | enum class Executor { ASYNC, SYNC, BURST }; |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 125 | enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT }; |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 126 | const float kDefaultAtol = 1e-5f; |
| 127 | const float kDefaultRtol = 1e-5f; |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 128 | template <typename T_IPreparedModel> |
| 129 | void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored, |
Michael K. Sanders | efa4c81 | 2018-10-30 14:44:48 +0000 | [diff] [blame] | 130 | const std::vector<MixedTypedExample>& examples, |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 131 | bool hasRelaxedFloat32Model, float fpAtol, float fpRtol, |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 132 | Executor executor, MeasureTiming measure, OutputType outputType) { |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 133 | const uint32_t INPUT = 0; |
| 134 | const uint32_t OUTPUT = 1; |
| 135 | |
| 136 | int example_no = 1; |
| 137 | for (auto& example : examples) { |
| 138 | SCOPED_TRACE(example_no++); |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 139 | const MixedTyped& inputs = example.operands.first; |
| 140 | const MixedTyped& golden = example.operands.second; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 141 | |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 142 | const bool hasFloat16Inputs = !inputs.float16Operands.empty(); |
Michael K. Sanders | efa4c81 | 2018-10-30 14:44:48 +0000 | [diff] [blame] | 143 | if (hasRelaxedFloat32Model || hasFloat16Inputs) { |
| 144 | // TODO: Adjust the error limit based on testing. |
| 145 | // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16. |
| 146 | fpAtol = 5.0f * 0.0009765625f; |
| 147 | // Set the relative tolerance to be 5ULP of the corresponding FP precision. |
| 148 | fpRtol = 5.0f * 0.0009765625f; |
| 149 | } |
| 150 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 151 | std::vector<RequestArgument> inputs_info, outputs_info; |
| 152 | uint32_t inputSize = 0, outputSize = 0; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 153 | // This function only partially specifies the metadata (vector of RequestArguments). |
| 154 | // The contents are copied over below. |
| 155 | for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) { |
| 156 | if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1); |
| 157 | RequestArgument arg = { |
| 158 | .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)}, |
| 159 | .dimensions = {}, |
| 160 | }; |
I-Jui (Ray) Sung | 959cd78 | 2017-10-04 20:49:57 -0700 | [diff] [blame] | 161 | RequestArgument arg_empty = { |
| 162 | .hasNoValue = true, |
| 163 | }; |
| 164 | inputs_info[index] = s ? arg : arg_empty; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 165 | inputSize += s; |
| 166 | }); |
| 167 | // Compute offset for inputs 1 and so on |
| 168 | { |
| 169 | size_t offset = 0; |
| 170 | for (auto& i : inputs_info) { |
I-Jui (Ray) Sung | 959cd78 | 2017-10-04 20:49:57 -0700 | [diff] [blame] | 171 | if (!i.hasNoValue) i.location.offset = offset; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 172 | offset += i.location.length; |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | MixedTyped test; // holding test results |
| 177 | |
| 178 | // Go through all outputs, initialize RequestArgument descriptors |
I-Jui (Ray) Sung | f6b8550 | 2017-09-20 13:45:50 -0700 | [diff] [blame] | 179 | resize_accordingly(golden, test); |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 180 | bool sizeLargerThanOne = true; |
| 181 | for_all(golden, [&outputs_info, &outputSize, &outputType, &sizeLargerThanOne]( |
| 182 | int index, auto, auto s) { |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 183 | if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1); |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 184 | if (index == 0) { |
| 185 | // On OutputType::INSUFFICIENT, set the output operand with index 0 with |
| 186 | // buffer size one byte less than needed. |
| 187 | if (outputType == OutputType::INSUFFICIENT) { |
| 188 | if (s > 1) |
| 189 | s -= 1; |
| 190 | else |
| 191 | sizeLargerThanOne = false; |
| 192 | } |
| 193 | } |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 194 | RequestArgument arg = { |
| 195 | .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)}, |
| 196 | .dimensions = {}, |
| 197 | }; |
| 198 | outputs_info[index] = arg; |
| 199 | outputSize += s; |
| 200 | }); |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 201 | // If output0 does not have size larger than one byte, |
| 202 | // we can not provide an insufficient buffer |
| 203 | if (!sizeLargerThanOne && outputType == OutputType::INSUFFICIENT) return; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 204 | // Compute offset for outputs 1 and so on |
| 205 | { |
| 206 | size_t offset = 0; |
| 207 | for (auto& i : outputs_info) { |
| 208 | i.location.offset = offset; |
| 209 | offset += i.location.length; |
| 210 | } |
| 211 | } |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 212 | std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize), |
| 213 | nn::allocateSharedMemory(outputSize)}; |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 214 | ASSERT_NE(0ull, pools[INPUT].size()); |
| 215 | ASSERT_NE(0ull, pools[OUTPUT].size()); |
| 216 | |
| 217 | // load data |
| 218 | sp<IMemory> inputMemory = mapMemory(pools[INPUT]); |
| 219 | sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]); |
| 220 | ASSERT_NE(nullptr, inputMemory.get()); |
| 221 | ASSERT_NE(nullptr, outputMemory.get()); |
| 222 | char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer())); |
| 223 | char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer())); |
| 224 | ASSERT_NE(nullptr, inputPtr); |
| 225 | ASSERT_NE(nullptr, outputPtr); |
| 226 | inputMemory->update(); |
| 227 | outputMemory->update(); |
| 228 | |
| 229 | // Go through all inputs, copy the values |
| 230 | for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) { |
| 231 | char* begin = (char*)p; |
| 232 | char* end = begin + s; |
| 233 | // TODO: handle more than one input |
| 234 | std::copy(begin, end, inputPtr + inputs_info[index].location.offset); |
| 235 | }); |
| 236 | |
| 237 | inputMemory->commit(); |
| 238 | outputMemory->commit(); |
Michael Butler | cf22a57 | 2017-09-22 13:26:12 -0700 | [diff] [blame] | 239 | |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame] | 240 | const Request request = {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}; |
| 241 | |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 242 | ErrorStatus executionStatus; |
| 243 | hidl_vec<OutputShape> outputShapes; |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 244 | Timing timing; |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame] | 245 | switch (executor) { |
| 246 | case Executor::ASYNC: { |
| 247 | SCOPED_TRACE("asynchronous"); |
Michael Butler | cf22a57 | 2017-09-22 13:26:12 -0700 | [diff] [blame] | 248 | |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame] | 249 | // launch execution |
| 250 | sp<ExecutionCallback> executionCallback = new ExecutionCallback(); |
| 251 | ASSERT_NE(nullptr, executionCallback.get()); |
| 252 | Return<ErrorStatus> executionLaunchStatus = |
| 253 | ExecutePreparedModel(preparedModel, request, measure, executionCallback); |
| 254 | ASSERT_TRUE(executionLaunchStatus.isOk()); |
| 255 | EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus)); |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 256 | |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame] | 257 | // retrieve execution status |
| 258 | executionCallback->wait(); |
| 259 | executionStatus = executionCallback->getStatus(); |
| 260 | outputShapes = executionCallback->getOutputShapes(); |
| 261 | timing = executionCallback->getTiming(); |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 262 | |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame] | 263 | break; |
| 264 | } |
| 265 | case Executor::SYNC: { |
| 266 | SCOPED_TRACE("synchronous"); |
| 267 | |
| 268 | // execute |
| 269 | Return<ErrorStatus> executionReturnStatus = ExecutePreparedModel( |
| 270 | preparedModel, request, measure, &outputShapes, &timing); |
| 271 | ASSERT_TRUE(executionReturnStatus.isOk()); |
| 272 | executionStatus = static_cast<ErrorStatus>(executionReturnStatus); |
| 273 | |
| 274 | break; |
| 275 | } |
| 276 | case Executor::BURST: { |
| 277 | SCOPED_TRACE("burst"); |
| 278 | |
| 279 | // create burst |
| 280 | const std::unique_ptr<::android::nn::ExecutionBurstController> controller = |
| 281 | CreateBurst(preparedModel); |
| 282 | ASSERT_NE(nullptr, controller.get()); |
| 283 | |
| 284 | // create memory keys |
| 285 | std::vector<intptr_t> keys(request.pools.size()); |
| 286 | for (size_t i = 0; i < keys.size(); ++i) { |
| 287 | keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]); |
| 288 | } |
| 289 | |
| 290 | // execute burst |
| 291 | std::tie(executionStatus, outputShapes, timing) = |
| 292 | controller->compute(request, measure, keys); |
| 293 | |
| 294 | break; |
| 295 | } |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 296 | } |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 297 | |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 298 | if (outputType != OutputType::FULLY_SPECIFIED && |
| 299 | executionStatus == ErrorStatus::GENERAL_FAILURE) { |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 300 | LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " |
| 301 | "execute model that it does not support."; |
| 302 | std::cout << "[ ] Early termination of test because vendor service cannot " |
| 303 | "execute model that it does not support." |
| 304 | << std::endl; |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 305 | GTEST_SKIP(); |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 306 | } |
David Gross | e301349 | 2019-01-23 14:01:52 -0800 | [diff] [blame] | 307 | if (measure == MeasureTiming::NO) { |
| 308 | EXPECT_EQ(UINT64_MAX, timing.timeOnDevice); |
| 309 | EXPECT_EQ(UINT64_MAX, timing.timeInDriver); |
| 310 | } else { |
| 311 | if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) { |
| 312 | EXPECT_LE(timing.timeOnDevice, timing.timeInDriver); |
| 313 | } |
| 314 | } |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 315 | |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 316 | switch (outputType) { |
| 317 | case OutputType::FULLY_SPECIFIED: |
| 318 | // If the model output operands are fully specified, outputShapes must be either |
| 319 | // either empty, or have the same number of elements as the number of outputs. |
| 320 | ASSERT_EQ(ErrorStatus::NONE, executionStatus); |
| 321 | ASSERT_TRUE(outputShapes.size() == 0 || |
| 322 | outputShapes.size() == test.operandDimensions.size()); |
| 323 | break; |
| 324 | case OutputType::UNSPECIFIED: |
| 325 | // If the model output operands are not fully specified, outputShapes must have |
| 326 | // the same number of elements as the number of outputs. |
| 327 | ASSERT_EQ(ErrorStatus::NONE, executionStatus); |
| 328 | ASSERT_EQ(outputShapes.size(), test.operandDimensions.size()); |
| 329 | break; |
| 330 | case OutputType::INSUFFICIENT: |
| 331 | ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus); |
| 332 | ASSERT_EQ(outputShapes.size(), test.operandDimensions.size()); |
| 333 | ASSERT_FALSE(outputShapes[0].isSufficient); |
| 334 | return; |
| 335 | } |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 336 | // Go through all outputs, overwrite output dimensions with returned output shapes |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 337 | if (outputShapes.size() > 0) { |
Xusong Wang | a316581 | 2018-11-19 18:26:08 -0800 | [diff] [blame] | 338 | for_each<uint32_t>(test.operandDimensions, |
| 339 | [&outputShapes](int idx, std::vector<uint32_t>& dim) { |
| 340 | dim = outputShapes[idx].dimensions; |
| 341 | }); |
| 342 | } |
Xusong Wang | 187c597 | 2018-11-07 09:33:59 -0800 | [diff] [blame] | 343 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 344 | // validate results |
| 345 | outputMemory->read(); |
I-Jui (Ray) Sung | f6b8550 | 2017-09-20 13:45:50 -0700 | [diff] [blame] | 346 | copy_back(&test, outputs_info, outputPtr); |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 347 | outputMemory->commit(); |
I-Jui (Ray) Sung | 7d765bd | 2017-09-13 18:47:12 -0700 | [diff] [blame] | 348 | // Filter out don't cares |
I-Jui (Ray) Sung | 5bf4edf | 2017-10-06 13:22:39 -0700 | [diff] [blame] | 349 | MixedTyped filtered_golden = filter(golden, is_ignored); |
| 350 | MixedTyped filtered_test = filter(test, is_ignored); |
I-Jui (Ray) Sung | 7d765bd | 2017-09-13 18:47:12 -0700 | [diff] [blame] | 351 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 352 | // We want "close-enough" results for float |
Xusong Wang | 10d77e4 | 2018-08-28 16:50:01 -0700 | [diff] [blame] | 353 | compare(filtered_golden, filtered_test, fpAtol, fpRtol); |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 354 | |
| 355 | if (example.expectedMultinomialDistributionTolerance > 0) { |
| 356 | expectMultinomialDistributionWithinTolerance(test, example); |
| 357 | } |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 358 | } |
| 359 | } |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 360 | template <typename T_IPreparedModel> |
| 361 | void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored, |
| 362 | const std::vector<MixedTypedExample>& examples, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame] | 363 | bool hasRelaxedFloat32Model, Executor executor, MeasureTiming measure, |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 364 | OutputType outputType) { |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 365 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, kDefaultAtol, |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 366 | kDefaultRtol, executor, measure, outputType); |
David Gross | 49e4167 | 2018-12-21 11:20:26 -0800 | [diff] [blame] | 367 | } |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 368 | |
Xusong Wang | 3405878 | 2019-01-18 17:28:26 -0800 | [diff] [blame] | 369 | void EvaluatePreparedModel(sp<V1_2::IPreparedModel>& preparedModel, |
| 370 | std::function<bool(int)> is_ignored, |
| 371 | const std::vector<MixedTypedExample>& examples, |
| 372 | bool hasRelaxedFloat32Model, bool testDynamicOutputShape) { |
| 373 | if (testDynamicOutputShape) { |
| 374 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 375 | Executor::ASYNC, MeasureTiming::NO, OutputType::UNSPECIFIED); |
| 376 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 377 | Executor::SYNC, MeasureTiming::NO, OutputType::UNSPECIFIED); |
| 378 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 379 | Executor::BURST, MeasureTiming::NO, OutputType::UNSPECIFIED); |
| 380 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 381 | Executor::ASYNC, MeasureTiming::YES, OutputType::UNSPECIFIED); |
| 382 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 383 | Executor::SYNC, MeasureTiming::YES, OutputType::UNSPECIFIED); |
| 384 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 385 | Executor::BURST, MeasureTiming::YES, OutputType::UNSPECIFIED); |
| 386 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 387 | Executor::ASYNC, MeasureTiming::NO, OutputType::INSUFFICIENT); |
| 388 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 389 | Executor::SYNC, MeasureTiming::NO, OutputType::INSUFFICIENT); |
| 390 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 391 | Executor::BURST, MeasureTiming::NO, OutputType::INSUFFICIENT); |
| 392 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 393 | Executor::ASYNC, MeasureTiming::YES, OutputType::INSUFFICIENT); |
| 394 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 395 | Executor::SYNC, MeasureTiming::YES, OutputType::INSUFFICIENT); |
| 396 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 397 | Executor::BURST, MeasureTiming::YES, OutputType::INSUFFICIENT); |
| 398 | } else { |
| 399 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 400 | Executor::ASYNC, MeasureTiming::NO, OutputType::FULLY_SPECIFIED); |
| 401 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 402 | Executor::SYNC, MeasureTiming::NO, OutputType::FULLY_SPECIFIED); |
| 403 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 404 | Executor::BURST, MeasureTiming::NO, OutputType::FULLY_SPECIFIED); |
| 405 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 406 | Executor::ASYNC, MeasureTiming::YES, OutputType::FULLY_SPECIFIED); |
| 407 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 408 | Executor::SYNC, MeasureTiming::YES, OutputType::FULLY_SPECIFIED); |
| 409 | EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, |
| 410 | Executor::BURST, MeasureTiming::YES, OutputType::FULLY_SPECIFIED); |
| 411 | } |
| 412 | } |
| 413 | |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 414 | static void getPreparedModel(sp<PreparedModelCallback> callback, |
| 415 | sp<V1_0::IPreparedModel>* preparedModel) { |
| 416 | *preparedModel = callback->getPreparedModel(); |
| 417 | } |
| 418 | static void getPreparedModel(sp<PreparedModelCallback> callback, |
| 419 | sp<V1_2::IPreparedModel>* preparedModel) { |
| 420 | sp<V1_0::IPreparedModel> preparedModelV1_0 = callback->getPreparedModel(); |
| 421 | *preparedModel = V1_2::IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr); |
| 422 | } |
| 423 | |
Michael Butler | f76acd0 | 2018-03-22 16:37:57 -0700 | [diff] [blame] | 424 | void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model, |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 425 | std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) { |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 426 | V1_0::Model model = create_model(); |
| 427 | |
| 428 | // see if service can handle model |
| 429 | bool fullySupportsModel = false; |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 430 | Return<void> supportedCall = device->getSupportedOperations( |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 431 | model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) { |
| 432 | ASSERT_EQ(ErrorStatus::NONE, status); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 433 | ASSERT_NE(0ul, supported.size()); |
| 434 | fullySupportsModel = |
| 435 | std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); |
| 436 | }); |
| 437 | ASSERT_TRUE(supportedCall.isOk()); |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 438 | |
| 439 | // launch prepare model |
| 440 | sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); |
| 441 | ASSERT_NE(nullptr, preparedModelCallback.get()); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 442 | Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); |
| 443 | ASSERT_TRUE(prepareLaunchStatus.isOk()); |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 444 | ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus)); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 445 | |
| 446 | // retrieve prepared model |
| 447 | preparedModelCallback->wait(); |
| 448 | ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 449 | sp<V1_0::IPreparedModel> preparedModel; |
| 450 | getPreparedModel(preparedModelCallback, &preparedModel); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 451 | |
| 452 | // early termination if vendor service cannot fully prepare model |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 453 | if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 454 | ASSERT_EQ(nullptr, preparedModel.get()); |
| 455 | LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " |
| 456 | "prepare model that it does not support."; |
| 457 | std::cout << "[ ] Early termination of test because vendor service cannot " |
| 458 | "prepare model that it does not support." |
| 459 | << std::endl; |
Miao Wang | bb685a4 | 2019-01-08 12:27:35 -0800 | [diff] [blame] | 460 | GTEST_SKIP(); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 461 | } |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 462 | EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 463 | ASSERT_NE(nullptr, preparedModel.get()); |
| 464 | |
Xusong Wang | 10d77e4 | 2018-08-28 16:50:01 -0700 | [diff] [blame] | 465 | float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f; |
Michael K. Sanders | efa4c81 | 2018-10-30 14:44:48 +0000 | [diff] [blame] | 466 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame] | 467 | /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol, Executor::ASYNC, |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 468 | MeasureTiming::NO, OutputType::FULLY_SPECIFIED); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 469 | } |
| 470 | |
Michael Butler | f76acd0 | 2018-03-22 16:37:57 -0700 | [diff] [blame] | 471 | void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model, |
Michael K. Sanders | 941d61a | 2018-10-19 14:39:09 +0100 | [diff] [blame] | 472 | std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) { |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 473 | V1_1::Model model = create_model(); |
| 474 | |
| 475 | // see if service can handle model |
| 476 | bool fullySupportsModel = false; |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 477 | Return<void> supportedCall = device->getSupportedOperations_1_1( |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 478 | model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) { |
| 479 | ASSERT_EQ(ErrorStatus::NONE, status); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 480 | ASSERT_NE(0ul, supported.size()); |
| 481 | fullySupportsModel = |
| 482 | std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); |
| 483 | }); |
| 484 | ASSERT_TRUE(supportedCall.isOk()); |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 485 | |
| 486 | // launch prepare model |
| 487 | sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); |
| 488 | ASSERT_NE(nullptr, preparedModelCallback.get()); |
Michael Butler | 2504c2f | 2018-04-11 16:30:09 -0700 | [diff] [blame] | 489 | Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1( |
| 490 | model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 491 | ASSERT_TRUE(prepareLaunchStatus.isOk()); |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 492 | ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus)); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 493 | |
| 494 | // retrieve prepared model |
| 495 | preparedModelCallback->wait(); |
| 496 | ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); |
Xusong Wang | b5cb8f7 | 2018-10-31 08:43:12 -0700 | [diff] [blame] | 497 | sp<V1_0::IPreparedModel> preparedModel; |
| 498 | getPreparedModel(preparedModelCallback, &preparedModel); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 499 | |
| 500 | // early termination if vendor service cannot fully prepare model |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 501 | if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 502 | ASSERT_EQ(nullptr, preparedModel.get()); |
| 503 | LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " |
| 504 | "prepare model that it does not support."; |
| 505 | std::cout << "[ ] Early termination of test because vendor service cannot " |
| 506 | "prepare model that it does not support." |
| 507 | << std::endl; |
Miao Wang | bb685a4 | 2019-01-08 12:27:35 -0800 | [diff] [blame] | 508 | GTEST_SKIP(); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 509 | } |
Michael Butler | 4d5bb10 | 2018-02-26 15:24:46 -0800 | [diff] [blame] | 510 | EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 511 | ASSERT_NE(nullptr, preparedModel.get()); |
| 512 | |
Michael K. Sanders | efa4c81 | 2018-10-30 14:44:48 +0000 | [diff] [blame] | 513 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
Michael Butler | 814d837 | 2019-01-15 11:02:55 -0800 | [diff] [blame] | 514 | model.relaxComputationFloat32toFloat16, 1e-5f, 1e-5f, Executor::ASYNC, |
Xusong Wang | 929fd21 | 2019-01-27 23:08:12 -0800 | [diff] [blame] | 515 | MeasureTiming::NO, OutputType::FULLY_SPECIFIED); |
Miao Wang | a2d04c8 | 2018-02-05 17:26:54 -0800 | [diff] [blame] | 516 | } |
| 517 | |
Xusong Wang | 3405878 | 2019-01-18 17:28:26 -0800 | [diff] [blame] | 518 | void PrepareModel(const sp<V1_2::IDevice>& device, const V1_2::Model& model, |
| 519 | sp<V1_2::IPreparedModel>* preparedModel) { |
Slava Shklyaev | 871be94 | 2018-09-12 14:52:02 +0100 | [diff] [blame] | 520 | // see if service can handle model |
| 521 | bool fullySupportsModel = false; |
| 522 | Return<void> supportedCall = device->getSupportedOperations_1_2( |
| 523 | model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) { |
| 524 | ASSERT_EQ(ErrorStatus::NONE, status); |
| 525 | ASSERT_NE(0ul, supported.size()); |
| 526 | fullySupportsModel = |
| 527 | std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); |
| 528 | }); |
| 529 | ASSERT_TRUE(supportedCall.isOk()); |
| 530 | |
| 531 | // launch prepare model |
| 532 | sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); |
| 533 | ASSERT_NE(nullptr, preparedModelCallback.get()); |
| 534 | Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_2( |
| 535 | model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback); |
| 536 | ASSERT_TRUE(prepareLaunchStatus.isOk()); |
| 537 | ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus)); |
| 538 | |
| 539 | // retrieve prepared model |
| 540 | preparedModelCallback->wait(); |
| 541 | ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); |
Xusong Wang | 3405878 | 2019-01-18 17:28:26 -0800 | [diff] [blame] | 542 | getPreparedModel(preparedModelCallback, preparedModel); |
Slava Shklyaev | 871be94 | 2018-09-12 14:52:02 +0100 | [diff] [blame] | 543 | |
| 544 | // early termination if vendor service cannot fully prepare model |
| 545 | if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { |
Xusong Wang | 3405878 | 2019-01-18 17:28:26 -0800 | [diff] [blame] | 546 | ASSERT_EQ(nullptr, preparedModel->get()); |
Slava Shklyaev | 871be94 | 2018-09-12 14:52:02 +0100 | [diff] [blame] | 547 | LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " |
| 548 | "prepare model that it does not support."; |
| 549 | std::cout << "[ ] Early termination of test because vendor service cannot " |
| 550 | "prepare model that it does not support." |
| 551 | << std::endl; |
Miao Wang | 4135a8e | 2019-02-01 14:00:08 -0800 | [diff] [blame] | 552 | return; |
Slava Shklyaev | 871be94 | 2018-09-12 14:52:02 +0100 | [diff] [blame] | 553 | } |
| 554 | EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); |
Xusong Wang | 3405878 | 2019-01-18 17:28:26 -0800 | [diff] [blame] | 555 | ASSERT_NE(nullptr, preparedModel->get()); |
| 556 | } |
Slava Shklyaev | 871be94 | 2018-09-12 14:52:02 +0100 | [diff] [blame] | 557 | |
Xusong Wang | 3405878 | 2019-01-18 17:28:26 -0800 | [diff] [blame] | 558 | // TODO: Reduce code duplication. |
| 559 | void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model, |
| 560 | std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples, |
| 561 | bool testDynamicOutputShape) { |
| 562 | V1_2::Model model = create_model(); |
| 563 | sp<V1_2::IPreparedModel> preparedModel = nullptr; |
| 564 | PrepareModel(device, model, &preparedModel); |
Miao Wang | 4135a8e | 2019-02-01 14:00:08 -0800 | [diff] [blame] | 565 | if (preparedModel == nullptr) { |
| 566 | GTEST_SKIP(); |
| 567 | } |
Xusong Wang | 3405878 | 2019-01-18 17:28:26 -0800 | [diff] [blame] | 568 | EvaluatePreparedModel(preparedModel, is_ignored, examples, |
| 569 | model.relaxComputationFloat32toFloat16, testDynamicOutputShape); |
Slava Shklyaev | 871be94 | 2018-09-12 14:52:02 +0100 | [diff] [blame] | 570 | } |
| 571 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 572 | } // namespace generated_tests |
| 573 | |
I-Jui (Ray) Sung | 2c4e136 | 2017-09-06 02:15:54 -0700 | [diff] [blame] | 574 | } // namespace neuralnetworks |
| 575 | } // namespace hardware |
| 576 | } // namespace android |