blob: c78439c32d8b5c4da622b74a92114d541e191ff2 [file] [log] [blame]
Lev Proleev13fdfcd2019-08-30 11:35:34 +01001/*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "neuralnetworks_hidl_hal_test"
18
19#include "VtsHalNeuralnetworks.h"
20
21#include "1.2/Callbacks.h"
22#include "ExecutionBurstController.h"
23#include "ExecutionBurstServer.h"
24#include "GeneratedTestHarness.h"
25#include "TestHarness.h"
Lev Proleev13fdfcd2019-08-30 11:35:34 +010026
27#include <android-base/logging.h>
Michael Butler648ada52019-07-25 17:22:11 -070028#include <chrono>
Lev Proleev13fdfcd2019-08-30 11:35:34 +010029#include <cstring>
30
Lev Proleev26d1bc82019-08-30 11:57:18 +010031namespace android::hardware::neuralnetworks::V1_3::vts::functional {
Lev Proleev13fdfcd2019-08-30 11:35:34 +010032
33using nn::ExecutionBurstController;
34using nn::RequestChannelSender;
35using nn::ResultChannelReceiver;
Lev Proleev13fdfcd2019-08-30 11:35:34 +010036using V1_0::Request;
Lev Proleev26d1bc82019-08-30 11:57:18 +010037using V1_2::FmqRequestDatum;
38using V1_2::FmqResultDatum;
39using V1_2::IBurstCallback;
40using V1_2::IBurstContext;
Lev Proleev26d1bc82019-08-30 11:57:18 +010041using V1_2::MeasureTiming;
42using V1_2::Timing;
Lev Proleev13fdfcd2019-08-30 11:35:34 +010043using ExecutionBurstCallback = ExecutionBurstController::ExecutionBurstCallback;
44
Michael Butler68a6de72020-03-11 18:45:45 -070045using BurstExecutionMutation = std::function<void(std::vector<FmqRequestDatum>*)>;
46
Lev Proleev13fdfcd2019-08-30 11:35:34 +010047// This constant value represents the length of an FMQ that is large enough to
48// return a result from a burst execution for all of the generated test cases.
49constexpr size_t kExecutionBurstChannelLength = 1024;
50
51// This constant value represents a length of an FMQ that is not large enough
52// to return a result from a burst execution for some of the generated test
53// cases.
54constexpr size_t kExecutionBurstChannelSmallLength = 8;
55
56///////////////////////// UTILITY FUNCTIONS /////////////////////////
57
58static bool badTiming(Timing timing) {
59 return timing.timeOnDevice == UINT64_MAX && timing.timeInDriver == UINT64_MAX;
60}
61
62static void createBurst(const sp<IPreparedModel>& preparedModel, const sp<IBurstCallback>& callback,
63 std::unique_ptr<RequestChannelSender>* sender,
64 std::unique_ptr<ResultChannelReceiver>* receiver,
65 sp<IBurstContext>* context,
66 size_t resultChannelLength = kExecutionBurstChannelLength) {
67 ASSERT_NE(nullptr, preparedModel.get());
68 ASSERT_NE(nullptr, sender);
69 ASSERT_NE(nullptr, receiver);
70 ASSERT_NE(nullptr, context);
71
72 // create FMQ objects
73 auto [fmqRequestChannel, fmqRequestDescriptor] =
Michael Butler648ada52019-07-25 17:22:11 -070074 RequestChannelSender::create(kExecutionBurstChannelLength);
Lev Proleev13fdfcd2019-08-30 11:35:34 +010075 auto [fmqResultChannel, fmqResultDescriptor] =
Michael Butler648ada52019-07-25 17:22:11 -070076 ResultChannelReceiver::create(resultChannelLength, std::chrono::microseconds{0});
Lev Proleev13fdfcd2019-08-30 11:35:34 +010077 ASSERT_NE(nullptr, fmqRequestChannel.get());
78 ASSERT_NE(nullptr, fmqResultChannel.get());
79 ASSERT_NE(nullptr, fmqRequestDescriptor);
80 ASSERT_NE(nullptr, fmqResultDescriptor);
81
82 // configure burst
Michael Butler79a41d72019-12-11 19:08:08 -080083 V1_0::ErrorStatus errorStatus;
Lev Proleev13fdfcd2019-08-30 11:35:34 +010084 sp<IBurstContext> burstContext;
85 const Return<void> ret = preparedModel->configureExecutionBurst(
86 callback, *fmqRequestDescriptor, *fmqResultDescriptor,
Michael Butler79a41d72019-12-11 19:08:08 -080087 [&errorStatus, &burstContext](V1_0::ErrorStatus status,
88 const sp<IBurstContext>& context) {
Lev Proleev13fdfcd2019-08-30 11:35:34 +010089 errorStatus = status;
90 burstContext = context;
91 });
92 ASSERT_TRUE(ret.isOk());
Michael Butler79a41d72019-12-11 19:08:08 -080093 ASSERT_EQ(V1_0::ErrorStatus::NONE, errorStatus);
Lev Proleev13fdfcd2019-08-30 11:35:34 +010094 ASSERT_NE(nullptr, burstContext.get());
95
96 // return values
97 *sender = std::move(fmqRequestChannel);
98 *receiver = std::move(fmqResultChannel);
99 *context = burstContext;
100}
101
102static void createBurstWithResultChannelLength(
103 const sp<IPreparedModel>& preparedModel, size_t resultChannelLength,
104 std::shared_ptr<ExecutionBurstController>* controller) {
105 ASSERT_NE(nullptr, preparedModel.get());
106 ASSERT_NE(nullptr, controller);
107
108 // create FMQ objects
109 std::unique_ptr<RequestChannelSender> sender;
110 std::unique_ptr<ResultChannelReceiver> receiver;
111 sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
112 sp<IBurstContext> context;
113 ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context,
114 resultChannelLength));
115 ASSERT_NE(nullptr, sender.get());
116 ASSERT_NE(nullptr, receiver.get());
117 ASSERT_NE(nullptr, context.get());
118
119 // return values
120 *controller = std::make_shared<ExecutionBurstController>(std::move(sender), std::move(receiver),
121 context, callback);
122}
123
124// Primary validation function. This function will take a valid serialized
125// request, apply a mutation to it to invalidate the serialized request, then
Michael Butler68a6de72020-03-11 18:45:45 -0700126// pass it to interface calls that use the serialized request.
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100127static void validate(RequestChannelSender* sender, ResultChannelReceiver* receiver,
Michael Butler68a6de72020-03-11 18:45:45 -0700128 const std::string& message,
129 const std::vector<FmqRequestDatum>& originalSerialized,
130 const BurstExecutionMutation& mutate) {
131 std::vector<FmqRequestDatum> serialized = originalSerialized;
132 mutate(&serialized);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100133
134 // skip if packet is too large to send
135 if (serialized.size() > kExecutionBurstChannelLength) {
136 return;
137 }
138
139 SCOPED_TRACE(message);
140
141 // send invalid packet
142 ASSERT_TRUE(sender->sendPacket(serialized));
143
144 // receive error
145 auto results = receiver->getBlocking();
146 ASSERT_TRUE(results.has_value());
147 const auto [status, outputShapes, timing] = std::move(*results);
Michael Butler79a41d72019-12-11 19:08:08 -0800148 EXPECT_NE(V1_0::ErrorStatus::NONE, status);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100149 EXPECT_EQ(0u, outputShapes.size());
150 EXPECT_TRUE(badTiming(timing));
151}
152
153// For validation, valid packet entries are mutated to invalid packet entries,
154// or invalid packet entries are inserted into valid packets. This function
155// creates pre-set invalid packet entries for convenience.
156static std::vector<FmqRequestDatum> createBadRequestPacketEntries() {
157 const FmqRequestDatum::PacketInformation packetInformation = {
158 /*.packetSize=*/10, /*.numberOfInputOperands=*/10, /*.numberOfOutputOperands=*/10,
159 /*.numberOfPools=*/10};
160 const FmqRequestDatum::OperandInformation operandInformation = {
161 /*.hasNoValue=*/false, /*.location=*/{}, /*.numberOfDimensions=*/10};
162 const int32_t invalidPoolIdentifier = std::numeric_limits<int32_t>::max();
163 std::vector<FmqRequestDatum> bad(7);
164 bad[0].packetInformation(packetInformation);
165 bad[1].inputOperandInformation(operandInformation);
166 bad[2].inputOperandDimensionValue(0);
167 bad[3].outputOperandInformation(operandInformation);
168 bad[4].outputOperandDimensionValue(0);
169 bad[5].poolIdentifier(invalidPoolIdentifier);
170 bad[6].measureTiming(MeasureTiming::YES);
171 return bad;
172}
173
174// For validation, valid packet entries are mutated to invalid packet entries,
175// or invalid packet entries are inserted into valid packets. This function
176// retrieves pre-set invalid packet entries for convenience. This function
177// caches these data so they can be reused on subsequent validation checks.
178static const std::vector<FmqRequestDatum>& getBadRequestPacketEntries() {
179 static const std::vector<FmqRequestDatum> bad = createBadRequestPacketEntries();
180 return bad;
181}
182
183///////////////////////// REMOVE DATUM ////////////////////////////////////
184
185static void removeDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
186 const std::vector<FmqRequestDatum>& serialized) {
187 for (size_t index = 0; index < serialized.size(); ++index) {
188 const std::string message = "removeDatum: removed datum at index " + std::to_string(index);
189 validate(sender, receiver, message, serialized,
190 [index](std::vector<FmqRequestDatum>* serialized) {
191 serialized->erase(serialized->begin() + index);
192 });
193 }
194}
195
196///////////////////////// ADD DATUM ////////////////////////////////////
197
198static void addDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
199 const std::vector<FmqRequestDatum>& serialized) {
200 const std::vector<FmqRequestDatum>& extra = getBadRequestPacketEntries();
201 for (size_t index = 0; index <= serialized.size(); ++index) {
202 for (size_t type = 0; type < extra.size(); ++type) {
203 const std::string message = "addDatum: added datum type " + std::to_string(type) +
204 " at index " + std::to_string(index);
205 validate(sender, receiver, message, serialized,
206 [index, type, &extra](std::vector<FmqRequestDatum>* serialized) {
207 serialized->insert(serialized->begin() + index, extra[type]);
208 });
209 }
210 }
211}
212
213///////////////////////// MUTATE DATUM ////////////////////////////////////
214
215static bool interestingCase(const FmqRequestDatum& lhs, const FmqRequestDatum& rhs) {
216 using Discriminator = FmqRequestDatum::hidl_discriminator;
217
218 const bool differentValues = (lhs != rhs);
219 const bool sameDiscriminator = (lhs.getDiscriminator() == rhs.getDiscriminator());
220 const auto discriminator = rhs.getDiscriminator();
221 const bool isDimensionValue = (discriminator == Discriminator::inputOperandDimensionValue ||
222 discriminator == Discriminator::outputOperandDimensionValue);
223
224 return differentValues && !(sameDiscriminator && isDimensionValue);
225}
226
227static void mutateDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
228 const std::vector<FmqRequestDatum>& serialized) {
229 const std::vector<FmqRequestDatum>& change = getBadRequestPacketEntries();
230 for (size_t index = 0; index < serialized.size(); ++index) {
231 for (size_t type = 0; type < change.size(); ++type) {
232 if (interestingCase(serialized[index], change[type])) {
233 const std::string message = "mutateDatum: changed datum at index " +
234 std::to_string(index) + " to datum type " +
235 std::to_string(type);
236 validate(sender, receiver, message, serialized,
237 [index, type, &change](std::vector<FmqRequestDatum>* serialized) {
238 (*serialized)[index] = change[type];
239 });
240 }
241 }
242 }
243}
244
245///////////////////////// BURST VALIATION TESTS ////////////////////////////////////
246
247static void validateBurstSerialization(const sp<IPreparedModel>& preparedModel,
248 const Request& request) {
249 // create burst
250 std::unique_ptr<RequestChannelSender> sender;
251 std::unique_ptr<ResultChannelReceiver> receiver;
252 sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
253 sp<IBurstContext> context;
254 ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context));
255 ASSERT_NE(nullptr, sender.get());
256 ASSERT_NE(nullptr, receiver.get());
257 ASSERT_NE(nullptr, context.get());
258
259 // load memory into callback slots
260 std::vector<intptr_t> keys;
261 keys.reserve(request.pools.size());
262 std::transform(request.pools.begin(), request.pools.end(), std::back_inserter(keys),
263 [](const auto& pool) { return reinterpret_cast<intptr_t>(&pool); });
264 const std::vector<int32_t> slots = callback->getSlots(request.pools, keys);
265
266 // ensure slot std::numeric_limits<int32_t>::max() doesn't exist (for
267 // subsequent slot validation testing)
268 ASSERT_TRUE(std::all_of(slots.begin(), slots.end(), [](int32_t slot) {
269 return slot != std::numeric_limits<int32_t>::max();
270 }));
271
272 // serialize the request
273 const auto serialized = android::nn::serialize(request, MeasureTiming::YES, slots);
274
275 // validations
276 removeDatumTest(sender.get(), receiver.get(), serialized);
277 addDatumTest(sender.get(), receiver.get(), serialized);
278 mutateDatumTest(sender.get(), receiver.get(), serialized);
279}
280
281// This test validates that when the Result message size exceeds length of the
282// result FMQ, the service instance gracefully fails and returns an error.
283static void validateBurstFmqLength(const sp<IPreparedModel>& preparedModel,
284 const Request& request) {
285 // create regular burst
286 std::shared_ptr<ExecutionBurstController> controllerRegular;
287 ASSERT_NO_FATAL_FAILURE(createBurstWithResultChannelLength(
288 preparedModel, kExecutionBurstChannelLength, &controllerRegular));
289 ASSERT_NE(nullptr, controllerRegular.get());
290
291 // create burst with small output channel
292 std::shared_ptr<ExecutionBurstController> controllerSmall;
293 ASSERT_NO_FATAL_FAILURE(createBurstWithResultChannelLength(
294 preparedModel, kExecutionBurstChannelSmallLength, &controllerSmall));
295 ASSERT_NE(nullptr, controllerSmall.get());
296
297 // load memory into callback slots
298 std::vector<intptr_t> keys(request.pools.size());
299 for (size_t i = 0; i < keys.size(); ++i) {
300 keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
301 }
302
303 // collect serialized result by running regular burst
Michael Butler648ada52019-07-25 17:22:11 -0700304 const auto [nRegular, outputShapesRegular, timingRegular, fallbackRegular] =
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100305 controllerRegular->compute(request, MeasureTiming::NO, keys);
Michael Butlerd09c0ee2020-03-12 15:12:23 -0700306 const V1_0::ErrorStatus statusRegular = nn::legacyConvertResultCodeToErrorStatus(nRegular);
Michael Butler648ada52019-07-25 17:22:11 -0700307 EXPECT_FALSE(fallbackRegular);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100308
309 // skip test if regular burst output isn't useful for testing a failure
310 // caused by having too small of a length for the result FMQ
311 const std::vector<FmqResultDatum> serialized =
312 android::nn::serialize(statusRegular, outputShapesRegular, timingRegular);
Michael Butler79a41d72019-12-11 19:08:08 -0800313 if (statusRegular != V1_0::ErrorStatus::NONE ||
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100314 serialized.size() <= kExecutionBurstChannelSmallLength) {
315 return;
316 }
317
318 // by this point, execution should fail because the result channel isn't
319 // large enough to return the serialized result
Michael Butler648ada52019-07-25 17:22:11 -0700320 const auto [nSmall, outputShapesSmall, timingSmall, fallbackSmall] =
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100321 controllerSmall->compute(request, MeasureTiming::NO, keys);
Michael Butlerd09c0ee2020-03-12 15:12:23 -0700322 const V1_0::ErrorStatus statusSmall = nn::legacyConvertResultCodeToErrorStatus(nSmall);
Michael Butler79a41d72019-12-11 19:08:08 -0800323 EXPECT_NE(V1_0::ErrorStatus::NONE, statusSmall);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100324 EXPECT_EQ(0u, outputShapesSmall.size());
325 EXPECT_TRUE(badTiming(timingSmall));
Michael Butler648ada52019-07-25 17:22:11 -0700326 EXPECT_FALSE(fallbackSmall);
Lev Proleev13fdfcd2019-08-30 11:35:34 +0100327}
328
329static bool isSanitized(const FmqResultDatum& datum) {
330 using Discriminator = FmqResultDatum::hidl_discriminator;
331
332 // check to ensure the padding values in the returned
333 // FmqResultDatum::OperandInformation are initialized to 0
334 if (datum.getDiscriminator() == Discriminator::operandInformation) {
335 static_assert(
336 offsetof(FmqResultDatum::OperandInformation, isSufficient) == 0,
337 "unexpected value for offset of FmqResultDatum::OperandInformation::isSufficient");
338 static_assert(
339 sizeof(FmqResultDatum::OperandInformation::isSufficient) == 1,
340 "unexpected value for size of FmqResultDatum::OperandInformation::isSufficient");
341 static_assert(offsetof(FmqResultDatum::OperandInformation, numberOfDimensions) == 4,
342 "unexpected value for offset of "
343 "FmqResultDatum::OperandInformation::numberOfDimensions");
344 static_assert(sizeof(FmqResultDatum::OperandInformation::numberOfDimensions) == 4,
345 "unexpected value for size of "
346 "FmqResultDatum::OperandInformation::numberOfDimensions");
347 static_assert(sizeof(FmqResultDatum::OperandInformation) == 8,
348 "unexpected value for size of "
349 "FmqResultDatum::OperandInformation");
350
351 constexpr size_t paddingOffset =
352 offsetof(FmqResultDatum::OperandInformation, isSufficient) +
353 sizeof(FmqResultDatum::OperandInformation::isSufficient);
354 constexpr size_t paddingSize =
355 offsetof(FmqResultDatum::OperandInformation, numberOfDimensions) - paddingOffset;
356
357 FmqResultDatum::OperandInformation initialized{};
358 std::memset(&initialized, 0, sizeof(initialized));
359
360 const char* initializedPaddingStart =
361 reinterpret_cast<const char*>(&initialized) + paddingOffset;
362 const char* datumPaddingStart =
363 reinterpret_cast<const char*>(&datum.operandInformation()) + paddingOffset;
364
365 return std::memcmp(datumPaddingStart, initializedPaddingStart, paddingSize) == 0;
366 }
367
368 // there are no other padding initialization checks required, so return true
369 // for any sum-type that isn't FmqResultDatum::OperandInformation
370 return true;
371}
372
373static void validateBurstSanitized(const sp<IPreparedModel>& preparedModel,
374 const Request& request) {
375 // create burst
376 std::unique_ptr<RequestChannelSender> sender;
377 std::unique_ptr<ResultChannelReceiver> receiver;
378 sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
379 sp<IBurstContext> context;
380 ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context));
381 ASSERT_NE(nullptr, sender.get());
382 ASSERT_NE(nullptr, receiver.get());
383 ASSERT_NE(nullptr, context.get());
384
385 // load memory into callback slots
386 std::vector<intptr_t> keys;
387 keys.reserve(request.pools.size());
388 std::transform(request.pools.begin(), request.pools.end(), std::back_inserter(keys),
389 [](const auto& pool) { return reinterpret_cast<intptr_t>(&pool); });
390 const std::vector<int32_t> slots = callback->getSlots(request.pools, keys);
391
392 // send valid request
393 ASSERT_TRUE(sender->send(request, MeasureTiming::YES, slots));
394
395 // receive valid result
396 auto serialized = receiver->getPacketBlocking();
397 ASSERT_TRUE(serialized.has_value());
398
399 // sanitize result
400 ASSERT_TRUE(std::all_of(serialized->begin(), serialized->end(), isSanitized))
401 << "The result serialized data is not properly sanitized";
402}
403
404///////////////////////////// ENTRY POINT //////////////////////////////////
405
406void validateBurst(const sp<IPreparedModel>& preparedModel, const Request& request) {
407 ASSERT_NO_FATAL_FAILURE(validateBurstSerialization(preparedModel, request));
408 ASSERT_NO_FATAL_FAILURE(validateBurstFmqLength(preparedModel, request));
409 ASSERT_NO_FATAL_FAILURE(validateBurstSanitized(preparedModel, request));
410}
411
Lev Proleev26d1bc82019-08-30 11:57:18 +0100412} // namespace android::hardware::neuralnetworks::V1_3::vts::functional