Add NNAPI loop timeout VTS test
Fix: 149554639
Bug: 145906499
Bug: 136735929
Test: VtsHalNeuralnetworksV1_3TargetTest
Change-Id: I642772d3c00c12d72380598d9d86743706179d72
Merged-In: I642772d3c00c12d72380598d9d86743706179d72
(cherry picked from commit cc873aee8fb7921aa56e8528e9fe8cb0cdc3a0ba)
diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp
index 89edfb7..b04abe2 100644
--- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp
@@ -74,7 +74,7 @@
enum class Executor { ASYNC, SYNC, BURST, FENCED };
-enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT };
+enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT, MISSED_DEADLINE };
enum class MemoryType { SHARED, DEVICE };
@@ -495,16 +495,18 @@
static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
const Request& request, MeasureTiming measure,
+ const OptionalTimeoutDuration& loopTimeoutDuration,
sp<ExecutionCallback>& callback) {
- return preparedModel->execute_1_3(request, measure, {}, {}, callback);
+ return preparedModel->execute_1_3(request, measure, {}, loopTimeoutDuration, callback);
}
static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
const Request& request, MeasureTiming measure,
+ const OptionalTimeoutDuration& loopTimeoutDuration,
hidl_vec<OutputShape>* outputShapes,
Timing* timing) {
ErrorStatus result;
Return<void> ret = preparedModel->executeSynchronously_1_3(
- request, measure, {}, {},
+ request, measure, {}, loopTimeoutDuration,
[&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes,
const Timing& time) {
result = error;
@@ -545,6 +547,17 @@
makeOutputInsufficientSize(/*outputIndex=*/0, &request);
}
+ OptionalTimeoutDuration loopTimeoutDuration;
+ // OutputType::MISSED_DEADLINE is only used by
+ // TestKind::INTINITE_LOOP_TIMEOUT tests to verify that an infinite loop is
+ // aborted after a timeout.
+ if (testConfig.outputType == OutputType::MISSED_DEADLINE) {
+ // Override the default loop timeout duration with a small value to
+ // speed up test execution.
+ constexpr uint64_t kMillisecond = 1'000'000;
+ loopTimeoutDuration.nanoseconds(1 * kMillisecond);
+ }
+
ErrorStatus executionStatus;
hidl_vec<OutputShape> outputShapes;
Timing timing;
@@ -554,8 +567,9 @@
// launch execution
sp<ExecutionCallback> executionCallback = new ExecutionCallback();
- Return<ErrorStatus> executionLaunchStatus = ExecutePreparedModel(
- preparedModel, request, testConfig.measureTiming, executionCallback);
+ Return<ErrorStatus> executionLaunchStatus =
+ ExecutePreparedModel(preparedModel, request, testConfig.measureTiming,
+ loopTimeoutDuration, executionCallback);
ASSERT_TRUE(executionLaunchStatus.isOk());
EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
@@ -571,8 +585,9 @@
SCOPED_TRACE("synchronous");
// execute
- Return<ErrorStatus> executionReturnStatus = ExecutePreparedModel(
- preparedModel, request, testConfig.measureTiming, &outputShapes, &timing);
+ Return<ErrorStatus> executionReturnStatus =
+ ExecutePreparedModel(preparedModel, request, testConfig.measureTiming,
+ loopTimeoutDuration, &outputShapes, &timing);
ASSERT_TRUE(executionReturnStatus.isOk());
executionStatus = static_cast<ErrorStatus>(executionReturnStatus);
@@ -612,7 +627,7 @@
hidl_handle syncFenceHandle;
sp<IFencedExecutionCallback> fencedCallback;
Return<void> ret = preparedModel->executeFenced(
- request, {}, testConfig.measureTiming, {}, {}, {},
+ request, {}, testConfig.measureTiming, {}, loopTimeoutDuration, {},
[&result, &syncFenceHandle, &fencedCallback](
ErrorStatus error, const hidl_handle& handle,
const sp<IFencedExecutionCallback>& callback) {
@@ -686,6 +701,11 @@
ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size());
ASSERT_FALSE(outputShapes[0].isSufficient);
return;
+ case OutputType::MISSED_DEADLINE:
+ ASSERT_TRUE(executionStatus == ErrorStatus::MISSED_DEADLINE_TRANSIENT ||
+ executionStatus == ErrorStatus::MISSED_DEADLINE_PERSISTENT)
+ << "executionStatus = " << executionStatus;
+ return;
}
// Go through all outputs, check returned output shapes.
@@ -736,6 +756,12 @@
LOG(FATAL) << "Wrong TestKind for EvaluatePreparedModel";
return;
} break;
+ case TestKind::INTINITE_LOOP_TIMEOUT: {
+ outputTypesList = {OutputType::MISSED_DEADLINE};
+ measureTimingList = {MeasureTiming::NO, MeasureTiming::YES};
+ // Burst does not support V1_3 loop timeout.
+ executorList = {Executor::ASYNC, Executor::SYNC, Executor::FENCED};
+ } break;
}
for (const OutputType outputType : outputTypesList) {
@@ -794,7 +820,8 @@
case TestKind::GENERAL:
case TestKind::DYNAMIC_SHAPE:
case TestKind::MEMORY_DOMAIN:
- case TestKind::FENCED_COMPUTE: {
+ case TestKind::FENCED_COMPUTE:
+ case TestKind::INTINITE_LOOP_TIMEOUT: {
createPreparedModel(device, model, &preparedModel);
if (preparedModel == nullptr) return;
EvaluatePreparedModel(device, preparedModel, testModel, testKind);
@@ -863,24 +890,31 @@
// Tag for the dynamic output shape tests
class QuantizationCouplingTest : public GeneratedTest {};
+// Tag for the loop timeout tests
+class InfiniteLoopTimeoutTest : public GeneratedTest {};
+
TEST_P(GeneratedTest, Test) {
- Execute(kDevice, kTestModel, /*testKind=*/TestKind::GENERAL);
+ Execute(kDevice, kTestModel, TestKind::GENERAL);
}
TEST_P(DynamicOutputShapeTest, Test) {
- Execute(kDevice, kTestModel, /*testKind=*/TestKind::DYNAMIC_SHAPE);
+ Execute(kDevice, kTestModel, TestKind::DYNAMIC_SHAPE);
}
TEST_P(MemoryDomainTest, Test) {
- Execute(kDevice, kTestModel, /*testKind=*/TestKind::MEMORY_DOMAIN);
+ Execute(kDevice, kTestModel, TestKind::MEMORY_DOMAIN);
}
TEST_P(FencedComputeTest, Test) {
- Execute(kDevice, kTestModel, /*testKind=*/TestKind::FENCED_COMPUTE);
+ Execute(kDevice, kTestModel, TestKind::FENCED_COMPUTE);
}
TEST_P(QuantizationCouplingTest, Test) {
- Execute(kDevice, kTestModel, /*testKind=*/TestKind::QUANTIZATION_COUPLING);
+ Execute(kDevice, kTestModel, TestKind::QUANTIZATION_COUPLING);
+}
+
+TEST_P(InfiniteLoopTimeoutTest, Test) {
+ Execute(kDevice, kTestModel, TestKind::INTINITE_LOOP_TIMEOUT);
}
INSTANTIATE_GENERATED_TEST(GeneratedTest,
@@ -900,4 +934,8 @@
return testModel.hasQuant8CoupledOperands() && testModel.main.operations.size() == 1;
});
+INSTANTIATE_GENERATED_TEST(InfiniteLoopTimeoutTest, [](const TestModel& testModel) {
+ return testModel.isInfiniteLoopTimeoutTest();
+});
+
} // namespace android::hardware::neuralnetworks::V1_3::vts::functional