Add VTS tests for NeuralNetworks v1.2
This is a copy the v1.1 tests since we don't have any new ops
implemented in v1.2 yet.
Bug: 114365802
Test: mm
Test: NNAPI VTS
Change-Id: Ida7525fcd3ae0fd6f88ff9591e06aba922bdae64
diff --git a/neuralnetworks/1.0/vts/functional/Android.bp b/neuralnetworks/1.0/vts/functional/Android.bp
index e28113b..18f35c1 100644
--- a/neuralnetworks/1.0/vts/functional/Android.bp
+++ b/neuralnetworks/1.0/vts/functional/Android.bp
@@ -25,6 +25,7 @@
static_libs: [
"android.hardware.neuralnetworks@1.0",
"android.hardware.neuralnetworks@1.1",
+ "android.hardware.neuralnetworks@1.2",
"android.hidl.allocator@1.0",
"android.hidl.memory@1.0",
"libhidlmemory",
@@ -49,8 +50,9 @@
],
defaults: ["VtsHalTargetTestDefaults"],
static_libs: [
- "android.hardware.neuralnetworks@1.1",
"android.hardware.neuralnetworks@1.0",
+ "android.hardware.neuralnetworks@1.1",
+ "android.hardware.neuralnetworks@1.2",
"android.hidl.allocator@1.0",
"android.hidl.memory@1.0",
"libhidlmemory",
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index 64495cf..b8046c7 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -275,6 +275,58 @@
EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol);
}
+// TODO: Reduce code duplication.
+void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,
+ std::function<bool(int)> is_ignored,
+ const std::vector<MixedTypedExampleType>& examples) {
+ V1_2::Model model = create_model();
+
+ // see if service can handle model
+ bool fullySupportsModel = false;
+ Return<void> supportedCall = device->getSupportedOperations_1_2(
+ model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
+ ASSERT_EQ(ErrorStatus::NONE, status);
+ ASSERT_NE(0ul, supported.size());
+ fullySupportsModel =
+ std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
+ });
+ ASSERT_TRUE(supportedCall.isOk());
+
+ // launch prepare model
+ sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+ ASSERT_NE(nullptr, preparedModelCallback.get());
+ Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_2(
+ model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
+ ASSERT_TRUE(prepareLaunchStatus.isOk());
+ ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
+
+ // retrieve prepared model
+ preparedModelCallback->wait();
+ ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
+ sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
+
+ // early termination if vendor service cannot fully prepare model
+ if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
+ ASSERT_EQ(nullptr, preparedModel.get());
+ LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
+ "prepare model that it does not support.";
+ std::cout << "[ ] Early termination of test because vendor service cannot "
+ "prepare model that it does not support."
+ << std::endl;
+ return;
+ }
+ EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
+ ASSERT_NE(nullptr, preparedModel.get());
+
+ // TODO: Adjust the error limit based on testing.
+ // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
+ float fpAtol = !model.relaxComputationFloat32toFloat16 ? 1e-5f : 5.0f * 0.0009765625f;
+ // Set the relative tolerance to be 5ULP of the corresponding FP precision.
+ float fpRtol = !model.relaxComputationFloat32toFloat16 ? 5.0f * 1.1920928955078125e-7f
+ : 5.0f * 0.0009765625f;
+ EvaluatePreparedModel(preparedModel, is_ignored, examples, fpAtol, fpRtol);
+}
+
} // namespace generated_tests
} // namespace neuralnetworks