Refactor NN API VTS tests and add v1.1 tests
- Create VtsHalNeuralnetworksTest_utils module to be shared between
v1.x tests.
- Split the existing tests into two categories: basic, and generated.
- Created v1.1 VTS tests ensuring no regression in existing ML models.
Bug: 63911257
Test: mm
Test: NNAPI VTS tests pass on v1.0 and v1.1 sample drivers
Merged-In: Ic77c90a3a5bbd96b0ce2acd03764dde4b3034cc9
Change-Id: Ic77c90a3a5bbd96b0ce2acd03764dde4b3034cc9
(cherry picked from commit a2d04c828e98bdadc6dd44c6235556451e4e2a88)
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index d740b5f..5fe8415 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -16,9 +16,15 @@
#include "Callbacks.h"
#include "TestHarness.h"
-#include "VtsHalNeuralnetworksV1_0TargetTest.h"
+#include "Utils.h"
#include <android-base/logging.h>
+#include <android/hardware/neuralnetworks/1.0/IDevice.h>
+#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
+#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
+#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
+#include <android/hardware/neuralnetworks/1.0/types.h>
+#include <android/hidl/allocator/1.0/IAllocator.h>
#include <android/hidl/memory/1.0/IMemory.h>
#include <hidlmemory/mapping.h>
#include <iostream>
@@ -26,11 +32,6 @@
namespace android {
namespace hardware {
namespace neuralnetworks {
-namespace V1_0 {
-namespace vts {
-namespace functional {
-// allocator helper
-hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem");
namespace generated_tests {
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
@@ -64,54 +65,10 @@
// Top level driver for models and examples generated by test_generator.py
// Test driver for those generated from ml/nn/runtime/test/spec
-void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
- std::function<bool(int)> is_ignored,
- const std::vector<MixedTypedExampleType>& examples) {
+void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
+ const std::vector<MixedTypedExampleType>& examples) {
const uint32_t INPUT = 0;
const uint32_t OUTPUT = 1;
- Model model = create_model();
-
- // see if service can handle model
- ErrorStatus supportedStatus;
- bool fullySupportsModel = false;
- Return<void> supportedCall = device->getSupportedOperations(
- model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
- supportedStatus = status;
- ASSERT_NE(0ul, supported.size());
- fullySupportsModel =
- std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
- });
- ASSERT_TRUE(supportedCall.isOk());
- ASSERT_EQ(ErrorStatus::NONE, supportedStatus);
-
- // launch prepare model
- sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
- ASSERT_NE(nullptr, preparedModelCallback.get());
- Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
- ASSERT_TRUE(prepareLaunchStatus.isOk());
-
- // retrieve prepared model
- preparedModelCallback->wait();
- ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
- sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
- if (fullySupportsModel) {
- EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
- } else {
- EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||
- prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);
- }
-
- // early termination if vendor service cannot fully prepare model
- if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) {
- ASSERT_EQ(nullptr, preparedModel.get());
- LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
- "prepare model that it does not support.";
- std::cout << "[ ] Early termination of test because vendor service cannot "
- "prepare model that it does not support."
- << std::endl;
- return;
- }
- ASSERT_NE(nullptr, preparedModel.get());
int example_no = 1;
for (auto& example : examples) {
@@ -167,8 +124,8 @@
offset += i.location.length;
}
}
- std::vector<hidl_memory> pools = {allocateSharedMemory(inputSize),
- allocateSharedMemory(outputSize)};
+ std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
+ nn::allocateSharedMemory(outputSize)};
ASSERT_NE(0ull, pools[INPUT].size());
ASSERT_NE(0ull, pools[OUTPUT].size());
@@ -221,11 +178,107 @@
}
}
+void Execute(sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,
+ std::function<bool(int)> is_ignored,
+ const std::vector<MixedTypedExampleType>& examples) {
+ V1_0::Model model = create_model();
+
+ // see if service can handle model
+ bool fullySupportsModel = false;
+ ErrorStatus supportedStatus;
+ sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+ ASSERT_NE(nullptr, preparedModelCallback.get());
+
+ Return<void> supportedCall = device->getSupportedOperations(
+ model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
+ supportedStatus = status;
+ ASSERT_NE(0ul, supported.size());
+ fullySupportsModel =
+ std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
+ });
+ ASSERT_TRUE(supportedCall.isOk());
+ ASSERT_EQ(ErrorStatus::NONE, supportedStatus);
+ Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
+ ASSERT_TRUE(prepareLaunchStatus.isOk());
+
+ // retrieve prepared model
+ preparedModelCallback->wait();
+ ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
+ sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
+ if (fullySupportsModel) {
+ EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
+ } else {
+ EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||
+ prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);
+ }
+
+ // early termination if vendor service cannot fully prepare model
+ if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) {
+ ASSERT_EQ(nullptr, preparedModel.get());
+ LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
+ "prepare model that it does not support.";
+ std::cout << "[ ] Early termination of test because vendor service cannot "
+ "prepare model that it does not support."
+ << std::endl;
+ return;
+ }
+ ASSERT_NE(nullptr, preparedModel.get());
+
+ EvaluatePreparedModel(preparedModel, is_ignored, examples);
+}
+
+void Execute(sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
+ std::function<bool(int)> is_ignored,
+ const std::vector<MixedTypedExampleType>& examples) {
+ V1_1::Model model = create_model();
+
+ // see if service can handle model
+ bool fullySupportsModel = false;
+ ErrorStatus supportedStatus;
+ sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+ ASSERT_NE(nullptr, preparedModelCallback.get());
+
+ Return<void> supportedCall = device->getSupportedOperations_1_1(
+ model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
+ supportedStatus = status;
+ ASSERT_NE(0ul, supported.size());
+ fullySupportsModel =
+ std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
+ });
+ ASSERT_TRUE(supportedCall.isOk());
+ ASSERT_EQ(ErrorStatus::NONE, supportedStatus);
+ Return<ErrorStatus> prepareLaunchStatus =
+ device->prepareModel_1_1(model, preparedModelCallback);
+ ASSERT_TRUE(prepareLaunchStatus.isOk());
+
+ // retrieve prepared model
+ preparedModelCallback->wait();
+ ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
+ sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
+ if (fullySupportsModel) {
+ EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
+ } else {
+ EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||
+ prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);
+ }
+
+ // early termination if vendor service cannot fully prepare model
+ if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) {
+ ASSERT_EQ(nullptr, preparedModel.get());
+ LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
+ "prepare model that it does not support.";
+ std::cout << "[ ] Early termination of test because vendor service cannot "
+ "prepare model that it does not support."
+ << std::endl;
+ return;
+ }
+ ASSERT_NE(nullptr, preparedModel.get());
+
+ EvaluatePreparedModel(preparedModel, is_ignored, examples);
+}
+
} // namespace generated_tests
-} // namespace functional
-} // namespace vts
-} // namespace V1_0
} // namespace neuralnetworks
} // namespace hardware
} // namespace android