Support ignored output in test harness
Filtered out outputs that are not needed for correctness verification
during result validation.
Bug: 63905942
Test: VtsHalNeuralnetworksV1_0TargetTest on Angler
Change-Id: Ic967a821c2a60194c9d9f386601a865f5c409a4f
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index 9c85c8c..db90ac2 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -33,6 +33,7 @@
namespace generated_tests {
using ::android::hardware::neuralnetworks::V1_0::implementation::Event;
+using ::generated_tests::filter;
using ::generated_tests::for_all;
using ::generated_tests::for_each;
using ::generated_tests::resize_accordingly;
@@ -44,6 +45,7 @@
// Top level driver for models and examples generated by test_generator.py
// Test driver for those generated from ml/nn/runtime/test/spec
void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
+ std::function<bool(int)> is_ignored,
const std::vector<MixedTypedExampleType>& examples) {
Model model = create_model();
sp<IPreparedModel> preparedModel;
@@ -166,18 +168,28 @@
COPY_BACK(uint8_t);
#undef COPY_BACK
outputMemory->commit();
+ // Filter out don't cares
+ MixedTyped filtered_golden;
+ MixedTyped filtered_test;
+ filter<float>(golden, &filtered_golden, is_ignored);
+ filter<float>(test, &filtered_test, is_ignored);
+ filter<int32_t>(golden, &filtered_golden, is_ignored);
+ filter<int32_t>(test, &filtered_test, is_ignored);
+ filter<uint8_t>(golden, &filtered_golden, is_ignored);
+ filter<uint8_t>(test, &filtered_test, is_ignored);
+
// We want "close-enough" results for float
- for_each<float>(golden, [&test](int index, auto& golden_float) {
- auto& test_float_operands = std::get<Float32Operands>(test);
+ for_each<float>(filtered_golden, [&filtered_test](int index, auto& golden_float) {
+ auto& test_float_operands = std::get<Float32Operands>(filtered_test);
auto& test_float = test_float_operands[index];
for (unsigned int i = 0; i < golden_float.size(); i++) {
SCOPED_TRACE(i);
EXPECT_NEAR(golden_float[i], test_float[i], 1.e-5);
}
});
-
- EXPECT_EQ(std::get<Int32Operands>(golden), std::get<Int32Operands>(test));
- EXPECT_EQ(std::get<Quant8Operands>(golden), std::get<Quant8Operands>(test));
+ EXPECT_EQ(std::get<Int32Operands>(filtered_golden), std::get<Int32Operands>(filtered_test));
+ EXPECT_EQ(std::get<Quant8Operands>(filtered_golden),
+ std::get<Quant8Operands>(filtered_test));
}
}
diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp
index 0302717..453e3e5 100644
--- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp
+++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworksV1_0TargetTest.cpp
@@ -34,7 +34,7 @@
using ::android::hardware::neuralnetworks::V1_0::implementation::Event;
using ::generated_tests::MixedTypedExampleType;
namespace generated_tests {
-extern void Execute(const sp<IDevice>&, std::function<Model(void)>,
+extern void Execute(const sp<IDevice>&, std::function<Model(void)>, std::function<bool(int)>,
const std::vector<MixedTypedExampleType>&);
}