Implements RANDOM_MULTINOMIAL CTS/VTS tests.
Bug: 113562775
Test: VtsHalNeuralnetworksV1_2TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.2::IDevice/sample-all
Change-Id: I3d39d056bf5034e01ce9c5c5e3f416a63d7ec8eb
diff --git a/neuralnetworks/1.0/vts/functional/Android.bp b/neuralnetworks/1.0/vts/functional/Android.bp
index 18f35c1..ffba45c 100644
--- a/neuralnetworks/1.0/vts/functional/Android.bp
+++ b/neuralnetworks/1.0/vts/functional/Android.bp
@@ -28,6 +28,7 @@
"android.hardware.neuralnetworks@1.2",
"android.hidl.allocator@1.0",
"android.hidl.memory@1.0",
+ "libgmock",
"libhidlmemory",
"libneuralnetworks_utils",
],
@@ -55,6 +56,7 @@
"android.hardware.neuralnetworks@1.2",
"android.hidl.allocator@1.0",
"android.hidl.memory@1.0",
+ "libgmock",
"libhidlmemory",
"libneuralnetworks_utils",
"VtsHalNeuralnetworksTest_utils",
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index b8046c7..cc19978 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -36,16 +36,17 @@
namespace generated_tests {
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
+using ::test_helper::compare;
+using ::test_helper::expectMultinomialDistributionWithinTolerance;
using ::test_helper::filter;
+using ::test_helper::Float32Operands;
using ::test_helper::for_all;
using ::test_helper::for_each;
-using ::test_helper::resize_accordingly;
-using ::test_helper::MixedTyped;
-using ::test_helper::MixedTypedExampleType;
-using ::test_helper::Float32Operands;
using ::test_helper::Int32Operands;
+using ::test_helper::MixedTyped;
+using ::test_helper::MixedTypedExample;
using ::test_helper::Quant8Operands;
-using ::test_helper::compare;
+using ::test_helper::resize_accordingly;
template <typename T>
void copy_back_(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
@@ -66,7 +67,7 @@
// Top level driver for models and examples generated by test_generator.py
// Test driver for those generated from ml/nn/runtime/test/spec
void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
- const std::vector<MixedTypedExampleType>& examples, float fpAtol = 1e-5f,
+ const std::vector<MixedTypedExample>& examples, float fpAtol = 1e-5f,
float fpRtol = 1e-5f) {
const uint32_t INPUT = 0;
const uint32_t OUTPUT = 1;
@@ -75,8 +76,8 @@
for (auto& example : examples) {
SCOPED_TRACE(example_no++);
- const MixedTyped& inputs = example.first;
- const MixedTyped& golden = example.second;
+ const MixedTyped& inputs = example.operands.first;
+ const MixedTyped& golden = example.operands.second;
std::vector<RequestArgument> inputs_info, outputs_info;
uint32_t inputSize = 0, outputSize = 0;
@@ -176,12 +177,15 @@
// We want "close-enough" results for float
compare(filtered_golden, filtered_test, fpAtol, fpRtol);
+
+ if (example.expectedMultinomialDistributionTolerance > 0) {
+ expectMultinomialDistributionWithinTolerance(test, example);
+ }
}
}
void Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model,
- std::function<bool(int)> is_ignored,
- const std::vector<MixedTypedExampleType>& examples) {
+ std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
V1_0::Model model = create_model();
// see if service can handle model
@@ -225,8 +229,7 @@
}
void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
- std::function<bool(int)> is_ignored,
- const std::vector<MixedTypedExampleType>& examples) {
+ std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
V1_1::Model model = create_model();
// see if service can handle model
@@ -277,8 +280,7 @@
// TODO: Reduce code duplication.
void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,
- std::function<bool(int)> is_ignored,
- const std::vector<MixedTypedExampleType>& examples) {
+ std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
V1_2::Model model = create_model();
// see if service can handle model
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp
index d84479c..ac1ae60 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp
@@ -31,9 +31,9 @@
namespace neuralnetworks {
namespace generated_tests {
-using ::test_helper::MixedTypedExampleType;
+using ::test_helper::MixedTypedExample;
extern void Execute(const sp<V1_0::IDevice>&, std::function<V1_0::Model(void)>,
- std::function<bool(int)>, const std::vector<MixedTypedExampleType>&);
+ std::function<bool(int)>, const std::vector<MixedTypedExample>&);
} // namespace generated_tests
namespace V1_0 {
@@ -43,9 +43,7 @@
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
using ::android::nn::allocateSharedMemory;
-
-// Mixed-typed examples
-typedef test_helper::MixedTypedExampleType MixedTypedExample;
+using ::test_helper::MixedTypedExample;
// in frameworks/ml/nn/runtime/tests/generated/
#include "all_generated_V1_0_vts_tests.cpp"
diff --git a/neuralnetworks/1.0/vts/functional/Models.h b/neuralnetworks/1.0/vts/functional/Models.h
index 751ab32..268e671 100644
--- a/neuralnetworks/1.0/vts/functional/Models.h
+++ b/neuralnetworks/1.0/vts/functional/Models.h
@@ -30,7 +30,7 @@
namespace vts {
namespace functional {
-using MixedTypedExample = test_helper::MixedTypedExampleType;
+using MixedTypedExample = test_helper::MixedTypedExample;
#define FOR_EACH_TEST_MODEL(FN) \
FN(add_broadcast_quant8) \
diff --git a/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp
index 09c1878..1d3dee3 100644
--- a/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp
+++ b/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp
@@ -36,9 +36,9 @@
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
using ::android::hidl::memory::V1_0::IMemory;
-using test_helper::MixedTyped;
-using test_helper::MixedTypedExampleType;
using test_helper::for_all;
+using test_helper::MixedTyped;
+using test_helper::MixedTypedExample;
///////////////////////// UTILITY FUNCTIONS /////////////////////////
@@ -151,15 +151,15 @@
///////////////////////////// ENTRY POINT //////////////////////////////////
-std::vector<Request> createRequests(const std::vector<MixedTypedExampleType>& examples) {
+std::vector<Request> createRequests(const std::vector<MixedTypedExample>& examples) {
const uint32_t INPUT = 0;
const uint32_t OUTPUT = 1;
std::vector<Request> requests;
- for (auto& example : examples) {
- const MixedTyped& inputs = example.first;
- const MixedTyped& outputs = example.second;
+ for (const MixedTypedExample& example : examples) {
+ const MixedTyped& inputs = example.operands.first;
+ const MixedTyped& outputs = example.operands.second;
std::vector<RequestArgument> inputs_info, outputs_info;
uint32_t inputSize = 0, outputSize = 0;
diff --git a/neuralnetworks/1.0/vts/functional/ValidationTests.cpp b/neuralnetworks/1.0/vts/functional/ValidationTests.cpp
index 98fc1c5..d3cbcff 100644
--- a/neuralnetworks/1.0/vts/functional/ValidationTests.cpp
+++ b/neuralnetworks/1.0/vts/functional/ValidationTests.cpp
@@ -27,7 +27,7 @@
namespace functional {
// forward declarations
-std::vector<Request> createRequests(const std::vector<MixedTypedExample>& examples);
+std::vector<Request> createRequests(const std::vector<::test_helper::MixedTypedExample>& examples);
// generate validation tests
#define VTS_CURRENT_TEST_CASE(TestName) \
diff --git a/neuralnetworks/1.1/vts/functional/Android.bp b/neuralnetworks/1.1/vts/functional/Android.bp
index 52a804a..a1c0f1f 100644
--- a/neuralnetworks/1.1/vts/functional/Android.bp
+++ b/neuralnetworks/1.1/vts/functional/Android.bp
@@ -31,6 +31,7 @@
"android.hardware.neuralnetworks@1.2",
"android.hidl.allocator@1.0",
"android.hidl.memory@1.0",
+ "libgmock",
"libhidlmemory",
"libneuralnetworks_utils",
"VtsHalNeuralnetworksTest_utils",
diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp
index 95c2b1a..1f49904 100644
--- a/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp
+++ b/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp
@@ -31,9 +31,9 @@
namespace neuralnetworks {
namespace generated_tests {
-using ::test_helper::MixedTypedExampleType;
+using ::test_helper::MixedTypedExample;
extern void Execute(const sp<V1_1::IDevice>&, std::function<V1_1::Model(void)>,
- std::function<bool(int)>, const std::vector<MixedTypedExampleType>&);
+ std::function<bool(int)>, const std::vector<MixedTypedExample>&);
} // namespace generated_tests
namespace V1_1 {
@@ -43,9 +43,7 @@
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
using ::android::nn::allocateSharedMemory;
-
-// Mixed-typed examples
-typedef generated_tests::MixedTypedExampleType MixedTypedExample;
+using ::test_helper::MixedTypedExample;
// in frameworks/ml/nn/runtime/tests/generated/
#include "all_generated_V1_0_vts_tests.cpp"
diff --git a/neuralnetworks/1.1/vts/functional/Models.h b/neuralnetworks/1.1/vts/functional/Models.h
index cc0fac1..62bc95e 100644
--- a/neuralnetworks/1.1/vts/functional/Models.h
+++ b/neuralnetworks/1.1/vts/functional/Models.h
@@ -31,7 +31,7 @@
namespace vts {
namespace functional {
-using MixedTypedExample = test_helper::MixedTypedExampleType;
+using MixedTypedExample = test_helper::MixedTypedExample;
#define FOR_EACH_TEST_MODEL(FN) \
FN(add) \
diff --git a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
index 687b760..e7d96c7 100644
--- a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
+++ b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
@@ -36,9 +36,9 @@
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
using ::android::hidl::memory::V1_0::IMemory;
-using test_helper::MixedTyped;
-using test_helper::MixedTypedExampleType;
using test_helper::for_all;
+using test_helper::MixedTyped;
+using test_helper::MixedTypedExample;
///////////////////////// UTILITY FUNCTIONS /////////////////////////
@@ -152,15 +152,15 @@
///////////////////////////// ENTRY POINT //////////////////////////////////
-std::vector<Request> createRequests(const std::vector<MixedTypedExampleType>& examples) {
+std::vector<Request> createRequests(const std::vector<MixedTypedExample>& examples) {
const uint32_t INPUT = 0;
const uint32_t OUTPUT = 1;
std::vector<Request> requests;
for (auto& example : examples) {
- const MixedTyped& inputs = example.first;
- const MixedTyped& outputs = example.second;
+ const MixedTyped& inputs = example.operands.first;
+ const MixedTyped& outputs = example.operands.second;
std::vector<RequestArgument> inputs_info, outputs_info;
uint32_t inputSize = 0, outputSize = 0;
diff --git a/neuralnetworks/1.2/vts/functional/Android.bp b/neuralnetworks/1.2/vts/functional/Android.bp
index 2dc19cc..09d0dc3 100644
--- a/neuralnetworks/1.2/vts/functional/Android.bp
+++ b/neuralnetworks/1.2/vts/functional/Android.bp
@@ -31,6 +31,7 @@
"android.hardware.neuralnetworks@1.2",
"android.hidl.allocator@1.0",
"android.hidl.memory@1.0",
+ "libgmock",
"libhidlmemory",
"libneuralnetworks_utils",
"VtsHalNeuralnetworksTest_utils",
diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp
index 662c531..e87fa6b 100644
--- a/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp
+++ b/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp
@@ -31,9 +31,9 @@
namespace neuralnetworks {
namespace generated_tests {
-using ::test_helper::MixedTypedExampleType;
+using ::test_helper::MixedTypedExample;
extern void Execute(const sp<V1_2::IDevice>&, std::function<V1_2::Model(void)>,
- std::function<bool(int)>, const std::vector<MixedTypedExampleType>&);
+ std::function<bool(int)>, const std::vector<MixedTypedExample>&);
} // namespace generated_tests
namespace V1_2 {
@@ -43,9 +43,7 @@
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
using ::android::nn::allocateSharedMemory;
-
-// Mixed-typed examples
-typedef generated_tests::MixedTypedExampleType MixedTypedExample;
+using ::test_helper::MixedTypedExample;
// in frameworks/ml/nn/runtime/tests/generated/
#include "all_generated_V1_0_vts_tests.cpp"
diff --git a/neuralnetworks/1.2/vts/functional/Models.h b/neuralnetworks/1.2/vts/functional/Models.h
index f3769bc..2d512fe 100644
--- a/neuralnetworks/1.2/vts/functional/Models.h
+++ b/neuralnetworks/1.2/vts/functional/Models.h
@@ -32,7 +32,7 @@
namespace vts {
namespace functional {
-using MixedTypedExample = test_helper::MixedTypedExampleType;
+using MixedTypedExample = test_helper::MixedTypedExample;
#define FOR_EACH_TEST_MODEL(FN) \
FN(add) \
@@ -243,6 +243,7 @@
FN(pad_float_1) \
FN(pad_float_1_relaxed) \
FN(pad_relaxed) \
+ FN(random_multinomial) \
FN(relu1_float_1) \
FN(relu1_float_1_relaxed) \
FN(relu1_float_2) \
diff --git a/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp
index f4476fa..b663535 100644
--- a/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp
+++ b/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp
@@ -38,7 +38,7 @@
using ::android::hidl::memory::V1_0::IMemory;
using test_helper::for_all;
using test_helper::MixedTyped;
-using test_helper::MixedTypedExampleType;
+using test_helper::MixedTypedExample;
///////////////////////// UTILITY FUNCTIONS /////////////////////////
@@ -152,15 +152,15 @@
///////////////////////////// ENTRY POINT //////////////////////////////////
-std::vector<Request> createRequests(const std::vector<MixedTypedExampleType>& examples) {
+std::vector<Request> createRequests(const std::vector<MixedTypedExample>& examples) {
const uint32_t INPUT = 0;
const uint32_t OUTPUT = 1;
std::vector<Request> requests;
for (auto& example : examples) {
- const MixedTyped& inputs = example.first;
- const MixedTyped& outputs = example.second;
+ const MixedTyped& inputs = example.operands.first;
+ const MixedTyped& outputs = example.operands.second;
std::vector<RequestArgument> inputs_info, outputs_info;
uint32_t inputSize = 0, outputSize = 0;