Update neuralnetworks HAL to allow collecting execution duration.
Test: VtsHalNeuralnetworksV1_0TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.0::IDevice/sample-all
Test: VtsHalNeuralnetworksV1_1TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.1::IDevice/sample-all
Test: VtsHalNeuralnetworksV1_2TargetTest --hal_service_instance=android.hardware.neuralnetworks@1.2::IDevice/sample-all
Bug: 115390094
Change-Id: If67a5ffe39cfdd78498e01f26251734fdc8e66c7
Merged-In: If67a5ffe39cfdd78498e01f26251734fdc8e66c7
(cherry picked from commit e301349b0597f2cc136703f2aaa182e6a546b50c)
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index b5a8607..65c425e 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -45,20 +45,16 @@
using ::test_helper::compare;
using ::test_helper::expectMultinomialDistributionWithinTolerance;
using ::test_helper::filter;
-using ::test_helper::Float32Operands;
using ::test_helper::for_all;
using ::test_helper::for_each;
-using ::test_helper::Int32Operands;
using ::test_helper::MixedTyped;
using ::test_helper::MixedTypedExample;
-using ::test_helper::MixedTypedIndex;
-using ::test_helper::Quant8Operands;
using ::test_helper::resize_accordingly;
template <typename T>
-void copy_back_(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
- MixedTyped& test = *dst;
- for_each<T>(test, [&ra, src](int index, std::vector<T>& m) {
+void copy_back_(std::map<int, std::vector<T>>* dst, const std::vector<RequestArgument>& ra,
+ char* src) {
+ for_each<T>(*dst, [&ra, src](int index, std::vector<T>& m) {
ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T));
char* begin = src + ra[index].location.offset;
memcpy(m.data(), begin, ra[index].location.length);
@@ -66,43 +62,48 @@
}
void copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) {
- copy_back_<float>(dst, ra, src);
- copy_back_<int32_t>(dst, ra, src);
- copy_back_<uint8_t>(dst, ra, src);
- copy_back_<int16_t>(dst, ra, src);
- copy_back_<_Float16>(dst, ra, src);
- copy_back_<bool8>(dst, ra, src);
- copy_back_<int8_t>(dst, ra, src);
- static_assert(7 == std::tuple_size<MixedTyped>::value,
+ copy_back_(&dst->float32Operands, ra, src);
+ copy_back_(&dst->int32Operands, ra, src);
+ copy_back_(&dst->quant8AsymmOperands, ra, src);
+ copy_back_(&dst->quant16SymmOperands, ra, src);
+ copy_back_(&dst->float16Operands, ra, src);
+ copy_back_(&dst->bool8Operands, ra, src);
+ copy_back_(&dst->quant8ChannelOperands, ra, src);
+ copy_back_(&dst->quant16AsymmOperands, ra, src);
+ static_assert(8 == MixedTyped::kNumTypes,
"Number of types in MixedTyped changed, but copy_back function wasn't updated");
}
// Top level driver for models and examples generated by test_generator.py
// Test driver for those generated from ml/nn/runtime/test/spec
static Return<ErrorStatus> ExecutePreparedModel(sp<V1_0::IPreparedModel>& preparedModel,
- const Request& request,
+ const Request& request, MeasureTiming,
sp<ExecutionCallback>& callback) {
return preparedModel->execute(request, callback);
}
static Return<ErrorStatus> ExecutePreparedModel(sp<V1_2::IPreparedModel>& preparedModel,
- const Request& request,
+ const Request& request, MeasureTiming measure,
sp<ExecutionCallback>& callback) {
- return preparedModel->execute_1_2(request, callback);
+ return preparedModel->execute_1_2(request, measure, callback);
}
static Return<ErrorStatus> ExecutePreparedModel(sp<V1_0::IPreparedModel>&, const Request&,
- hidl_vec<OutputShape>*) {
+ MeasureTiming, hidl_vec<OutputShape>*, Timing*) {
ADD_FAILURE() << "asking for synchronous execution at V1_0";
return ErrorStatus::GENERAL_FAILURE;
}
static Return<ErrorStatus> ExecutePreparedModel(sp<V1_2::IPreparedModel>& preparedModel,
- const Request& request,
- hidl_vec<OutputShape>* outputShapes) {
+ const Request& request, MeasureTiming measure,
+ hidl_vec<OutputShape>* outputShapes,
+ Timing* timing) {
ErrorStatus result;
Return<void> ret = preparedModel->executeSynchronously(
- request, [&result, &outputShapes](ErrorStatus error, const hidl_vec<OutputShape>& shapes) {
- result = error;
- *outputShapes = shapes;
- });
+ request, measure,
+ [&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes,
+ const Timing& time) {
+ result = error;
+ *outputShapes = shapes;
+ *timing = time;
+ });
if (!ret.isOk()) {
return ErrorStatus::GENERAL_FAILURE;
}
@@ -114,8 +115,8 @@
template <typename T_IPreparedModel>
void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
const std::vector<MixedTypedExample>& examples,
- bool hasRelaxedFloat32Model = false, float fpAtol = kDefaultAtol,
- float fpRtol = kDefaultRtol, Synchronously sync = Synchronously::NO) {
+ bool hasRelaxedFloat32Model, float fpAtol, float fpRtol,
+ Synchronously sync, MeasureTiming measure, bool testDynamicOutputShape) {
const uint32_t INPUT = 0;
const uint32_t OUTPUT = 1;
@@ -125,7 +126,7 @@
const MixedTyped& inputs = example.operands.first;
const MixedTyped& golden = example.operands.second;
- const bool hasFloat16Inputs = !std::get<MixedTypedIndex<_Float16>::index>(inputs).empty();
+ const bool hasFloat16Inputs = !inputs.float16Operands.empty();
if (hasRelaxedFloat32Model || hasFloat16Inputs) {
// TODO: Adjust the error limit based on testing.
// If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
@@ -210,6 +211,7 @@
ErrorStatus executionStatus;
hidl_vec<OutputShape> outputShapes;
+ Timing timing;
if (sync == Synchronously::NO) {
SCOPED_TRACE("asynchronous");
@@ -217,8 +219,8 @@
sp<ExecutionCallback> executionCallback = new ExecutionCallback();
ASSERT_NE(nullptr, executionCallback.get());
Return<ErrorStatus> executionLaunchStatus = ExecutePreparedModel(
- preparedModel, {.inputs = inputs_info, .outputs = outputs_info, .pools = pools},
- executionCallback);
+ preparedModel, {.inputs = inputs_info, .outputs = outputs_info, .pools = pools},
+ measure, executionCallback);
ASSERT_TRUE(executionLaunchStatus.isOk());
EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
@@ -226,21 +228,44 @@
executionCallback->wait();
executionStatus = executionCallback->getStatus();
outputShapes = executionCallback->getOutputShapes();
+ timing = executionCallback->getTiming();
} else {
SCOPED_TRACE("synchronous");
// execute
Return<ErrorStatus> executionReturnStatus = ExecutePreparedModel(
- preparedModel, {.inputs = inputs_info, .outputs = outputs_info, .pools = pools},
- &outputShapes);
+ preparedModel, {.inputs = inputs_info, .outputs = outputs_info, .pools = pools},
+ measure, &outputShapes, &timing);
ASSERT_TRUE(executionReturnStatus.isOk());
executionStatus = static_cast<ErrorStatus>(executionReturnStatus);
}
+ if (testDynamicOutputShape && executionStatus != ErrorStatus::NONE) {
+ LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
+ "execute model that it does not support.";
+ std::cout << "[ ] Early termination of test because vendor service cannot "
+ "execute model that it does not support."
+ << std::endl;
+ return;
+ }
ASSERT_EQ(ErrorStatus::NONE, executionStatus);
- // TODO(xusongw): Check if the returned output shapes match with expectation once the
- // sample driver implementation of dynamic output shape is finished.
- ASSERT_EQ(outputShapes.size(), 0);
+ if (measure == MeasureTiming::NO) {
+ EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);
+ EXPECT_EQ(UINT64_MAX, timing.timeInDriver);
+ } else {
+ if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) {
+ EXPECT_LE(timing.timeOnDevice, timing.timeInDriver);
+ }
+ }
+
+ // Go through all outputs, overwrite output dimensions with returned output shapes
+ if (testDynamicOutputShape) {
+ ASSERT_NE(outputShapes.size(), 0);
+ for_each<uint32_t>(test.operandDimensions,
+ [&outputShapes](int idx, std::vector<uint32_t>& dim) {
+ dim = outputShapes[idx].dimensions;
+ });
+ }
// validate results
outputMemory->read();
@@ -261,9 +286,10 @@
template <typename T_IPreparedModel>
void EvaluatePreparedModel(sp<T_IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
const std::vector<MixedTypedExample>& examples,
- bool hasRelaxedFloat32Model, Synchronously sync) {
+ bool hasRelaxedFloat32Model, Synchronously sync, MeasureTiming measure,
+ bool testDynamicOutputShape) {
EvaluatePreparedModel(preparedModel, is_ignored, examples, hasRelaxedFloat32Model, kDefaultAtol,
- kDefaultRtol, sync);
+ kDefaultRtol, sync, measure, testDynamicOutputShape);
}
static void getPreparedModel(sp<PreparedModelCallback> callback,
@@ -319,7 +345,8 @@
float fpAtol = 1e-5f, fpRtol = 5.0f * 1.1920928955078125e-7f;
EvaluatePreparedModel(preparedModel, is_ignored, examples,
- /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol);
+ /*hasRelaxedFloat32Model=*/false, fpAtol, fpRtol, Synchronously::NO,
+ MeasureTiming::NO, /*testDynamicOutputShape=*/false);
}
void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
@@ -365,12 +392,14 @@
ASSERT_NE(nullptr, preparedModel.get());
EvaluatePreparedModel(preparedModel, is_ignored, examples,
- model.relaxComputationFloat32toFloat16);
+ model.relaxComputationFloat32toFloat16, 1e-5f, 1e-5f, Synchronously::NO,
+ MeasureTiming::NO, /*testDynamicOutputShape=*/false);
}
// TODO: Reduce code duplication.
void Execute(const sp<V1_2::IDevice>& device, std::function<V1_2::Model(void)> create_model,
- std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
+ std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples,
+ bool testDynamicOutputShape) {
V1_2::Model model = create_model();
// see if service can handle model
@@ -412,9 +441,17 @@
ASSERT_NE(nullptr, preparedModel.get());
EvaluatePreparedModel(preparedModel, is_ignored, examples,
- model.relaxComputationFloat32toFloat16, Synchronously::NO);
+ model.relaxComputationFloat32toFloat16, Synchronously::NO,
+ MeasureTiming::NO, testDynamicOutputShape);
EvaluatePreparedModel(preparedModel, is_ignored, examples,
- model.relaxComputationFloat32toFloat16, Synchronously::YES);
+ model.relaxComputationFloat32toFloat16, Synchronously::YES,
+ MeasureTiming::NO, testDynamicOutputShape);
+ EvaluatePreparedModel(preparedModel, is_ignored, examples,
+ model.relaxComputationFloat32toFloat16, Synchronously::NO,
+ MeasureTiming::YES, testDynamicOutputShape);
+ EvaluatePreparedModel(preparedModel, is_ignored, examples,
+ model.relaxComputationFloat32toFloat16, Synchronously::YES,
+ MeasureTiming::YES, testDynamicOutputShape);
}
} // namespace generated_tests