Modify 1.1 VTS tests to consume test struct directly.
This CL is very similar to the 1.0 VTS CL. The only difference is that
1.1 Models have an additional field for relaxed model computation.
Bug: 123092187
Bug: 138718240
Test: All VTS
Change-Id: I9264e5b2468b9c6db47d86683d24f8c2c5ec46aa
Merged-In: I9264e5b2468b9c6db47d86683d24f8c2c5ec46aa
(cherry picked from commit 6aad0402780ba007ca027bf40707c0dc14c5f5fc)
diff --git a/neuralnetworks/1.1/vts/functional/Android.bp b/neuralnetworks/1.1/vts/functional/Android.bp
index 1b31008..86002d2 100644
--- a/neuralnetworks/1.1/vts/functional/Android.bp
+++ b/neuralnetworks/1.1/vts/functional/Android.bp
@@ -34,13 +34,12 @@
"android.hidl.memory@1.0",
"libgmock",
"libhidlmemory",
+ "libneuralnetworks_generated_test_harness",
"libneuralnetworks_utils",
"VtsHalNeuralNetworksV1_0_utils",
],
header_libs: [
"libneuralnetworks_headers",
- "libneuralnetworks_generated_test_harness_headers",
- "libneuralnetworks_generated_tests",
],
test_suites: ["general-tests"],
}
diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp
index e7d59ec..73eeb93 100644
--- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp
@@ -24,6 +24,7 @@
#include <android/hidl/memory/1.0/IMemory.h>
#include <hidlmemory/mapping.h>
+#include <gtest/gtest.h>
#include <iostream>
#include "1.0/Callbacks.h"
@@ -37,8 +38,13 @@
namespace V1_1 {
namespace generated_tests {
+using namespace test_helper;
+using ::android::hardware::neuralnetworks::V1_0::DataLocation;
using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
using ::android::hardware::neuralnetworks::V1_0::IPreparedModel;
+using ::android::hardware::neuralnetworks::V1_0::Operand;
+using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime;
+using ::android::hardware::neuralnetworks::V1_0::OperandType;
using ::android::hardware::neuralnetworks::V1_0::Request;
using ::android::hardware::neuralnetworks::V1_0::RequestArgument;
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
@@ -47,144 +53,112 @@
using ::android::hardware::neuralnetworks::V1_1::IDevice;
using ::android::hardware::neuralnetworks::V1_1::Model;
using ::android::hidl::memory::V1_0::IMemory;
-using ::test_helper::compare;
-using ::test_helper::filter;
-using ::test_helper::for_all;
-using ::test_helper::MixedTyped;
-using ::test_helper::MixedTypedExample;
-using ::test_helper::resize_accordingly;
+
+Model createModel(const TestModel& testModel) {
+ // Model operands.
+ hidl_vec<Operand> operands(testModel.operands.size());
+ size_t constCopySize = 0, constRefSize = 0;
+ for (uint32_t i = 0; i < testModel.operands.size(); i++) {
+ const auto& op = testModel.operands[i];
+
+ DataLocation loc = {};
+ if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
+ loc = {.poolIndex = 0,
+ .offset = static_cast<uint32_t>(constCopySize),
+ .length = static_cast<uint32_t>(op.data.size())};
+ constCopySize += op.data.alignedSize();
+ } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
+ loc = {.poolIndex = 0,
+ .offset = static_cast<uint32_t>(constRefSize),
+ .length = static_cast<uint32_t>(op.data.size())};
+ constRefSize += op.data.alignedSize();
+ }
+
+ operands[i] = {.type = static_cast<OperandType>(op.type),
+ .dimensions = op.dimensions,
+ .numberOfConsumers = op.numberOfConsumers,
+ .scale = op.scale,
+ .zeroPoint = op.zeroPoint,
+ .lifetime = static_cast<OperandLifeTime>(op.lifetime),
+ .location = loc};
+ }
+
+ // Model operations.
+ hidl_vec<Operation> operations(testModel.operations.size());
+ std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),
+ [](const TestOperation& op) -> Operation {
+ return {.type = static_cast<OperationType>(op.type),
+ .inputs = op.inputs,
+ .outputs = op.outputs};
+ });
+
+ // Constant copies.
+ hidl_vec<uint8_t> operandValues(constCopySize);
+ for (uint32_t i = 0; i < testModel.operands.size(); i++) {
+ const auto& op = testModel.operands[i];
+ if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
+ const uint8_t* begin = op.data.get<uint8_t>();
+ const uint8_t* end = begin + op.data.size();
+ std::copy(begin, end, operandValues.data() + operands[i].location.offset);
+ }
+ }
+
+ // Shared memory.
+ hidl_vec<hidl_memory> pools;
+ if (constRefSize > 0) {
+ hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
+ CHECK_NE(pools[0].size(), 0u);
+
+ // load data
+ sp<IMemory> mappedMemory = mapMemory(pools[0]);
+ CHECK(mappedMemory.get() != nullptr);
+ uint8_t* mappedPtr =
+ reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));
+ CHECK(mappedPtr != nullptr);
+
+ for (uint32_t i = 0; i < testModel.operands.size(); i++) {
+ const auto& op = testModel.operands[i];
+ if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
+ const uint8_t* begin = op.data.get<uint8_t>();
+ const uint8_t* end = begin + op.data.size();
+ std::copy(begin, end, mappedPtr + operands[i].location.offset);
+ }
+ }
+ }
+
+ return {.operands = std::move(operands),
+ .operations = std::move(operations),
+ .inputIndexes = testModel.inputIndexes,
+ .outputIndexes = testModel.outputIndexes,
+ .operandValues = std::move(operandValues),
+ .pools = std::move(pools),
+ .relaxComputationFloat32toFloat16 = testModel.isRelaxed};
+}
// Top level driver for models and examples generated by test_generator.py
// Test driver for those generated from ml/nn/runtime/test/spec
-void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
- const std::vector<MixedTypedExample>& examples,
- bool hasRelaxedFloat32Model, float fpAtol, float fpRtol) {
- const uint32_t INPUT = 0;
- const uint32_t OUTPUT = 1;
+void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel) {
+ const Request request = createRequest(testModel);
- int example_no = 1;
- for (auto& example : examples) {
- SCOPED_TRACE(example_no++);
- const MixedTyped& inputs = example.operands.first;
- const MixedTyped& golden = example.operands.second;
+ // Launch execution.
+ sp<ExecutionCallback> executionCallback = new ExecutionCallback();
+ Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(request, executionCallback);
+ ASSERT_TRUE(executionLaunchStatus.isOk());
+ EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
- const bool hasFloat16Inputs = !inputs.float16Operands.empty();
- if (hasRelaxedFloat32Model || hasFloat16Inputs) {
- // TODO: Adjust the error limit based on testing.
- // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
- fpAtol = 5.0f * 0.0009765625f;
- // Set the relative tolerance to be 5ULP of the corresponding FP precision.
- fpRtol = 5.0f * 0.0009765625f;
- }
+ // Retrieve execution status.
+ executionCallback->wait();
+ ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus());
- std::vector<RequestArgument> inputs_info, outputs_info;
- uint32_t inputSize = 0, outputSize = 0;
- // This function only partially specifies the metadata (vector of RequestArguments).
- // The contents are copied over below.
- for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
- if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
- RequestArgument arg = {
- .location = {.poolIndex = INPUT,
- .offset = 0,
- .length = static_cast<uint32_t>(s)},
- .dimensions = {},
- };
- RequestArgument arg_empty = {
- .hasNoValue = true,
- };
- inputs_info[index] = s ? arg : arg_empty;
- inputSize += s;
- });
- // Compute offset for inputs 1 and so on
- {
- size_t offset = 0;
- for (auto& i : inputs_info) {
- if (!i.hasNoValue) i.location.offset = offset;
- offset += i.location.length;
- }
- }
+ // Retrieve execution results.
+ const std::vector<TestBuffer> outputs = getOutputBuffers(request);
- MixedTyped test; // holding test results
-
- // Go through all outputs, initialize RequestArgument descriptors
- resize_accordingly(golden, test);
- for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
- if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
- RequestArgument arg = {
- .location = {.poolIndex = OUTPUT,
- .offset = 0,
- .length = static_cast<uint32_t>(s)},
- .dimensions = {},
- };
- outputs_info[index] = arg;
- outputSize += s;
- });
- // Compute offset for outputs 1 and so on
- {
- size_t offset = 0;
- for (auto& i : outputs_info) {
- i.location.offset = offset;
- offset += i.location.length;
- }
- }
- std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
- nn::allocateSharedMemory(outputSize)};
- ASSERT_NE(0ull, pools[INPUT].size());
- ASSERT_NE(0ull, pools[OUTPUT].size());
-
- // load data
- sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
- sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
- ASSERT_NE(nullptr, inputMemory.get());
- ASSERT_NE(nullptr, outputMemory.get());
- char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
- char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));
- ASSERT_NE(nullptr, inputPtr);
- ASSERT_NE(nullptr, outputPtr);
- inputMemory->update();
- outputMemory->update();
-
- // Go through all inputs, copy the values
- for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
- char* begin = (char*)p;
- char* end = begin + s;
- // TODO: handle more than one input
- std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
- });
-
- inputMemory->commit();
- outputMemory->commit();
-
- const Request request = {.inputs = inputs_info, .outputs = outputs_info, .pools = pools};
-
- // launch execution
- sp<ExecutionCallback> executionCallback = new ExecutionCallback();
- ASSERT_NE(nullptr, executionCallback.get());
- Return<ErrorStatus> executionLaunchStatus =
- preparedModel->execute(request, executionCallback);
- ASSERT_TRUE(executionLaunchStatus.isOk());
- EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
-
- // retrieve execution status
- executionCallback->wait();
- ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus());
-
- // validate results
- outputMemory->read();
- copy_back(&test, outputs_info, outputPtr);
- outputMemory->commit();
- // Filter out don't cares
- MixedTyped filtered_golden = filter(golden, is_ignored);
- MixedTyped filtered_test = filter(test, is_ignored);
-
- // We want "close-enough" results for float
- compare(filtered_golden, filtered_test, fpAtol, fpRtol);
- }
+ // We want "close-enough" results.
+ checkResults(testModel, outputs);
}
-void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
- std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
- Model model = create_model();
+void Execute(const sp<IDevice>& device, const TestModel& testModel) {
+ Model model = createModel(testModel);
// see if service can handle model
bool fullySupportsModel = false;
@@ -199,7 +173,6 @@
// launch prepare model
sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
- ASSERT_NE(nullptr, preparedModelCallback.get());
Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
ASSERT_TRUE(prepareLaunchStatus.isOk());
@@ -223,8 +196,7 @@
EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
ASSERT_NE(nullptr, preparedModel.get());
- EvaluatePreparedModel(preparedModel, is_ignored, examples,
- model.relaxComputationFloat32toFloat16, 1e-5f, 1e-5f);
+ EvaluatePreparedModel(preparedModel, testModel);
}
} // namespace generated_tests
diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h
index 64b88dd..56fc825 100644
--- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h
+++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h
@@ -18,9 +18,6 @@
#define ANDROID_HARDWARE_NEURALNETWORKS_V1_1_GENERATED_TEST_HARNESS_H
#include <android/hardware/neuralnetworks/1.1/IDevice.h>
-#include <android/hardware/neuralnetworks/1.1/types.h>
-#include <functional>
-#include <vector>
#include "TestHarness.h"
namespace android {
@@ -29,9 +26,9 @@
namespace V1_1 {
namespace generated_tests {
-void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
- std::function<bool(int)> is_ignored,
- const std::vector<::test_helper::MixedTypedExample>& examples);
+Model createModel(const ::test_helper::TestModel& testModel);
+
+void Execute(const sp<V1_1::IDevice>& device, const ::test_helper::TestModel& testModel);
} // namespace generated_tests
} // namespace V1_1
diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTests.h b/neuralnetworks/1.1/vts/functional/GeneratedTests.h
index 80442bf..a55213d 100644
--- a/neuralnetworks/1.1/vts/functional/GeneratedTests.h
+++ b/neuralnetworks/1.1/vts/functional/GeneratedTests.h
@@ -14,20 +14,11 @@
* limitations under the License.
*/
-#include <android/hidl/memory/1.0/IMemory.h>
-#include <hidlmemory/mapping.h>
-
+#include "1.0/Utils.h"
#include "GeneratedTestHarness.h"
-#include "MemoryUtils.h"
#include "TestHarness.h"
#include "VtsHalNeuralnetworks.h"
-namespace android::hardware::neuralnetworks::V1_1::vts::functional {
-
-std::vector<Request> createRequests(const std::vector<::test_helper::MixedTypedExample>& examples);
-
-} // namespace android::hardware::neuralnetworks::V1_1::vts::functional
-
namespace android::hardware::neuralnetworks::V1_1::generated_tests {
using namespace android::hardware::neuralnetworks::V1_1::vts::functional;
diff --git a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
index c549728..757bee9 100644
--- a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
+++ b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
@@ -16,14 +16,8 @@
#define LOG_TAG "neuralnetworks_hidl_hal_test"
-#include <android-base/logging.h>
-#include <android/hidl/memory/1.0/IMemory.h>
-#include <hidlmemory/mapping.h>
-
#include "1.0/Callbacks.h"
#include "1.0/Utils.h"
-#include "MemoryUtils.h"
-#include "TestHarness.h"
#include "VtsHalNeuralnetworks.h"
namespace android {
@@ -35,13 +29,8 @@
using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
using ::android::hardware::neuralnetworks::V1_0::Request;
-using ::android::hardware::neuralnetworks::V1_0::RequestArgument;
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
using ::android::hardware::neuralnetworks::V1_1::IPreparedModel;
-using ::android::hidl::memory::V1_0::IMemory;
-using ::test_helper::for_all;
-using ::test_helper::MixedTyped;
-using ::test_helper::MixedTypedExample;
///////////////////////// UTILITY FUNCTIONS /////////////////////////
@@ -87,103 +76,10 @@
///////////////////////////// ENTRY POINT //////////////////////////////////
-std::vector<Request> createRequests(const std::vector<MixedTypedExample>& examples) {
- const uint32_t INPUT = 0;
- const uint32_t OUTPUT = 1;
-
- std::vector<Request> requests;
-
- for (auto& example : examples) {
- const MixedTyped& inputs = example.operands.first;
- const MixedTyped& outputs = example.operands.second;
-
- std::vector<RequestArgument> inputs_info, outputs_info;
- uint32_t inputSize = 0, outputSize = 0;
-
- // This function only partially specifies the metadata (vector of RequestArguments).
- // The contents are copied over below.
- for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
- if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
- RequestArgument arg = {
- .location = {.poolIndex = INPUT,
- .offset = 0,
- .length = static_cast<uint32_t>(s)},
- .dimensions = {},
- };
- RequestArgument arg_empty = {
- .hasNoValue = true,
- };
- inputs_info[index] = s ? arg : arg_empty;
- inputSize += s;
- });
- // Compute offset for inputs 1 and so on
- {
- size_t offset = 0;
- for (auto& i : inputs_info) {
- if (!i.hasNoValue) i.location.offset = offset;
- offset += i.location.length;
- }
- }
-
- // Go through all outputs, initialize RequestArgument descriptors
- for_all(outputs, [&outputs_info, &outputSize](int index, auto, auto s) {
- if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
- RequestArgument arg = {
- .location = {.poolIndex = OUTPUT,
- .offset = 0,
- .length = static_cast<uint32_t>(s)},
- .dimensions = {},
- };
- outputs_info[index] = arg;
- outputSize += s;
- });
- // Compute offset for outputs 1 and so on
- {
- size_t offset = 0;
- for (auto& i : outputs_info) {
- i.location.offset = offset;
- offset += i.location.length;
- }
- }
- std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
- nn::allocateSharedMemory(outputSize)};
- if (pools[INPUT].size() == 0 || pools[OUTPUT].size() == 0) {
- return {};
- }
-
- // map pool
- sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
- if (inputMemory == nullptr) {
- return {};
- }
- char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
- if (inputPtr == nullptr) {
- return {};
- }
-
- // initialize pool
- inputMemory->update();
- for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
- char* begin = (char*)p;
- char* end = begin + s;
- // TODO: handle more than one input
- std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
- });
- inputMemory->commit();
-
- requests.push_back({.inputs = inputs_info, .outputs = outputs_info, .pools = pools});
- }
-
- return requests;
-}
-
-void ValidationTest::validateRequests(const sp<IPreparedModel>& preparedModel,
- const std::vector<Request>& requests) {
- // validate each request
- for (const Request& request : requests) {
- removeInputTest(preparedModel, request);
- removeOutputTest(preparedModel, request);
- }
+void ValidationTest::validateRequest(const sp<IPreparedModel>& preparedModel,
+ const Request& request) {
+ removeInputTest(preparedModel, request);
+ removeOutputTest(preparedModel, request);
}
} // namespace functional
diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp
index 12bdd3f..b3b15fa 100644
--- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp
+++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp
@@ -122,7 +122,7 @@
::testing::VtsHalHidlTargetTestBase::TearDown();
}
-void ValidationTest::validateEverything(const Model& model, const std::vector<Request>& requests) {
+void ValidationTest::validateEverything(const Model& model, const Request& request) {
validateModel(model);
// create IPreparedModel
@@ -132,7 +132,7 @@
return;
}
- validateRequests(preparedModel, requests);
+ validateRequest(preparedModel, request);
}
} // namespace functional
diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h
index 3156784..2d6a20c 100644
--- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h
+++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h
@@ -72,12 +72,11 @@
// Tag for the validation tests
class ValidationTest : public NeuralnetworksHidlTest {
protected:
- void validateEverything(const Model& model, const std::vector<Request>& request);
+ void validateEverything(const Model& model, const Request& request);
private:
void validateModel(const Model& model);
- void validateRequests(const sp<IPreparedModel>& preparedModel,
- const std::vector<Request>& requests);
+ void validateRequest(const sp<IPreparedModel>& preparedModel, const Request& request);
};
// Tag for the generated tests