Modify 1.1 VTS tests to consume test struct directly. am: 0c03b85005
am: 37f937b7ce
Change-Id: I9df06729490dd2c5669062e740354dadff43b165
diff --git a/neuralnetworks/1.1/vts/functional/Android.bp b/neuralnetworks/1.1/vts/functional/Android.bp
index 1b31008..86002d2 100644
--- a/neuralnetworks/1.1/vts/functional/Android.bp
+++ b/neuralnetworks/1.1/vts/functional/Android.bp
@@ -34,13 +34,12 @@
"android.hidl.memory@1.0",
"libgmock",
"libhidlmemory",
+ "libneuralnetworks_generated_test_harness",
"libneuralnetworks_utils",
"VtsHalNeuralNetworksV1_0_utils",
],
header_libs: [
"libneuralnetworks_headers",
- "libneuralnetworks_generated_test_harness_headers",
- "libneuralnetworks_generated_tests",
],
test_suites: ["general-tests"],
}
diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp
index e7d59ec..73eeb93 100644
--- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp
@@ -24,6 +24,7 @@
#include <android/hidl/memory/1.0/IMemory.h>
#include <hidlmemory/mapping.h>
+#include <gtest/gtest.h>
#include <iostream>
#include "1.0/Callbacks.h"
@@ -37,8 +38,13 @@
namespace V1_1 {
namespace generated_tests {
+using namespace test_helper;
+using ::android::hardware::neuralnetworks::V1_0::DataLocation;
using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
using ::android::hardware::neuralnetworks::V1_0::IPreparedModel;
+using ::android::hardware::neuralnetworks::V1_0::Operand;
+using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime;
+using ::android::hardware::neuralnetworks::V1_0::OperandType;
using ::android::hardware::neuralnetworks::V1_0::Request;
using ::android::hardware::neuralnetworks::V1_0::RequestArgument;
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
@@ -47,144 +53,112 @@
using ::android::hardware::neuralnetworks::V1_1::IDevice;
using ::android::hardware::neuralnetworks::V1_1::Model;
using ::android::hidl::memory::V1_0::IMemory;
-using ::test_helper::compare;
-using ::test_helper::filter;
-using ::test_helper::for_all;
-using ::test_helper::MixedTyped;
-using ::test_helper::MixedTypedExample;
-using ::test_helper::resize_accordingly;
+
+Model createModel(const TestModel& testModel) {
+ // Model operands.
+ hidl_vec<Operand> operands(testModel.operands.size());
+ size_t constCopySize = 0, constRefSize = 0;
+ for (uint32_t i = 0; i < testModel.operands.size(); i++) {
+ const auto& op = testModel.operands[i];
+
+ DataLocation loc = {};
+ if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
+ loc = {.poolIndex = 0,
+ .offset = static_cast<uint32_t>(constCopySize),
+ .length = static_cast<uint32_t>(op.data.size())};
+ constCopySize += op.data.alignedSize();
+ } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
+ loc = {.poolIndex = 0,
+ .offset = static_cast<uint32_t>(constRefSize),
+ .length = static_cast<uint32_t>(op.data.size())};
+ constRefSize += op.data.alignedSize();
+ }
+
+ operands[i] = {.type = static_cast<OperandType>(op.type),
+ .dimensions = op.dimensions,
+ .numberOfConsumers = op.numberOfConsumers,
+ .scale = op.scale,
+ .zeroPoint = op.zeroPoint,
+ .lifetime = static_cast<OperandLifeTime>(op.lifetime),
+ .location = loc};
+ }
+
+ // Model operations.
+ hidl_vec<Operation> operations(testModel.operations.size());
+ std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),
+ [](const TestOperation& op) -> Operation {
+ return {.type = static_cast<OperationType>(op.type),
+ .inputs = op.inputs,
+ .outputs = op.outputs};
+ });
+
+ // Constant copies.
+ hidl_vec<uint8_t> operandValues(constCopySize);
+ for (uint32_t i = 0; i < testModel.operands.size(); i++) {
+ const auto& op = testModel.operands[i];
+ if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
+ const uint8_t* begin = op.data.get<uint8_t>();
+ const uint8_t* end = begin + op.data.size();
+ std::copy(begin, end, operandValues.data() + operands[i].location.offset);
+ }
+ }
+
+ // Shared memory.
+ hidl_vec<hidl_memory> pools;
+ if (constRefSize > 0) {
+ hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
+ CHECK_NE(pools[0].size(), 0u);
+
+ // load data
+ sp<IMemory> mappedMemory = mapMemory(pools[0]);
+ CHECK(mappedMemory.get() != nullptr);
+ uint8_t* mappedPtr =
+ reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));
+ CHECK(mappedPtr != nullptr);
+
+ for (uint32_t i = 0; i < testModel.operands.size(); i++) {
+ const auto& op = testModel.operands[i];
+ if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
+ const uint8_t* begin = op.data.get<uint8_t>();
+ const uint8_t* end = begin + op.data.size();
+ std::copy(begin, end, mappedPtr + operands[i].location.offset);
+ }
+ }
+ }
+
+ return {.operands = std::move(operands),
+ .operations = std::move(operations),
+ .inputIndexes = testModel.inputIndexes,
+ .outputIndexes = testModel.outputIndexes,
+ .operandValues = std::move(operandValues),
+ .pools = std::move(pools),
+ .relaxComputationFloat32toFloat16 = testModel.isRelaxed};
+}
// Top level driver for models and examples generated by test_generator.py
// Test driver for those generated from ml/nn/runtime/test/spec
-void EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored,
- const std::vector<MixedTypedExample>& examples,
- bool hasRelaxedFloat32Model, float fpAtol, float fpRtol) {
- const uint32_t INPUT = 0;
- const uint32_t OUTPUT = 1;
+void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel) {
+ const Request request = createRequest(testModel);
- int example_no = 1;
- for (auto& example : examples) {
- SCOPED_TRACE(example_no++);
- const MixedTyped& inputs = example.operands.first;
- const MixedTyped& golden = example.operands.second;
+ // Launch execution.
+ sp<ExecutionCallback> executionCallback = new ExecutionCallback();
+ Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(request, executionCallback);
+ ASSERT_TRUE(executionLaunchStatus.isOk());
+ EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
- const bool hasFloat16Inputs = !inputs.float16Operands.empty();
- if (hasRelaxedFloat32Model || hasFloat16Inputs) {
- // TODO: Adjust the error limit based on testing.
- // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
- fpAtol = 5.0f * 0.0009765625f;
- // Set the relative tolerance to be 5ULP of the corresponding FP precision.
- fpRtol = 5.0f * 0.0009765625f;
- }
+ // Retrieve execution status.
+ executionCallback->wait();
+ ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus());
- std::vector<RequestArgument> inputs_info, outputs_info;
- uint32_t inputSize = 0, outputSize = 0;
- // This function only partially specifies the metadata (vector of RequestArguments).
- // The contents are copied over below.
- for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
- if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
- RequestArgument arg = {
- .location = {.poolIndex = INPUT,
- .offset = 0,
- .length = static_cast<uint32_t>(s)},
- .dimensions = {},
- };
- RequestArgument arg_empty = {
- .hasNoValue = true,
- };
- inputs_info[index] = s ? arg : arg_empty;
- inputSize += s;
- });
- // Compute offset for inputs 1 and so on
- {
- size_t offset = 0;
- for (auto& i : inputs_info) {
- if (!i.hasNoValue) i.location.offset = offset;
- offset += i.location.length;
- }
- }
+ // Retrieve execution results.
+ const std::vector<TestBuffer> outputs = getOutputBuffers(request);
- MixedTyped test; // holding test results
-
- // Go through all outputs, initialize RequestArgument descriptors
- resize_accordingly(golden, test);
- for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
- if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
- RequestArgument arg = {
- .location = {.poolIndex = OUTPUT,
- .offset = 0,
- .length = static_cast<uint32_t>(s)},
- .dimensions = {},
- };
- outputs_info[index] = arg;
- outputSize += s;
- });
- // Compute offset for outputs 1 and so on
- {
- size_t offset = 0;
- for (auto& i : outputs_info) {
- i.location.offset = offset;
- offset += i.location.length;
- }
- }
- std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
- nn::allocateSharedMemory(outputSize)};
- ASSERT_NE(0ull, pools[INPUT].size());
- ASSERT_NE(0ull, pools[OUTPUT].size());
-
- // load data
- sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
- sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
- ASSERT_NE(nullptr, inputMemory.get());
- ASSERT_NE(nullptr, outputMemory.get());
- char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
- char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));
- ASSERT_NE(nullptr, inputPtr);
- ASSERT_NE(nullptr, outputPtr);
- inputMemory->update();
- outputMemory->update();
-
- // Go through all inputs, copy the values
- for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
- char* begin = (char*)p;
- char* end = begin + s;
- // TODO: handle more than one input
- std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
- });
-
- inputMemory->commit();
- outputMemory->commit();
-
- const Request request = {.inputs = inputs_info, .outputs = outputs_info, .pools = pools};
-
- // launch execution
- sp<ExecutionCallback> executionCallback = new ExecutionCallback();
- ASSERT_NE(nullptr, executionCallback.get());
- Return<ErrorStatus> executionLaunchStatus =
- preparedModel->execute(request, executionCallback);
- ASSERT_TRUE(executionLaunchStatus.isOk());
- EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
-
- // retrieve execution status
- executionCallback->wait();
- ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus());
-
- // validate results
- outputMemory->read();
- copy_back(&test, outputs_info, outputPtr);
- outputMemory->commit();
- // Filter out don't cares
- MixedTyped filtered_golden = filter(golden, is_ignored);
- MixedTyped filtered_test = filter(test, is_ignored);
-
- // We want "close-enough" results for float
- compare(filtered_golden, filtered_test, fpAtol, fpRtol);
- }
+ // We want "close-enough" results.
+ checkResults(testModel, outputs);
}
-void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
- std::function<bool(int)> is_ignored, const std::vector<MixedTypedExample>& examples) {
- Model model = create_model();
+void Execute(const sp<IDevice>& device, const TestModel& testModel) {
+ Model model = createModel(testModel);
// see if service can handle model
bool fullySupportsModel = false;
@@ -199,7 +173,6 @@
// launch prepare model
sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
- ASSERT_NE(nullptr, preparedModelCallback.get());
Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
ASSERT_TRUE(prepareLaunchStatus.isOk());
@@ -223,8 +196,7 @@
EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
ASSERT_NE(nullptr, preparedModel.get());
- EvaluatePreparedModel(preparedModel, is_ignored, examples,
- model.relaxComputationFloat32toFloat16, 1e-5f, 1e-5f);
+ EvaluatePreparedModel(preparedModel, testModel);
}
} // namespace generated_tests
diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h
index 64b88dd..56fc825 100644
--- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h
+++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h
@@ -18,9 +18,6 @@
#define ANDROID_HARDWARE_NEURALNETWORKS_V1_1_GENERATED_TEST_HARNESS_H
#include <android/hardware/neuralnetworks/1.1/IDevice.h>
-#include <android/hardware/neuralnetworks/1.1/types.h>
-#include <functional>
-#include <vector>
#include "TestHarness.h"
namespace android {
@@ -29,9 +26,9 @@
namespace V1_1 {
namespace generated_tests {
-void Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model,
- std::function<bool(int)> is_ignored,
- const std::vector<::test_helper::MixedTypedExample>& examples);
+Model createModel(const ::test_helper::TestModel& testModel);
+
+void Execute(const sp<V1_1::IDevice>& device, const ::test_helper::TestModel& testModel);
} // namespace generated_tests
} // namespace V1_1
diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTests.h b/neuralnetworks/1.1/vts/functional/GeneratedTests.h
index 80442bf..a55213d 100644
--- a/neuralnetworks/1.1/vts/functional/GeneratedTests.h
+++ b/neuralnetworks/1.1/vts/functional/GeneratedTests.h
@@ -14,20 +14,11 @@
* limitations under the License.
*/
-#include <android/hidl/memory/1.0/IMemory.h>
-#include <hidlmemory/mapping.h>
-
+#include "1.0/Utils.h"
#include "GeneratedTestHarness.h"
-#include "MemoryUtils.h"
#include "TestHarness.h"
#include "VtsHalNeuralnetworks.h"
-namespace android::hardware::neuralnetworks::V1_1::vts::functional {
-
-std::vector<Request> createRequests(const std::vector<::test_helper::MixedTypedExample>& examples);
-
-} // namespace android::hardware::neuralnetworks::V1_1::vts::functional
-
namespace android::hardware::neuralnetworks::V1_1::generated_tests {
using namespace android::hardware::neuralnetworks::V1_1::vts::functional;
diff --git a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
index c549728..757bee9 100644
--- a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
+++ b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
@@ -16,14 +16,8 @@
#define LOG_TAG "neuralnetworks_hidl_hal_test"
-#include <android-base/logging.h>
-#include <android/hidl/memory/1.0/IMemory.h>
-#include <hidlmemory/mapping.h>
-
#include "1.0/Callbacks.h"
#include "1.0/Utils.h"
-#include "MemoryUtils.h"
-#include "TestHarness.h"
#include "VtsHalNeuralnetworks.h"
namespace android {
@@ -35,13 +29,8 @@
using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
using ::android::hardware::neuralnetworks::V1_0::Request;
-using ::android::hardware::neuralnetworks::V1_0::RequestArgument;
using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
using ::android::hardware::neuralnetworks::V1_1::IPreparedModel;
-using ::android::hidl::memory::V1_0::IMemory;
-using ::test_helper::for_all;
-using ::test_helper::MixedTyped;
-using ::test_helper::MixedTypedExample;
///////////////////////// UTILITY FUNCTIONS /////////////////////////
@@ -87,103 +76,10 @@
///////////////////////////// ENTRY POINT //////////////////////////////////
-std::vector<Request> createRequests(const std::vector<MixedTypedExample>& examples) {
- const uint32_t INPUT = 0;
- const uint32_t OUTPUT = 1;
-
- std::vector<Request> requests;
-
- for (auto& example : examples) {
- const MixedTyped& inputs = example.operands.first;
- const MixedTyped& outputs = example.operands.second;
-
- std::vector<RequestArgument> inputs_info, outputs_info;
- uint32_t inputSize = 0, outputSize = 0;
-
- // This function only partially specifies the metadata (vector of RequestArguments).
- // The contents are copied over below.
- for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
- if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
- RequestArgument arg = {
- .location = {.poolIndex = INPUT,
- .offset = 0,
- .length = static_cast<uint32_t>(s)},
- .dimensions = {},
- };
- RequestArgument arg_empty = {
- .hasNoValue = true,
- };
- inputs_info[index] = s ? arg : arg_empty;
- inputSize += s;
- });
- // Compute offset for inputs 1 and so on
- {
- size_t offset = 0;
- for (auto& i : inputs_info) {
- if (!i.hasNoValue) i.location.offset = offset;
- offset += i.location.length;
- }
- }
-
- // Go through all outputs, initialize RequestArgument descriptors
- for_all(outputs, [&outputs_info, &outputSize](int index, auto, auto s) {
- if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
- RequestArgument arg = {
- .location = {.poolIndex = OUTPUT,
- .offset = 0,
- .length = static_cast<uint32_t>(s)},
- .dimensions = {},
- };
- outputs_info[index] = arg;
- outputSize += s;
- });
- // Compute offset for outputs 1 and so on
- {
- size_t offset = 0;
- for (auto& i : outputs_info) {
- i.location.offset = offset;
- offset += i.location.length;
- }
- }
- std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize),
- nn::allocateSharedMemory(outputSize)};
- if (pools[INPUT].size() == 0 || pools[OUTPUT].size() == 0) {
- return {};
- }
-
- // map pool
- sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
- if (inputMemory == nullptr) {
- return {};
- }
- char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
- if (inputPtr == nullptr) {
- return {};
- }
-
- // initialize pool
- inputMemory->update();
- for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
- char* begin = (char*)p;
- char* end = begin + s;
- // TODO: handle more than one input
- std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
- });
- inputMemory->commit();
-
- requests.push_back({.inputs = inputs_info, .outputs = outputs_info, .pools = pools});
- }
-
- return requests;
-}
-
-void ValidationTest::validateRequests(const sp<IPreparedModel>& preparedModel,
- const std::vector<Request>& requests) {
- // validate each request
- for (const Request& request : requests) {
- removeInputTest(preparedModel, request);
- removeOutputTest(preparedModel, request);
- }
+void ValidationTest::validateRequest(const sp<IPreparedModel>& preparedModel,
+ const Request& request) {
+ removeInputTest(preparedModel, request);
+ removeOutputTest(preparedModel, request);
}
} // namespace functional
diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp
index 12bdd3f..b3b15fa 100644
--- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp
+++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp
@@ -122,7 +122,7 @@
::testing::VtsHalHidlTargetTestBase::TearDown();
}
-void ValidationTest::validateEverything(const Model& model, const std::vector<Request>& requests) {
+void ValidationTest::validateEverything(const Model& model, const Request& request) {
validateModel(model);
// create IPreparedModel
@@ -132,7 +132,7 @@
return;
}
- validateRequests(preparedModel, requests);
+ validateRequest(preparedModel, request);
}
} // namespace functional
diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h
index 3156784..2d6a20c 100644
--- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h
+++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h
@@ -72,12 +72,11 @@
// Tag for the validation tests
class ValidationTest : public NeuralnetworksHidlTest {
protected:
- void validateEverything(const Model& model, const std::vector<Request>& request);
+ void validateEverything(const Model& model, const Request& request);
private:
void validateModel(const Model& model);
- void validateRequests(const sp<IPreparedModel>& preparedModel,
- const std::vector<Request>& requests);
+ void validateRequest(const sp<IPreparedModel>& preparedModel, const Request& request);
};
// Tag for the generated tests