blob: d8d1a31ac443c018b58cb92db50987e7e584a306 [file] [log] [blame]
Slava Shklyaev1d6b4652019-05-14 14:15:14 +01001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "GeneratedTestHarness.h"
18
19#include <android-base/logging.h>
20#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
21#include <android/hardware/neuralnetworks/1.0/types.h>
22#include <android/hardware/neuralnetworks/1.1/IDevice.h>
23#include <android/hidl/allocator/1.0/IAllocator.h>
24#include <android/hidl/memory/1.0/IMemory.h>
25#include <hidlmemory/mapping.h>
26
Xusong Wang6aad0402019-08-09 16:41:16 -070027#include <gtest/gtest.h>
Slava Shklyaev1d6b4652019-05-14 14:15:14 +010028#include <iostream>
29
30#include "1.0/Callbacks.h"
31#include "1.0/Utils.h"
32#include "MemoryUtils.h"
33#include "TestHarness.h"
Xusong Wang9e2b97b2019-08-23 16:10:54 -070034#include "VtsHalNeuralnetworks.h"
Slava Shklyaev1d6b4652019-05-14 14:15:14 +010035
36namespace android {
37namespace hardware {
38namespace neuralnetworks {
Slava Shklyaev2bcfdc82019-07-17 15:50:57 +010039namespace V1_1 {
Xusong Wang9e2b97b2019-08-23 16:10:54 -070040namespace vts {
41namespace functional {
Slava Shklyaev1d6b4652019-05-14 14:15:14 +010042
Xusong Wang6aad0402019-08-09 16:41:16 -070043using namespace test_helper;
44using ::android::hardware::neuralnetworks::V1_0::DataLocation;
Slava Shklyaev1d6b4652019-05-14 14:15:14 +010045using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
46using ::android::hardware::neuralnetworks::V1_0::IPreparedModel;
Xusong Wang6aad0402019-08-09 16:41:16 -070047using ::android::hardware::neuralnetworks::V1_0::Operand;
48using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime;
49using ::android::hardware::neuralnetworks::V1_0::OperandType;
Slava Shklyaev1d6b4652019-05-14 14:15:14 +010050using ::android::hardware::neuralnetworks::V1_0::Request;
51using ::android::hardware::neuralnetworks::V1_0::RequestArgument;
52using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
53using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
54using ::android::hardware::neuralnetworks::V1_1::ExecutionPreference;
55using ::android::hardware::neuralnetworks::V1_1::IDevice;
56using ::android::hardware::neuralnetworks::V1_1::Model;
57using ::android::hidl::memory::V1_0::IMemory;
Xusong Wang6aad0402019-08-09 16:41:16 -070058
59Model createModel(const TestModel& testModel) {
60 // Model operands.
61 hidl_vec<Operand> operands(testModel.operands.size());
62 size_t constCopySize = 0, constRefSize = 0;
63 for (uint32_t i = 0; i < testModel.operands.size(); i++) {
64 const auto& op = testModel.operands[i];
65
66 DataLocation loc = {};
67 if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
68 loc = {.poolIndex = 0,
69 .offset = static_cast<uint32_t>(constCopySize),
70 .length = static_cast<uint32_t>(op.data.size())};
71 constCopySize += op.data.alignedSize();
72 } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
73 loc = {.poolIndex = 0,
74 .offset = static_cast<uint32_t>(constRefSize),
75 .length = static_cast<uint32_t>(op.data.size())};
76 constRefSize += op.data.alignedSize();
77 }
78
79 operands[i] = {.type = static_cast<OperandType>(op.type),
80 .dimensions = op.dimensions,
81 .numberOfConsumers = op.numberOfConsumers,
82 .scale = op.scale,
83 .zeroPoint = op.zeroPoint,
84 .lifetime = static_cast<OperandLifeTime>(op.lifetime),
85 .location = loc};
86 }
87
88 // Model operations.
89 hidl_vec<Operation> operations(testModel.operations.size());
90 std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),
91 [](const TestOperation& op) -> Operation {
92 return {.type = static_cast<OperationType>(op.type),
93 .inputs = op.inputs,
94 .outputs = op.outputs};
95 });
96
97 // Constant copies.
98 hidl_vec<uint8_t> operandValues(constCopySize);
99 for (uint32_t i = 0; i < testModel.operands.size(); i++) {
100 const auto& op = testModel.operands[i];
101 if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
102 const uint8_t* begin = op.data.get<uint8_t>();
103 const uint8_t* end = begin + op.data.size();
104 std::copy(begin, end, operandValues.data() + operands[i].location.offset);
105 }
106 }
107
108 // Shared memory.
109 hidl_vec<hidl_memory> pools;
110 if (constRefSize > 0) {
111 hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
112 CHECK_NE(pools[0].size(), 0u);
113
114 // load data
115 sp<IMemory> mappedMemory = mapMemory(pools[0]);
116 CHECK(mappedMemory.get() != nullptr);
117 uint8_t* mappedPtr =
118 reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));
119 CHECK(mappedPtr != nullptr);
120
121 for (uint32_t i = 0; i < testModel.operands.size(); i++) {
122 const auto& op = testModel.operands[i];
123 if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
124 const uint8_t* begin = op.data.get<uint8_t>();
125 const uint8_t* end = begin + op.data.size();
126 std::copy(begin, end, mappedPtr + operands[i].location.offset);
127 }
128 }
129 }
130
131 return {.operands = std::move(operands),
132 .operations = std::move(operations),
133 .inputIndexes = testModel.inputIndexes,
134 .outputIndexes = testModel.outputIndexes,
135 .operandValues = std::move(operandValues),
136 .pools = std::move(pools),
137 .relaxComputationFloat32toFloat16 = testModel.isRelaxed};
138}
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100139
140// Top level driver for models and examples generated by test_generator.py
141// Test driver for those generated from ml/nn/runtime/test/spec
Xusong Wang6aad0402019-08-09 16:41:16 -0700142void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel) {
143 const Request request = createRequest(testModel);
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100144
Xusong Wang6aad0402019-08-09 16:41:16 -0700145 // Launch execution.
146 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
147 Return<ErrorStatus> executionLaunchStatus = preparedModel->execute(request, executionCallback);
148 ASSERT_TRUE(executionLaunchStatus.isOk());
149 EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100150
Xusong Wang6aad0402019-08-09 16:41:16 -0700151 // Retrieve execution status.
152 executionCallback->wait();
153 ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus());
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100154
Xusong Wang6aad0402019-08-09 16:41:16 -0700155 // Retrieve execution results.
156 const std::vector<TestBuffer> outputs = getOutputBuffers(request);
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100157
Xusong Wang6aad0402019-08-09 16:41:16 -0700158 // We want "close-enough" results.
159 checkResults(testModel, outputs);
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100160}
161
Xusong Wang9e2b97b2019-08-23 16:10:54 -0700162// Tag for the generated tests
163class GeneratedTest : public GeneratedTestBase {
164 protected:
165 void Execute(const TestModel& testModel) {
166 Model model = createModel(testModel);
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100167
Xusong Wang9e2b97b2019-08-23 16:10:54 -0700168 // see if service can handle model
169 bool fullySupportsModel = false;
170 Return<void> supportedCall = device->getSupportedOperations_1_1(
171 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
172 ASSERT_EQ(ErrorStatus::NONE, status);
173 ASSERT_NE(0ul, supported.size());
174 fullySupportsModel = std::all_of(supported.begin(), supported.end(),
175 [](bool valid) { return valid; });
176 });
177 ASSERT_TRUE(supportedCall.isOk());
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100178
Xusong Wang9e2b97b2019-08-23 16:10:54 -0700179 // launch prepare model
180 sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
181 Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
182 model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
183 ASSERT_TRUE(prepareLaunchStatus.isOk());
184 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100185
Xusong Wang9e2b97b2019-08-23 16:10:54 -0700186 // retrieve prepared model
187 preparedModelCallback->wait();
188 ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
189 sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100190
Xusong Wang9e2b97b2019-08-23 16:10:54 -0700191 // early termination if vendor service cannot fully prepare model
192 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
193 ASSERT_EQ(nullptr, preparedModel.get());
194 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
195 "prepare model that it does not support.";
196 std::cout << "[ ] Early termination of test because vendor service cannot "
197 "prepare model that it does not support."
198 << std::endl;
199 GTEST_SKIP();
200 }
201 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
202 ASSERT_NE(nullptr, preparedModel.get());
203
204 EvaluatePreparedModel(preparedModel, testModel);
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100205 }
Xusong Wang9e2b97b2019-08-23 16:10:54 -0700206};
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100207
Xusong Wang9e2b97b2019-08-23 16:10:54 -0700208TEST_P(GeneratedTest, Test) {
209 Execute(*mTestModel);
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100210}
211
Xusong Wang9e2b97b2019-08-23 16:10:54 -0700212INSTANTIATE_GENERATED_TEST(GeneratedTest,
213 [](const TestModel& testModel) { return !testModel.expectFailure; });
214
215} // namespace functional
216} // namespace vts
Slava Shklyaev2bcfdc82019-07-17 15:50:57 +0100217} // namespace V1_1
Slava Shklyaev1d6b4652019-05-14 14:15:14 +0100218} // namespace neuralnetworks
219} // namespace hardware
220} // namespace android