blob: 684b43366a074bada0d1042b9ca987f11ec18e72 [file] [log] [blame]
Slava Shklyaevfeb87a92018-09-12 14:52:02 +01001/*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "neuralnetworks_hidl_hal_test"
18
Slava Shklyaev73ee79d2019-05-14 14:15:14 +010019#include "1.0/Utils.h"
20#include "1.2/Callbacks.h"
21#include "ExecutionBurstController.h"
Xusong Wangbcaa7822019-08-23 16:10:54 -070022#include "GeneratedTestHarness.h"
Slava Shklyaev73ee79d2019-05-14 14:15:14 +010023#include "TestHarness.h"
24#include "Utils.h"
25#include "VtsHalNeuralnetworks.h"
26
Michael Butler62749b92019-08-26 23:55:47 -070027namespace android::hardware::neuralnetworks::V1_2::vts::functional {
Slava Shklyaevfeb87a92018-09-12 14:52:02 +010028
Michael Butler62749b92019-08-26 23:55:47 -070029using implementation::ExecutionCallback;
30using V1_0::ErrorStatus;
31using V1_0::Request;
Slava Shklyaevfeb87a92018-09-12 14:52:02 +010032
33///////////////////////// UTILITY FUNCTIONS /////////////////////////
34
David Gross55a3d322019-01-23 14:01:52 -080035static bool badTiming(Timing timing) {
36 return timing.timeOnDevice == UINT64_MAX && timing.timeInDriver == UINT64_MAX;
37}
38
Slava Shklyaevfeb87a92018-09-12 14:52:02 +010039// Primary validation function. This function will take a valid request, apply a
40// mutation to it to invalidate the request, then pass it to interface calls
41// that use the request. Note that the request here is passed by value, and any
42// mutation to the request does not leave this function.
43static void validate(const sp<IPreparedModel>& preparedModel, const std::string& message,
44 Request request, const std::function<void(Request*)>& mutation) {
45 mutation(&request);
Slava Shklyaevfeb87a92018-09-12 14:52:02 +010046
David Gross55a3d322019-01-23 14:01:52 -080047 // We'd like to test both with timing requested and without timing
48 // requested. Rather than running each test both ways, we'll decide whether
49 // to request timing by hashing the message. We do not use std::hash because
50 // it is not guaranteed stable across executions.
51 char hash = 0;
52 for (auto c : message) {
53 hash ^= c;
54 };
55 MeasureTiming measure = (hash & 1) ? MeasureTiming::YES : MeasureTiming::NO;
56
Michael Butler29471a82019-01-15 11:02:55 -080057 // asynchronous
David Gross4592ed12018-12-21 11:20:26 -080058 {
59 SCOPED_TRACE(message + " [execute_1_2]");
Slava Shklyaevfeb87a92018-09-12 14:52:02 +010060
David Gross4592ed12018-12-21 11:20:26 -080061 sp<ExecutionCallback> executionCallback = new ExecutionCallback();
David Gross4592ed12018-12-21 11:20:26 -080062 Return<ErrorStatus> executeLaunchStatus =
David Gross55a3d322019-01-23 14:01:52 -080063 preparedModel->execute_1_2(request, measure, executionCallback);
David Gross4592ed12018-12-21 11:20:26 -080064 ASSERT_TRUE(executeLaunchStatus.isOk());
65 ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(executeLaunchStatus));
66
67 executionCallback->wait();
68 ErrorStatus executionReturnStatus = executionCallback->getStatus();
Xusong Wangb50bc312018-11-07 09:33:59 -080069 const auto& outputShapes = executionCallback->getOutputShapes();
David Gross55a3d322019-01-23 14:01:52 -080070 Timing timing = executionCallback->getTiming();
David Gross4592ed12018-12-21 11:20:26 -080071 ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus);
Xusong Wangb50bc312018-11-07 09:33:59 -080072 ASSERT_EQ(outputShapes.size(), 0);
David Gross55a3d322019-01-23 14:01:52 -080073 ASSERT_TRUE(badTiming(timing));
David Gross4592ed12018-12-21 11:20:26 -080074 }
75
Michael Butler29471a82019-01-15 11:02:55 -080076 // synchronous
David Gross4592ed12018-12-21 11:20:26 -080077 {
78 SCOPED_TRACE(message + " [executeSynchronously]");
79
Xusong Wangb50bc312018-11-07 09:33:59 -080080 Return<void> executeStatus = preparedModel->executeSynchronously(
David Gross55a3d322019-01-23 14:01:52 -080081 request, measure,
82 [](ErrorStatus error, const hidl_vec<OutputShape>& outputShapes,
83 const Timing& timing) {
84 ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, error);
85 EXPECT_EQ(outputShapes.size(), 0);
86 EXPECT_TRUE(badTiming(timing));
87 });
David Gross4592ed12018-12-21 11:20:26 -080088 ASSERT_TRUE(executeStatus.isOk());
David Gross4592ed12018-12-21 11:20:26 -080089 }
Michael Butler29471a82019-01-15 11:02:55 -080090
91 // burst
92 {
93 SCOPED_TRACE(message + " [burst]");
94
95 // create burst
Michael Butler51c72182019-03-27 10:59:25 -070096 std::shared_ptr<::android::nn::ExecutionBurstController> burst =
Michael Butler60a22532019-03-28 13:41:14 -070097 ::android::nn::ExecutionBurstController::create(preparedModel, /*blocking=*/true);
Michael Butler29471a82019-01-15 11:02:55 -080098 ASSERT_NE(nullptr, burst.get());
99
100 // create memory keys
101 std::vector<intptr_t> keys(request.pools.size());
102 for (size_t i = 0; i < keys.size(); ++i) {
103 keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
104 }
105
106 // execute and verify
107 ErrorStatus error;
108 std::vector<OutputShape> outputShapes;
109 Timing timing;
110 std::tie(error, outputShapes, timing) = burst->compute(request, measure, keys);
111 EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, error);
112 EXPECT_EQ(outputShapes.size(), 0);
113 EXPECT_TRUE(badTiming(timing));
114
115 // additional burst testing
116 if (request.pools.size() > 0) {
117 // valid free
118 burst->freeMemory(keys.front());
119
120 // negative test: invalid free of unknown (blank) memory
121 burst->freeMemory(intptr_t{});
122
123 // negative test: double free of memory
124 burst->freeMemory(keys.front());
125 }
126 }
Slava Shklyaevfeb87a92018-09-12 14:52:02 +0100127}
128
Slava Shklyaevfeb87a92018-09-12 14:52:02 +0100129///////////////////////// REMOVE INPUT ////////////////////////////////////
130
131static void removeInputTest(const sp<IPreparedModel>& preparedModel, const Request& request) {
132 for (size_t input = 0; input < request.inputs.size(); ++input) {
133 const std::string message = "removeInput: removed input " + std::to_string(input);
134 validate(preparedModel, message, request,
135 [input](Request* request) { hidl_vec_removeAt(&request->inputs, input); });
136 }
137}
138
139///////////////////////// REMOVE OUTPUT ////////////////////////////////////
140
141static void removeOutputTest(const sp<IPreparedModel>& preparedModel, const Request& request) {
142 for (size_t output = 0; output < request.outputs.size(); ++output) {
143 const std::string message = "removeOutput: removed Output " + std::to_string(output);
144 validate(preparedModel, message, request,
145 [output](Request* request) { hidl_vec_removeAt(&request->outputs, output); });
146 }
147}
148
149///////////////////////////// ENTRY POINT //////////////////////////////////
150
Xusong Wangead950d2019-08-09 16:45:24 -0700151void ValidationTest::validateRequest(const sp<IPreparedModel>& preparedModel,
152 const Request& request) {
153 removeInputTest(preparedModel, request);
154 removeOutputTest(preparedModel, request);
Slava Shklyaevfeb87a92018-09-12 14:52:02 +0100155}
156
Slava Shklyaev95a59782019-05-10 16:08:30 +0100157void ValidationTest::validateRequestFailure(const sp<IPreparedModel>& preparedModel,
Xusong Wangead950d2019-08-09 16:45:24 -0700158 const Request& request) {
159 SCOPED_TRACE("Expecting request to fail [executeSynchronously]");
160 Return<void> executeStatus = preparedModel->executeSynchronously(
161 request, MeasureTiming::NO,
162 [](ErrorStatus error, const hidl_vec<OutputShape>& outputShapes, const Timing& timing) {
163 ASSERT_NE(ErrorStatus::NONE, error);
164 EXPECT_EQ(outputShapes.size(), 0);
165 EXPECT_TRUE(badTiming(timing));
166 });
167 ASSERT_TRUE(executeStatus.isOk());
Slava Shklyaev95a59782019-05-10 16:08:30 +0100168}
169
Michael Butler62749b92019-08-26 23:55:47 -0700170} // namespace android::hardware::neuralnetworks::V1_2::vts::functional