Refactor NN API VTS tests and add v1.1 tests
- Create VtsHalNeuralnetworksTest_utils module to be shared between
v1.x tests.
- Split the existing tests into two categories: basic, and generated.
- Created v1.1 VTS tests ensuring no regression in existing ML models.
Bug: 63911257
Test: mm
Test: NNAPI VTS tests pass on v1.0 and v1.1 sample drivers
Merged-In: Ic77c90a3a5bbd96b0ce2acd03764dde4b3034cc9
Change-Id: Ic77c90a3a5bbd96b0ce2acd03764dde4b3034cc9
(cherry picked from commit a2d04c828e98bdadc6dd44c6235556451e4e2a88)
diff --git a/neuralnetworks/1.1/vts/functional/Android.bp b/neuralnetworks/1.1/vts/functional/Android.bp
new file mode 100644
index 0000000..623b441
--- /dev/null
+++ b/neuralnetworks/1.1/vts/functional/Android.bp
@@ -0,0 +1,39 @@
+//
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+cc_test {
+ name: "VtsHalNeuralnetworksV1_1TargetTest",
+ srcs: [
+ "VtsHalNeuralnetworksV1_1.cpp",
+ "VtsHalNeuralnetworksV1_1BasicTest.cpp",
+ "VtsHalNeuralnetworksV1_1GeneratedTest.cpp",
+ ],
+ defaults: ["VtsHalTargetTestDefaults"],
+ static_libs: [
+ "android.hardware.neuralnetworks@1.0",
+ "android.hardware.neuralnetworks@1.1",
+ "android.hidl.allocator@1.0",
+ "android.hidl.memory@1.0",
+ "libhidlmemory",
+ "libneuralnetworks_utils",
+ "VtsHalNeuralnetworksTest_utils",
+ ],
+ header_libs: [
+ "libneuralnetworks_headers",
+ "libneuralnetworks_generated_test_harness_headers",
+ "libneuralnetworks_generated_tests",
+ ],
+}
diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.cpp
new file mode 100644
index 0000000..b1d3be7
--- /dev/null
+++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.cpp
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "neuralnetworks_hidl_hal_test"
+
+#include "VtsHalNeuralnetworksV1_1.h"
+#include "Utils.h"
+
+#include <android-base/logging.h>
+#include <hidlmemory/mapping.h>
+
+using ::android::hardware::hidl_memory;
+using ::android::hidl::allocator::V1_0::IAllocator;
+using ::android::hidl::memory::V1_0::IMemory;
+using ::android::sp;
+
+namespace android {
+namespace hardware {
+namespace neuralnetworks {
+namespace V1_1 {
+namespace vts {
+namespace functional {
+
+// allocator helper
+hidl_memory allocateSharedMemory(int64_t size) {
+ return nn::allocateSharedMemory(size);
+}
+
+// A class for test environment setup
+NeuralnetworksHidlEnvironment::NeuralnetworksHidlEnvironment() {}
+
+NeuralnetworksHidlEnvironment::~NeuralnetworksHidlEnvironment() {}
+
+NeuralnetworksHidlEnvironment* NeuralnetworksHidlEnvironment::getInstance() {
+ // This has to return a "new" object because it is freed inside
+ // ::testing::AddGlobalTestEnvironment when the gtest is being torn down
+ static NeuralnetworksHidlEnvironment* instance = new NeuralnetworksHidlEnvironment();
+ return instance;
+}
+
+void NeuralnetworksHidlEnvironment::registerTestServices() {
+ registerTestService<V1_1::IDevice>();
+}
+
+// The main test class for NEURALNETWORK HIDL HAL.
+NeuralnetworksHidlTest::~NeuralnetworksHidlTest() {}
+
+void NeuralnetworksHidlTest::SetUp() {
+ device = ::testing::VtsHalHidlTargetTestBase::getService<V1_1::IDevice>(
+ NeuralnetworksHidlEnvironment::getInstance());
+ ASSERT_NE(nullptr, device.get());
+}
+
+void NeuralnetworksHidlTest::TearDown() {}
+
+} // namespace functional
+} // namespace vts
+} // namespace V1_1
+} // namespace neuralnetworks
+} // namespace hardware
+} // namespace android
diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.h b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.h
new file mode 100644
index 0000000..426246c
--- /dev/null
+++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef VTS_HAL_NEURALNETWORKS_V1_1_H
+#define VTS_HAL_NEURALNETWORKS_V1_1_H
+
+#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
+#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
+#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
+#include <android/hardware/neuralnetworks/1.1/IDevice.h>
+#include <android/hardware/neuralnetworks/1.1/types.h>
+#include <android/hidl/allocator/1.0/IAllocator.h>
+
+#include <VtsHalHidlTargetTestBase.h>
+#include <VtsHalHidlTargetTestEnvBase.h>
+#include <gtest/gtest.h>
+#include <string>
+
+namespace android {
+namespace hardware {
+namespace neuralnetworks {
+namespace V1_1 {
+namespace vts {
+namespace functional {
+hidl_memory allocateSharedMemory(int64_t size);
+
+// A class for test environment setup
+class NeuralnetworksHidlEnvironment : public ::testing::VtsHalHidlTargetTestEnvBase {
+ NeuralnetworksHidlEnvironment();
+ NeuralnetworksHidlEnvironment(const NeuralnetworksHidlEnvironment&) = delete;
+ NeuralnetworksHidlEnvironment(NeuralnetworksHidlEnvironment&&) = delete;
+ NeuralnetworksHidlEnvironment& operator=(const NeuralnetworksHidlEnvironment&) = delete;
+ NeuralnetworksHidlEnvironment& operator=(NeuralnetworksHidlEnvironment&&) = delete;
+
+ public:
+ ~NeuralnetworksHidlEnvironment() override;
+ static NeuralnetworksHidlEnvironment* getInstance();
+ void registerTestServices() override;
+};
+
+// The main test class for NEURALNETWORKS HIDL HAL.
+class NeuralnetworksHidlTest : public ::testing::VtsHalHidlTargetTestBase {
+ public:
+ ~NeuralnetworksHidlTest() override;
+ void SetUp() override;
+ void TearDown() override;
+
+ sp<V1_1::IDevice> device;
+};
+} // namespace functional
+} // namespace vts
+
+// pretty-print values for error messages
+
+template <typename CharT, typename Traits>
+::std::basic_ostream<CharT, Traits>& operator<<(::std::basic_ostream<CharT, Traits>& os,
+ V1_0::ErrorStatus errorStatus) {
+ return os << toString(errorStatus);
+}
+
+template <typename CharT, typename Traits>
+::std::basic_ostream<CharT, Traits>& operator<<(::std::basic_ostream<CharT, Traits>& os,
+ V1_0::DeviceStatus deviceStatus) {
+ return os << toString(deviceStatus);
+}
+
+} // namespace V1_1
+} // namespace neuralnetworks
+} // namespace hardware
+} // namespace android
+
+#endif // VTS_HAL_NEURALNETWORKS_V1_1_H
diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp
new file mode 100644
index 0000000..51eff2a
--- /dev/null
+++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1BasicTest.cpp
@@ -0,0 +1,280 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "neuralnetworks_hidl_hal_test"
+
+#include "VtsHalNeuralnetworksV1_1.h"
+
+#include "Callbacks.h"
+#include "Models.h"
+#include "TestHarness.h"
+
+#include <android-base/logging.h>
+#include <android/hardware/neuralnetworks/1.1/IDevice.h>
+#include <android/hardware/neuralnetworks/1.1/types.h>
+#include <android/hidl/memory/1.0/IMemory.h>
+#include <hidlmemory/mapping.h>
+
+using ::android::hardware::neuralnetworks::V1_0::IPreparedModel;
+using ::android::hardware::neuralnetworks::V1_0::Capabilities;
+using ::android::hardware::neuralnetworks::V1_0::DeviceStatus;
+using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
+using ::android::hardware::neuralnetworks::V1_0::FusedActivationFunc;
+using ::android::hardware::neuralnetworks::V1_0::Operand;
+using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime;
+using ::android::hardware::neuralnetworks::V1_0::OperandType;
+using ::android::hardware::neuralnetworks::V1_0::Request;
+using ::android::hardware::neuralnetworks::V1_1::IDevice;
+using ::android::hardware::neuralnetworks::V1_1::Model;
+using ::android::hardware::neuralnetworks::V1_1::Operation;
+using ::android::hardware::neuralnetworks::V1_1::OperationType;
+using ::android::hardware::Return;
+using ::android::hardware::Void;
+using ::android::hardware::hidl_memory;
+using ::android::hardware::hidl_string;
+using ::android::hardware::hidl_vec;
+using ::android::hidl::allocator::V1_0::IAllocator;
+using ::android::hidl::memory::V1_0::IMemory;
+using ::android::sp;
+
+namespace android {
+namespace hardware {
+namespace neuralnetworks {
+namespace V1_1 {
+namespace vts {
+namespace functional {
+using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
+using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
+
+inline sp<IPreparedModel> doPrepareModelShortcut(sp<IDevice>& device) {
+ Model model = createValidTestModel_1_1();
+
+ sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+ if (preparedModelCallback == nullptr) {
+ return nullptr;
+ }
+ Return<ErrorStatus> prepareLaunchStatus =
+ device->prepareModel_1_1(model, preparedModelCallback);
+ if (!prepareLaunchStatus.isOk() || prepareLaunchStatus != ErrorStatus::NONE) {
+ return nullptr;
+ }
+
+ preparedModelCallback->wait();
+ ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
+ sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
+ if (prepareReturnStatus != ErrorStatus::NONE || preparedModel == nullptr) {
+ return nullptr;
+ }
+
+ return preparedModel;
+}
+
+// create device test
+TEST_F(NeuralnetworksHidlTest, CreateDevice) {}
+
+// status test
+TEST_F(NeuralnetworksHidlTest, StatusTest) {
+ Return<DeviceStatus> status = device->getStatus();
+ ASSERT_TRUE(status.isOk());
+ EXPECT_EQ(DeviceStatus::AVAILABLE, static_cast<DeviceStatus>(status));
+}
+
+// initialization
+TEST_F(NeuralnetworksHidlTest, GetCapabilitiesTest) {
+ Return<void> ret =
+ device->getCapabilities([](ErrorStatus status, const Capabilities& capabilities) {
+ EXPECT_EQ(ErrorStatus::NONE, status);
+ EXPECT_LT(0.0f, capabilities.float32Performance.execTime);
+ EXPECT_LT(0.0f, capabilities.float32Performance.powerUsage);
+ EXPECT_LT(0.0f, capabilities.quantized8Performance.execTime);
+ EXPECT_LT(0.0f, capabilities.quantized8Performance.powerUsage);
+ });
+ EXPECT_TRUE(ret.isOk());
+}
+
+// supported operations positive test
+TEST_F(NeuralnetworksHidlTest, SupportedOperationsPositiveTest) {
+ Model model = createValidTestModel_1_1();
+ Return<void> ret = device->getSupportedOperations_1_1(
+ model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
+ EXPECT_EQ(ErrorStatus::NONE, status);
+ EXPECT_EQ(model.operations.size(), supported.size());
+ });
+ EXPECT_TRUE(ret.isOk());
+}
+
+// supported operations negative test 1
+TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest1) {
+ Model model = createInvalidTestModel1_1_1();
+ Return<void> ret = device->getSupportedOperations_1_1(
+ model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
+ EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status);
+ (void)supported;
+ });
+ EXPECT_TRUE(ret.isOk());
+}
+
+// supported operations negative test 2
+TEST_F(NeuralnetworksHidlTest, SupportedOperationsNegativeTest2) {
+ Model model = createInvalidTestModel2_1_1();
+ Return<void> ret = device->getSupportedOperations_1_1(
+ model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
+ EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status);
+ (void)supported;
+ });
+ EXPECT_TRUE(ret.isOk());
+}
+
+// prepare simple model positive test
+TEST_F(NeuralnetworksHidlTest, SimplePrepareModelPositiveTest) {
+ Model model = createValidTestModel_1_1();
+ sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+ ASSERT_NE(nullptr, preparedModelCallback.get());
+ Return<ErrorStatus> prepareLaunchStatus =
+ device->prepareModel_1_1(model, preparedModelCallback);
+ ASSERT_TRUE(prepareLaunchStatus.isOk());
+ EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
+
+ preparedModelCallback->wait();
+ ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
+ EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
+ sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
+ EXPECT_NE(nullptr, preparedModel.get());
+}
+
+// prepare simple model negative test 1
+TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest1) {
+ Model model = createInvalidTestModel1_1_1();
+ sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+ ASSERT_NE(nullptr, preparedModelCallback.get());
+ Return<ErrorStatus> prepareLaunchStatus =
+ device->prepareModel_1_1(model, preparedModelCallback);
+ ASSERT_TRUE(prepareLaunchStatus.isOk());
+ EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(prepareLaunchStatus));
+
+ preparedModelCallback->wait();
+ ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
+ EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus);
+ sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
+ EXPECT_EQ(nullptr, preparedModel.get());
+}
+
+// prepare simple model negative test 2
+TEST_F(NeuralnetworksHidlTest, SimplePrepareModelNegativeTest2) {
+ Model model = createInvalidTestModel2_1_1();
+ sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+ ASSERT_NE(nullptr, preparedModelCallback.get());
+ Return<ErrorStatus> prepareLaunchStatus =
+ device->prepareModel_1_1(model, preparedModelCallback);
+ ASSERT_TRUE(prepareLaunchStatus.isOk());
+ EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(prepareLaunchStatus));
+
+ preparedModelCallback->wait();
+ ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
+ EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus);
+ sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
+ EXPECT_EQ(nullptr, preparedModel.get());
+}
+
+// execute simple graph positive test
+TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphPositiveTest) {
+ std::vector<float> outputData = {-1.0f, -1.0f, -1.0f, -1.0f};
+ std::vector<float> expectedData = {6.0f, 8.0f, 10.0f, 12.0f};
+ const uint32_t OUTPUT = 1;
+
+ sp<IPreparedModel> preparedModel = doPrepareModelShortcut(device);
+ ASSERT_NE(nullptr, preparedModel.get());
+ Request request = createValidTestRequest();
+
+ auto postWork = [&] {
+ sp<IMemory> outputMemory = mapMemory(request.pools[OUTPUT]);
+ if (outputMemory == nullptr) {
+ return false;
+ }
+ float* outputPtr = reinterpret_cast<float*>(static_cast<void*>(outputMemory->getPointer()));
+ if (outputPtr == nullptr) {
+ return false;
+ }
+ outputMemory->read();
+ std::copy(outputPtr, outputPtr + outputData.size(), outputData.begin());
+ outputMemory->commit();
+ return true;
+ };
+
+ sp<ExecutionCallback> executionCallback = new ExecutionCallback();
+ ASSERT_NE(nullptr, executionCallback.get());
+ executionCallback->on_finish(postWork);
+ Return<ErrorStatus> executeLaunchStatus = preparedModel->execute(request, executionCallback);
+ ASSERT_TRUE(executeLaunchStatus.isOk());
+ EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executeLaunchStatus));
+
+ executionCallback->wait();
+ ErrorStatus executionReturnStatus = executionCallback->getStatus();
+ EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
+ EXPECT_EQ(expectedData, outputData);
+}
+
+// execute simple graph negative test 1
+TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest1) {
+ sp<IPreparedModel> preparedModel = doPrepareModelShortcut(device);
+ ASSERT_NE(nullptr, preparedModel.get());
+ Request request = createInvalidTestRequest1();
+
+ sp<ExecutionCallback> executionCallback = new ExecutionCallback();
+ ASSERT_NE(nullptr, executionCallback.get());
+ Return<ErrorStatus> executeLaunchStatus = preparedModel->execute(request, executionCallback);
+ ASSERT_TRUE(executeLaunchStatus.isOk());
+ EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(executeLaunchStatus));
+
+ executionCallback->wait();
+ ErrorStatus executionReturnStatus = executionCallback->getStatus();
+ EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus);
+}
+
+// execute simple graph negative test 2
+TEST_F(NeuralnetworksHidlTest, SimpleExecuteGraphNegativeTest2) {
+ sp<IPreparedModel> preparedModel = doPrepareModelShortcut(device);
+ ASSERT_NE(nullptr, preparedModel.get());
+ Request request = createInvalidTestRequest2();
+
+ sp<ExecutionCallback> executionCallback = new ExecutionCallback();
+ ASSERT_NE(nullptr, executionCallback.get());
+ Return<ErrorStatus> executeLaunchStatus = preparedModel->execute(request, executionCallback);
+ ASSERT_TRUE(executeLaunchStatus.isOk());
+ EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(executeLaunchStatus));
+
+ executionCallback->wait();
+ ErrorStatus executionReturnStatus = executionCallback->getStatus();
+ EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus);
+}
+
+} // namespace functional
+} // namespace vts
+} // namespace V1_1
+} // namespace neuralnetworks
+} // namespace hardware
+} // namespace android
+
+using android::hardware::neuralnetworks::V1_1::vts::functional::NeuralnetworksHidlEnvironment;
+
+int main(int argc, char** argv) {
+ ::testing::AddGlobalTestEnvironment(NeuralnetworksHidlEnvironment::getInstance());
+ ::testing::InitGoogleTest(&argc, argv);
+ NeuralnetworksHidlEnvironment::getInstance()->init(&argc, argv);
+
+ int status = RUN_ALL_TESTS();
+ return status;
+}
diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1GeneratedTest.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1GeneratedTest.cpp
new file mode 100644
index 0000000..025d9fe
--- /dev/null
+++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworksV1_1GeneratedTest.cpp
@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "neuralnetworks_hidl_hal_test"
+
+#include "VtsHalNeuralnetworksV1_1.h"
+
+#include "Callbacks.h"
+#include "TestHarness.h"
+
+#include <android-base/logging.h>
+#include <android/hardware/neuralnetworks/1.1/IDevice.h>
+#include <android/hardware/neuralnetworks/1.1/types.h>
+#include <android/hidl/memory/1.0/IMemory.h>
+#include <hidlmemory/mapping.h>
+
+using ::android::hardware::neuralnetworks::V1_0::IPreparedModel;
+using ::android::hardware::neuralnetworks::V1_0::Capabilities;
+using ::android::hardware::neuralnetworks::V1_0::DeviceStatus;
+using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
+using ::android::hardware::neuralnetworks::V1_0::FusedActivationFunc;
+using ::android::hardware::neuralnetworks::V1_0::Operand;
+using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime;
+using ::android::hardware::neuralnetworks::V1_0::OperandType;
+using ::android::hardware::neuralnetworks::V1_0::Request;
+using ::android::hardware::neuralnetworks::V1_1::IDevice;
+using ::android::hardware::neuralnetworks::V1_1::Model;
+using ::android::hardware::neuralnetworks::V1_1::Operation;
+using ::android::hardware::neuralnetworks::V1_1::OperationType;
+using ::android::hardware::Return;
+using ::android::hardware::Void;
+using ::android::hardware::hidl_memory;
+using ::android::hardware::hidl_string;
+using ::android::hardware::hidl_vec;
+using ::android::hidl::allocator::V1_0::IAllocator;
+using ::android::hidl::memory::V1_0::IMemory;
+using ::android::sp;
+
+namespace android {
+namespace hardware {
+namespace neuralnetworks {
+
+namespace generated_tests {
+using ::generated_tests::MixedTypedExampleType;
+extern void Execute(sp<V1_1::IDevice>&, std::function<Model(void)>, std::function<bool(int)>,
+ const std::vector<MixedTypedExampleType>&);
+} // namespace generated_tests
+
+namespace V1_1 {
+namespace vts {
+namespace functional {
+using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback;
+using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback;
+
+// Mixed-typed examples
+typedef generated_tests::MixedTypedExampleType MixedTypedExample;
+
+// in frameworks/ml/nn/runtime/tests/generated/
+#include "all_generated_V1_0_vts_tests.cpp"
+#include "all_generated_V1_1_vts_tests.cpp"
+
+} // namespace functional
+} // namespace vts
+} // namespace V1_1
+} // namespace neuralnetworks
+} // namespace hardware
+} // namespace android