Update VTS tests with the new test harness.
Bug: 120601396
Test: All VTS
Change-Id: I539e75585b2cc01d153565814491361adfa048be
diff --git a/neuralnetworks/1.0/vts/functional/Android.bp b/neuralnetworks/1.0/vts/functional/Android.bp
index abff213..664436c 100644
--- a/neuralnetworks/1.0/vts/functional/Android.bp
+++ b/neuralnetworks/1.0/vts/functional/Android.bp
@@ -74,7 +74,7 @@
defaults: ["VtsHalNeuralNetworksV1_0TargetTestDefaults"],
srcs: [
"BasicTests.cpp",
- ":VtsHalNeuralNetworksV1_0_all_generated_V1_0_tests",
+ ":VtsHalNeuralNetworksV1_0_all_generated_tests",
],
}
@@ -83,7 +83,7 @@
defaults: ["VtsHalNeuralNetworksV1_0TargetTestDefaults"],
srcs: [
"BasicTests.cpp",
- ":VtsHalNeuralNetworksV1_0_all_generated_V1_0_tests",
+ ":VtsHalNeuralNetworksV1_0_all_generated_tests",
],
cflags: [
"-DPRESUBMIT_NOT_VTS",
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index 0fd9947..5f96539 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -20,6 +20,7 @@
#include "1.0/Utils.h"
#include "MemoryUtils.h"
#include "TestHarness.h"
+#include "VtsHalNeuralnetworks.h"
#include <android-base/logging.h>
#include <android/hardware/neuralnetworks/1.0/IDevice.h>
@@ -36,7 +37,8 @@
namespace hardware {
namespace neuralnetworks {
namespace V1_0 {
-namespace generated_tests {
+namespace vts {
+namespace functional {
using namespace test_helper;
using ::android::hardware::neuralnetworks::V1_0::ErrorStatus;
@@ -151,48 +153,61 @@
checkResults(testModel, outputs);
}
-void Execute(const sp<IDevice>& device, const TestModel& testModel) {
- Model model = createModel(testModel);
+// Tag for the generated tests
+class GeneratedTest : public GeneratedTestBase {
+ protected:
+ void Execute(const TestModel& testModel) {
+ Model model = createModel(testModel);
- // see if service can handle model
- bool fullySupportsModel = false;
- Return<void> supportedCall = device->getSupportedOperations(
- model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
- ASSERT_EQ(ErrorStatus::NONE, status);
- ASSERT_NE(0ul, supported.size());
- fullySupportsModel = std::all_of(supported.begin(), supported.end(),
- [](bool valid) { return valid; });
- });
- ASSERT_TRUE(supportedCall.isOk());
+ // see if service can handle model
+ bool fullySupportsModel = false;
+ Return<void> supportedCall = device->getSupportedOperations(
+ model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
+ ASSERT_EQ(ErrorStatus::NONE, status);
+ ASSERT_NE(0ul, supported.size());
+ fullySupportsModel = std::all_of(supported.begin(), supported.end(),
+ [](bool valid) { return valid; });
+ });
+ ASSERT_TRUE(supportedCall.isOk());
- // launch prepare model
- sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
- Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
- ASSERT_TRUE(prepareLaunchStatus.isOk());
- ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
+ // launch prepare model
+ sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+ Return<ErrorStatus> prepareLaunchStatus =
+ device->prepareModel(model, preparedModelCallback);
+ ASSERT_TRUE(prepareLaunchStatus.isOk());
+ ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
- // retrieve prepared model
- preparedModelCallback->wait();
- ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
- sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
+ // retrieve prepared model
+ preparedModelCallback->wait();
+ ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
+ sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
- // early termination if vendor service cannot fully prepare model
- if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
- ASSERT_EQ(nullptr, preparedModel.get());
- LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
- "prepare model that it does not support.";
- std::cout << "[ ] Early termination of test because vendor service cannot "
- "prepare model that it does not support."
- << std::endl;
- GTEST_SKIP();
+ // early termination if vendor service cannot fully prepare model
+ if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
+ ASSERT_EQ(nullptr, preparedModel.get());
+ LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
+ "prepare model that it does not support.";
+ std::cout << "[ ] Early termination of test because vendor service cannot "
+ "prepare model that it does not support."
+ << std::endl;
+ GTEST_SKIP();
+ }
+ EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
+ ASSERT_NE(nullptr, preparedModel.get());
+
+ EvaluatePreparedModel(preparedModel, testModel);
}
- EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
- ASSERT_NE(nullptr, preparedModel.get());
+};
- EvaluatePreparedModel(preparedModel, testModel);
+TEST_P(GeneratedTest, Test) {
+ Execute(*mTestModel);
}
-} // namespace generated_tests
+INSTANTIATE_GENERATED_TEST(GeneratedTest,
+ [](const TestModel& testModel) { return !testModel.expectFailure; });
+
+} // namespace functional
+} // namespace vts
} // namespace V1_0
} // namespace neuralnetworks
} // namespace hardware
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h
index 5d22158..f86e8b3 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h
@@ -19,18 +19,48 @@
#include <android/hardware/neuralnetworks/1.0/IDevice.h>
#include "TestHarness.h"
+#include "VtsHalNeuralnetworks.h"
namespace android {
namespace hardware {
namespace neuralnetworks {
namespace V1_0 {
-namespace generated_tests {
+namespace vts {
+namespace functional {
+
+class GeneratedTestBase
+ : public NeuralnetworksHidlTest,
+ public ::testing::WithParamInterface<test_helper::TestModelManager::TestParam> {
+ protected:
+ void SetUp() override {
+ NeuralnetworksHidlTest::SetUp();
+ ASSERT_NE(mTestModel, nullptr);
+ }
+
+ const test_helper::TestModel* mTestModel = GetParam().second;
+};
+
+#define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \
+ INSTANTIATE_TEST_SUITE_P( \
+ TestGenerated, TestSuite, \
+ ::testing::ValuesIn(::test_helper::TestModelManager::get().getTestModels(filter)), \
+ [](const auto& info) { return info.param.first; })
+
+// Tag for the validation tests, instantiated in VtsHalNeuralnetworks.cpp.
+// TODO: Clean up the hierarchy for ValidationTest.
+class ValidationTest : public GeneratedTestBase {
+ protected:
+ void validateEverything(const Model& model, const Request& request);
+
+ private:
+ void validateModel(const Model& model);
+ void validateRequest(const sp<IPreparedModel>& preparedModel, const Request& request);
+};
Model createModel(const ::test_helper::TestModel& testModel);
-void Execute(const sp<V1_0::IDevice>& device, const ::test_helper::TestModel& testModel);
-
-} // namespace generated_tests
+} // namespace functional
+} // namespace vts
} // namespace V1_0
} // namespace neuralnetworks
} // namespace hardware
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTests.h b/neuralnetworks/1.0/vts/functional/GeneratedTests.h
deleted file mode 100644
index 9528905..0000000
--- a/neuralnetworks/1.0/vts/functional/GeneratedTests.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright (C) 2019 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "1.0/Utils.h"
-#include "GeneratedTestHarness.h"
-#include "TestHarness.h"
-#include "VtsHalNeuralnetworks.h"
-
-namespace android::hardware::neuralnetworks::V1_0::generated_tests {
-
-using namespace android::hardware::neuralnetworks::V1_0::vts::functional;
-
-} // namespace android::hardware::neuralnetworks::V1_0::generated_tests
diff --git a/neuralnetworks/1.0/vts/functional/ValidateModel.cpp b/neuralnetworks/1.0/vts/functional/ValidateModel.cpp
index 72c4a2b..5845aab 100644
--- a/neuralnetworks/1.0/vts/functional/ValidateModel.cpp
+++ b/neuralnetworks/1.0/vts/functional/ValidateModel.cpp
@@ -16,9 +16,9 @@
#define LOG_TAG "neuralnetworks_hidl_hal_test"
-#include "VtsHalNeuralnetworks.h"
-
#include "1.0/Callbacks.h"
+#include "GeneratedTestHarness.h"
+#include "VtsHalNeuralnetworks.h"
namespace android {
namespace hardware {
diff --git a/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp
index d62365c..730e054 100644
--- a/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp
+++ b/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp
@@ -17,6 +17,7 @@
#define LOG_TAG "neuralnetworks_hidl_hal_test"
#include "1.0/Callbacks.h"
+#include "GeneratedTestHarness.h"
#include "VtsHalNeuralnetworks.h"
namespace android {
diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp
index 626deac..a51f71f 100644
--- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp
+++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp
@@ -17,11 +17,13 @@
#define LOG_TAG "neuralnetworks_hidl_hal_test"
#include "VtsHalNeuralnetworks.h"
+#include "1.0/Callbacks.h"
+#include "1.0/Utils.h"
+#include "GeneratedTestHarness.h"
+#include "TestHarness.h"
#include <android-base/logging.h>
-#include "1.0/Callbacks.h"
-
namespace android {
namespace hardware {
namespace neuralnetworks {
@@ -134,6 +136,15 @@
validateRequest(preparedModel, request);
}
+TEST_P(ValidationTest, Test) {
+ const Model model = createModel(*mTestModel);
+ const Request request = createRequest(*mTestModel);
+ ASSERT_FALSE(mTestModel->expectFailure);
+ validateEverything(model, request);
+}
+
+INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; });
+
} // namespace functional
} // namespace vts
} // namespace V1_0
diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.h b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.h
index 3765314..9638a0e 100644
--- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.h
+++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.h
@@ -28,6 +28,8 @@
#include <iostream>
#include <vector>
+#include "TestHarness.h"
+
namespace android {
namespace hardware {
namespace neuralnetworks {
@@ -60,19 +62,6 @@
sp<IDevice> device;
};
-// Tag for the validation tests
-class ValidationTest : public NeuralnetworksHidlTest {
- protected:
- void validateEverything(const Model& model, const Request& request);
-
- private:
- void validateModel(const Model& model);
- void validateRequest(const sp<IPreparedModel>& preparedModel, const Request& request);
-};
-
-// Tag for the generated tests
-class GeneratedTest : public NeuralnetworksHidlTest {};
-
} // namespace functional
} // namespace vts
} // namespace V1_0