Update VTS tests with the new test harness.

Bug: 120601396
Test: All VTS
Change-Id: I539e75585b2cc01d153565814491361adfa048be
Merged-In: I539e75585b2cc01d153565814491361adfa048be
(cherry picked from commit 9e2b97b83cd3163750bebe55e6b55811ca8a6712)
diff --git a/neuralnetworks/1.1/vts/functional/Android.bp b/neuralnetworks/1.1/vts/functional/Android.bp
index 86002d2..05c7f6b 100644
--- a/neuralnetworks/1.1/vts/functional/Android.bp
+++ b/neuralnetworks/1.1/vts/functional/Android.bp
@@ -49,7 +49,7 @@
     name: "VtsHalNeuralnetworksV1_1CompatV1_0TargetTest",
     defaults: ["VtsHalNeuralNetworksV1_1TargetTestDefaults"],
     srcs: [
-        ":VtsHalNeuralNetworksV1_1_all_generated_V1_0_tests",
+        ":VtsHalNeuralNetworksV1_0_all_generated_tests",
     ],
 }
 
@@ -59,7 +59,7 @@
     defaults: ["VtsHalNeuralNetworksV1_1TargetTestDefaults"],
     srcs: [
         "BasicTests.cpp",
-        ":VtsHalNeuralNetworksV1_1_all_generated_V1_1_tests",
+        ":VtsHalNeuralNetworksV1_1_all_generated_tests",
     ],
 }
 
@@ -68,7 +68,7 @@
     defaults: ["VtsHalNeuralNetworksV1_1TargetTestDefaults"],
     srcs: [
         "BasicTests.cpp",
-        ":VtsHalNeuralNetworksV1_1_all_generated_V1_1_tests",
+        ":VtsHalNeuralNetworksV1_1_all_generated_tests",
     ],
     cflags: [
         "-DPRESUBMIT_NOT_VTS",
diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp
index 73eeb93..d8d1a31 100644
--- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp
@@ -31,12 +31,14 @@
 #include "1.0/Utils.h"
 #include "MemoryUtils.h"
 #include "TestHarness.h"
+#include "VtsHalNeuralnetworks.h"
 
 namespace android {
 namespace hardware {
 namespace neuralnetworks {
 namespace V1_1 {
-namespace generated_tests {
+namespace vts {
+namespace functional {
 
 using namespace test_helper;
 using ::android::hardware::neuralnetworks::V1_0::DataLocation;
@@ -157,49 +159,61 @@
     checkResults(testModel, outputs);
 }
 
-void Execute(const sp<IDevice>& device, const TestModel& testModel) {
-    Model model = createModel(testModel);
+// Tag for the generated tests
+class GeneratedTest : public GeneratedTestBase {
+  protected:
+    void Execute(const TestModel& testModel) {
+        Model model = createModel(testModel);
 
-    // see if service can handle model
-    bool fullySupportsModel = false;
-    Return<void> supportedCall = device->getSupportedOperations_1_1(
-            model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
-                ASSERT_EQ(ErrorStatus::NONE, status);
-                ASSERT_NE(0ul, supported.size());
-                fullySupportsModel = std::all_of(supported.begin(), supported.end(),
-                                                 [](bool valid) { return valid; });
-            });
-    ASSERT_TRUE(supportedCall.isOk());
+        // see if service can handle model
+        bool fullySupportsModel = false;
+        Return<void> supportedCall = device->getSupportedOperations_1_1(
+                model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
+                    ASSERT_EQ(ErrorStatus::NONE, status);
+                    ASSERT_NE(0ul, supported.size());
+                    fullySupportsModel = std::all_of(supported.begin(), supported.end(),
+                                                     [](bool valid) { return valid; });
+                });
+        ASSERT_TRUE(supportedCall.isOk());
 
-    // launch prepare model
-    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
-    Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
-            model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
-    ASSERT_TRUE(prepareLaunchStatus.isOk());
-    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
+        // launch prepare model
+        sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+        Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1(
+                model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback);
+        ASSERT_TRUE(prepareLaunchStatus.isOk());
+        ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
 
-    // retrieve prepared model
-    preparedModelCallback->wait();
-    ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
-    sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
+        // retrieve prepared model
+        preparedModelCallback->wait();
+        ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
+        sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
 
-    // early termination if vendor service cannot fully prepare model
-    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
-        ASSERT_EQ(nullptr, preparedModel.get());
-        LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
-                     "prepare model that it does not support.";
-        std::cout << "[          ]   Early termination of test because vendor service cannot "
-                     "prepare model that it does not support."
-                  << std::endl;
-        GTEST_SKIP();
+        // early termination if vendor service cannot fully prepare model
+        if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
+            ASSERT_EQ(nullptr, preparedModel.get());
+            LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
+                         "prepare model that it does not support.";
+            std::cout << "[          ]   Early termination of test because vendor service cannot "
+                         "prepare model that it does not support."
+                      << std::endl;
+            GTEST_SKIP();
+        }
+        EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
+        ASSERT_NE(nullptr, preparedModel.get());
+
+        EvaluatePreparedModel(preparedModel, testModel);
     }
-    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
-    ASSERT_NE(nullptr, preparedModel.get());
+};
 
-    EvaluatePreparedModel(preparedModel, testModel);
+TEST_P(GeneratedTest, Test) {
+    Execute(*mTestModel);
 }
 
-}  // namespace generated_tests
+INSTANTIATE_GENERATED_TEST(GeneratedTest,
+                           [](const TestModel& testModel) { return !testModel.expectFailure; });
+
+}  // namespace functional
+}  // namespace vts
 }  // namespace V1_1
 }  // namespace neuralnetworks
 }  // namespace hardware
diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h
index 56fc825..82fc551 100644
--- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h
+++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h
@@ -19,18 +19,48 @@
 
 #include <android/hardware/neuralnetworks/1.1/IDevice.h>
 #include "TestHarness.h"
+#include "VtsHalNeuralnetworks.h"
 
 namespace android {
 namespace hardware {
 namespace neuralnetworks {
 namespace V1_1 {
-namespace generated_tests {
+namespace vts {
+namespace functional {
+
+class GeneratedTestBase
+    : public NeuralnetworksHidlTest,
+      public ::testing::WithParamInterface<test_helper::TestModelManager::TestParam> {
+  protected:
+    void SetUp() override {
+        NeuralnetworksHidlTest::SetUp();
+        ASSERT_NE(mTestModel, nullptr);
+    }
+
+    const test_helper::TestModel* mTestModel = GetParam().second;
+};
+
+#define INSTANTIATE_GENERATED_TEST(TestSuite, filter)                                          \
+    INSTANTIATE_TEST_SUITE_P(                                                                  \
+            TestGenerated, TestSuite,                                                          \
+            ::testing::ValuesIn(::test_helper::TestModelManager::get().getTestModels(filter)), \
+            [](const auto& info) { return info.param.first; })
+
+// Tag for the validation tests, instantiated in VtsHalNeuralnetworks.cpp.
+// TODO: Clean up the hierarchy for ValidationTest.
+class ValidationTest : public GeneratedTestBase {
+  protected:
+    void validateEverything(const Model& model, const Request& request);
+
+  private:
+    void validateModel(const Model& model);
+    void validateRequest(const sp<IPreparedModel>& preparedModel, const Request& request);
+};
 
 Model createModel(const ::test_helper::TestModel& testModel);
 
-void Execute(const sp<V1_1::IDevice>& device, const ::test_helper::TestModel& testModel);
-
-}  // namespace generated_tests
+}  // namespace functional
+}  // namespace vts
 }  // namespace V1_1
 }  // namespace neuralnetworks
 }  // namespace hardware
diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTests.h b/neuralnetworks/1.1/vts/functional/GeneratedTests.h
deleted file mode 100644
index a55213d..0000000
--- a/neuralnetworks/1.1/vts/functional/GeneratedTests.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (C) 2019 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "1.0/Utils.h"
-#include "GeneratedTestHarness.h"
-#include "TestHarness.h"
-#include "VtsHalNeuralnetworks.h"
-
-namespace android::hardware::neuralnetworks::V1_1::generated_tests {
-
-using namespace android::hardware::neuralnetworks::V1_1::vts::functional;
-
-using ::android::hardware::neuralnetworks::V1_0::OperandLifeTime;
-using ::android::hardware::neuralnetworks::V1_0::Request;
-
-}  // namespace android::hardware::neuralnetworks::V1_1::generated_tests
diff --git a/neuralnetworks/1.1/vts/functional/ValidateModel.cpp b/neuralnetworks/1.1/vts/functional/ValidateModel.cpp
index fb80d13..d20dcd0 100644
--- a/neuralnetworks/1.1/vts/functional/ValidateModel.cpp
+++ b/neuralnetworks/1.1/vts/functional/ValidateModel.cpp
@@ -18,6 +18,7 @@
 
 #include "1.0/Callbacks.h"
 #include "1.0/Utils.h"
+#include "GeneratedTestHarness.h"
 #include "VtsHalNeuralnetworks.h"
 
 namespace android {
diff --git a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
index 757bee9..e0710f1 100644
--- a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
+++ b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp
@@ -18,6 +18,7 @@
 
 #include "1.0/Callbacks.h"
 #include "1.0/Utils.h"
+#include "GeneratedTestHarness.h"
 #include "VtsHalNeuralnetworks.h"
 
 namespace android {
diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp
index b3b15fa..9a11b10 100644
--- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp
+++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp
@@ -17,11 +17,13 @@
 #define LOG_TAG "neuralnetworks_hidl_hal_test"
 
 #include "VtsHalNeuralnetworks.h"
+#include "1.0/Callbacks.h"
+#include "1.0/Utils.h"
+#include "GeneratedTestHarness.h"
+#include "TestHarness.h"
 
 #include <android-base/logging.h>
 
-#include "1.0/Callbacks.h"
-
 namespace android {
 namespace hardware {
 namespace neuralnetworks {
@@ -135,6 +137,15 @@
     validateRequest(preparedModel, request);
 }
 
+TEST_P(ValidationTest, Test) {
+    const Model model = createModel(*mTestModel);
+    const Request request = createRequest(*mTestModel);
+    ASSERT_FALSE(mTestModel->expectFailure);
+    validateEverything(model, request);
+}
+
+INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; });
+
 }  // namespace functional
 }  // namespace vts
 }  // namespace V1_1
diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h
index 2d6a20c..8d44deb 100644
--- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h
+++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.h
@@ -29,6 +29,8 @@
 #include <iostream>
 #include <vector>
 
+#include "TestHarness.h"
+
 namespace android {
 namespace hardware {
 namespace neuralnetworks {
@@ -69,19 +71,6 @@
     sp<IDevice> device;
 };
 
-// Tag for the validation tests
-class ValidationTest : public NeuralnetworksHidlTest {
-   protected:
-     void validateEverything(const Model& model, const Request& request);
-
-   private:
-     void validateModel(const Model& model);
-     void validateRequest(const sp<IPreparedModel>& preparedModel, const Request& request);
-};
-
-// Tag for the generated tests
-class GeneratedTest : public NeuralnetworksHidlTest {};
-
 }  // namespace functional
 }  // namespace vts
 }  // namespace V1_1