Change NNAPI VTS to use TEST_P to iterate across all service instances

This CL removes a dependency on the VTS test runner by dynamically
discovering all NN HAL service instances in the gtest binary itself,
and runs through all service instances with parameterized tests.

This CL converts TEST_F cases to TEST_P cases, where the test parameter
is the name of the service instance. For existing TEST_P cases (such as
the generated test cases), the service instance name is made to be the
first test parameter.

This CL enables the NN VTS tests to be more portable, e.g., they can
run directly as a presubmit test.

Fixes: 124540002
Test: mma
Test: VtsHalNeuralnetworksV1_*TargetTest (with sample-all)
Test: cd $ANDROID_BUILD_TOP/hardware/interfaces/neuralnetworks && atest
Change-Id: I1e301d7c9f9342bb8f35a267bef180f510944b19
diff --git a/neuralnetworks/1.0/vts/functional/Android.bp b/neuralnetworks/1.0/vts/functional/Android.bp
index 0af7f79..3e9d5f7 100644
--- a/neuralnetworks/1.0/vts/functional/Android.bp
+++ b/neuralnetworks/1.0/vts/functional/Android.bp
@@ -40,10 +40,11 @@
     ],
 }
 
-cc_defaults {
-    name: "VtsHalNeuralNetworksV1_0TargetTestDefaults",
+cc_test {
+    name: "VtsHalNeuralnetworksV1_0TargetTest",
     defaults: ["VtsHalTargetTestDefaults"],
     srcs: [
+        "BasicTests.cpp",
         "TestAssertions.cpp",
         "ValidateModel.cpp",
         "ValidateRequest.cpp",
@@ -64,33 +65,11 @@
         "libneuralnetworks_utils",
         "VtsHalNeuralNetworksV1_0_utils",
     ],
+    whole_static_libs: [
+        "neuralnetworks_generated_V1_0_example",
+    ],
     header_libs: [
         "libneuralnetworks_headers",
     ],
     test_suites: ["general-tests"],
 }
-
-cc_test {
-    name: "VtsHalNeuralnetworksV1_0TargetTest",
-    defaults: ["VtsHalNeuralNetworksV1_0TargetTestDefaults"],
-    srcs: [
-        "BasicTests.cpp",
-    ],
-    whole_static_libs: [
-        "neuralnetworks_generated_V1_0_example",
-    ],
-}
-
-cc_test {
-    name: "PresubmitHalNeuralnetworksV1_0TargetTest",
-    defaults: ["VtsHalNeuralNetworksV1_0TargetTestDefaults"],
-    srcs: [
-        "BasicTests.cpp",
-    ],
-    whole_static_libs: [
-        "neuralnetworks_generated_V1_0_example",
-    ],
-    cflags: [
-        "-DPRESUBMIT_NOT_VTS",
-    ],
-}
diff --git a/neuralnetworks/1.0/vts/functional/BasicTests.cpp b/neuralnetworks/1.0/vts/functional/BasicTests.cpp
index 551ea67..cc44c9e 100644
--- a/neuralnetworks/1.0/vts/functional/BasicTests.cpp
+++ b/neuralnetworks/1.0/vts/functional/BasicTests.cpp
@@ -21,17 +21,17 @@
 namespace android::hardware::neuralnetworks::V1_0::vts::functional {
 
 // create device test
-TEST_F(NeuralnetworksHidlTest, CreateDevice) {}
+TEST_P(NeuralnetworksHidlTest, CreateDevice) {}
 
 // status test
-TEST_F(NeuralnetworksHidlTest, StatusTest) {
+TEST_P(NeuralnetworksHidlTest, StatusTest) {
     Return<DeviceStatus> status = kDevice->getStatus();
     ASSERT_TRUE(status.isOk());
     EXPECT_EQ(DeviceStatus::AVAILABLE, static_cast<DeviceStatus>(status));
 }
 
 // initialization
-TEST_F(NeuralnetworksHidlTest, GetCapabilitiesTest) {
+TEST_P(NeuralnetworksHidlTest, GetCapabilitiesTest) {
     Return<void> ret =
             kDevice->getCapabilities([](ErrorStatus status, const Capabilities& capabilities) {
                 EXPECT_EQ(ErrorStatus::NONE, status);
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index 1948c05..595ad85 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -148,6 +148,20 @@
     checkResults(testModel, outputs);
 }
 
+void GeneratedTestBase::SetUp() {
+    testing::TestWithParam<GeneratedTestParam>::SetUp();
+    ASSERT_NE(kDevice, nullptr);
+}
+
+std::vector<NamedModel> getNamedModels(const FilterFn& filter) {
+    return TestModelManager::get().getTestModels(filter);
+}
+
+std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info) {
+    const auto& [namedDevice, namedModel] = info.param;
+    return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel));
+}
+
 // Tag for the generated tests
 class GeneratedTest : public GeneratedTestBase {};
 
diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h
index 10e46b7..f230a02 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h
@@ -18,29 +18,38 @@
 #define ANDROID_HARDWARE_NEURALNETWORKS_V1_0_GENERATED_TEST_HARNESS_H
 
 #include <android/hardware/neuralnetworks/1.0/IDevice.h>
+#include <functional>
 #include "TestHarness.h"
 #include "VtsHalNeuralnetworks.h"
 
 namespace android::hardware::neuralnetworks::V1_0::vts::functional {
 
-class GeneratedTestBase
-    : public NeuralnetworksHidlTest,
-      public testing::WithParamInterface<test_helper::TestModelManager::TestParam> {
+using NamedModel = Named<const test_helper::TestModel*>;
+using GeneratedTestParam = std::tuple<NamedDevice, NamedModel>;
+
+class GeneratedTestBase : public testing::TestWithParam<GeneratedTestParam> {
   protected:
-    const test_helper::TestModel& kTestModel = *GetParam().second;
+    void SetUp() override;
+    const sp<IDevice> kDevice = getData(std::get<NamedDevice>(GetParam()));
+    const test_helper::TestModel& kTestModel = *getData(std::get<NamedModel>(GetParam()));
 };
 
-#define INSTANTIATE_GENERATED_TEST(TestSuite, filter)                                        \
-    INSTANTIATE_TEST_SUITE_P(                                                                \
-            TestGenerated, TestSuite,                                                        \
-            testing::ValuesIn(::test_helper::TestModelManager::get().getTestModels(filter)), \
-            [](const auto& info) { return info.param.first; })
+using FilterFn = std::function<bool(const test_helper::TestModel&)>;
+std::vector<NamedModel> getNamedModels(const FilterFn& filter);
+
+std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info);
+
+#define INSTANTIATE_GENERATED_TEST(TestSuite, filter)                                     \
+    INSTANTIATE_TEST_SUITE_P(TestGenerated, TestSuite,                                    \
+                             testing::Combine(testing::ValuesIn(getNamedDevices()),       \
+                                              testing::ValuesIn(getNamedModels(filter))), \
+                             printGeneratedTest)
 
 // Tag for the validation tests, instantiated in VtsHalNeuralnetworks.cpp.
 // TODO: Clean up the hierarchy for ValidationTest.
 class ValidationTest : public GeneratedTestBase {};
 
-Model createModel(const ::test_helper::TestModel& testModel);
+Model createModel(const test_helper::TestModel& testModel);
 
 }  // namespace android::hardware::neuralnetworks::V1_0::vts::functional
 
diff --git a/neuralnetworks/1.0/vts/functional/Utils.cpp b/neuralnetworks/1.0/vts/functional/Utils.cpp
index 307003c..5b630fd 100644
--- a/neuralnetworks/1.0/vts/functional/Utils.cpp
+++ b/neuralnetworks/1.0/vts/functional/Utils.cpp
@@ -117,6 +117,13 @@
     return outputBuffers;
 }
 
+std::string gtestCompliantName(std::string name) {
+    // gtest test names must only contain alphanumeric characters
+    std::replace_if(
+            name.begin(), name.end(), [](char c) { return !std::isalnum(c); }, '_');
+    return name;
+}
+
 }  // namespace android::hardware::neuralnetworks
 
 namespace android::hardware::neuralnetworks::V1_0 {
diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp
index 20b4565..cb22250 100644
--- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp
+++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp
@@ -18,11 +18,13 @@
 
 #include "VtsHalNeuralnetworks.h"
 #include "1.0/Callbacks.h"
-#include "1.0/Utils.h"
 #include "GeneratedTestHarness.h"
 #include "TestHarness.h"
 
 #include <android-base/logging.h>
+#include <hidl/ServiceManagement.h>
+#include <string>
+#include <utility>
 
 namespace android::hardware::neuralnetworks::V1_0::vts::functional {
 
@@ -76,34 +78,39 @@
     ASSERT_NE(nullptr, preparedModel->get());
 }
 
-// A class for test environment setup
-NeuralnetworksHidlEnvironment* NeuralnetworksHidlEnvironment::getInstance() {
-    // This has to return a "new" object because it is freed inside
-    // testing::AddGlobalTestEnvironment when the gtest is being torn down
-    static NeuralnetworksHidlEnvironment* instance = new NeuralnetworksHidlEnvironment();
-    return instance;
-}
-
-void NeuralnetworksHidlEnvironment::registerTestServices() {
-    registerTestService<IDevice>();
-}
-
-// The main test class for NEURALNETWORK HIDL HAL.
 void NeuralnetworksHidlTest::SetUp() {
-    testing::VtsHalHidlTargetTestBase::SetUp();
-
-#ifdef PRESUBMIT_NOT_VTS
-    const std::string name =
-            NeuralnetworksHidlEnvironment::getInstance()->getServiceName<IDevice>();
-    const std::string sampleDriver = "sample-";
-    if (kDevice == nullptr && name.substr(0, sampleDriver.size()) == sampleDriver) {
-        GTEST_SKIP();
-    }
-#endif  // PRESUBMIT_NOT_VTS
-
-    ASSERT_NE(nullptr, kDevice.get());
+    testing::TestWithParam<NeuralnetworksHidlTestParam>::SetUp();
+    ASSERT_NE(kDevice, nullptr);
 }
 
+static NamedDevice makeNamedDevice(const std::string& name) {
+    return {name, IDevice::getService(name)};
+}
+
+static std::vector<NamedDevice> getNamedDevicesImpl() {
+    // Retrieves the name of all service instances that implement IDevice,
+    // including any Lazy HAL instances.
+    const std::vector<std::string> names = hardware::getAllHalInstanceNames(IDevice::descriptor);
+
+    // Get a handle to each device and pair it with its name.
+    std::vector<NamedDevice> namedDevices;
+    namedDevices.reserve(names.size());
+    std::transform(names.begin(), names.end(), std::back_inserter(namedDevices), makeNamedDevice);
+    return namedDevices;
+}
+
+const std::vector<NamedDevice>& getNamedDevices() {
+    const static std::vector<NamedDevice> devices = getNamedDevicesImpl();
+    return devices;
+}
+
+std::string printNeuralnetworksHidlTest(
+        const testing::TestParamInfo<NeuralnetworksHidlTestParam>& info) {
+    return gtestCompliantName(getName(info.param));
+}
+
+INSTANTIATE_DEVICE_TEST(NeuralnetworksHidlTest);
+
 // Forward declaration from ValidateModel.cpp
 void validateModel(const sp<IDevice>& device, const Model& model);
 // Forward declaration from ValidateRequest.cpp
@@ -130,14 +137,3 @@
 INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; });
 
 }  // namespace android::hardware::neuralnetworks::V1_0::vts::functional
-
-using android::hardware::neuralnetworks::V1_0::vts::functional::NeuralnetworksHidlEnvironment;
-
-int main(int argc, char** argv) {
-    testing::AddGlobalTestEnvironment(NeuralnetworksHidlEnvironment::getInstance());
-    testing::InitGoogleTest(&argc, argv);
-    NeuralnetworksHidlEnvironment::getInstance()->init(&argc, argv);
-
-    int status = RUN_ALL_TESTS();
-    return status;
-}
diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.h b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.h
index 48dc237..17f4613 100644
--- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.h
+++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.h
@@ -17,40 +17,34 @@
 #ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_0_VTS_HAL_NEURALNETWORKS_H
 #define ANDROID_HARDWARE_NEURALNETWORKS_V1_0_VTS_HAL_NEURALNETWORKS_H
 
+#include "1.0/Utils.h"
+
 #include <android/hardware/neuralnetworks/1.0/IDevice.h>
 #include <android/hardware/neuralnetworks/1.0/types.h>
-
-#include <VtsHalHidlTargetTestBase.h>
-#include <VtsHalHidlTargetTestEnvBase.h>
-
-#include <android-base/macros.h>
 #include <gtest/gtest.h>
 
+#include <vector>
+
 namespace android::hardware::neuralnetworks::V1_0::vts::functional {
 
-// A class for test environment setup
-class NeuralnetworksHidlEnvironment : public testing::VtsHalHidlTargetTestEnvBase {
-    DISALLOW_COPY_AND_ASSIGN(NeuralnetworksHidlEnvironment);
-    NeuralnetworksHidlEnvironment() = default;
+using NamedDevice = Named<sp<IDevice>>;
+using NeuralnetworksHidlTestParam = NamedDevice;
 
-  public:
-    static NeuralnetworksHidlEnvironment* getInstance();
-    void registerTestServices() override;
-};
-
-// The main test class for NEURALNETWORKS HIDL HAL.
-class NeuralnetworksHidlTest : public testing::VtsHalHidlTargetTestBase {
-    DISALLOW_COPY_AND_ASSIGN(NeuralnetworksHidlTest);
-
-  public:
-    NeuralnetworksHidlTest() = default;
-    void SetUp() override;
-
+class NeuralnetworksHidlTest : public testing::TestWithParam<NeuralnetworksHidlTestParam> {
   protected:
-    const sp<IDevice> kDevice = testing::VtsHalHidlTargetTestBase::getService<IDevice>(
-            NeuralnetworksHidlEnvironment::getInstance());
+    void SetUp() override;
+    const sp<IDevice> kDevice = getData(GetParam());
 };
 
+const std::vector<NamedDevice>& getNamedDevices();
+
+std::string printNeuralnetworksHidlTest(
+        const testing::TestParamInfo<NeuralnetworksHidlTestParam>& info);
+
+#define INSTANTIATE_DEVICE_TEST(TestSuite)                                                 \
+    INSTANTIATE_TEST_SUITE_P(PerInstance, TestSuite, testing::ValuesIn(getNamedDevices()), \
+                             printNeuralnetworksHidlTest)
+
 // Create an IPreparedModel object. If the model cannot be prepared,
 // "preparedModel" will be nullptr instead.
 void createPreparedModel(const sp<IDevice>& device, const Model& model,
diff --git a/neuralnetworks/1.0/vts/functional/include/1.0/Utils.h b/neuralnetworks/1.0/vts/functional/include/1.0/Utils.h
index 1ce751c..6d4534c 100644
--- a/neuralnetworks/1.0/vts/functional/include/1.0/Utils.h
+++ b/neuralnetworks/1.0/vts/functional/include/1.0/Utils.h
@@ -21,13 +21,15 @@
 #include <android/hardware/neuralnetworks/1.0/types.h>
 #include <algorithm>
 #include <iosfwd>
+#include <string>
+#include <utility>
 #include <vector>
 #include "TestHarness.h"
 
 namespace android::hardware::neuralnetworks {
 
 // Create HIDL Request from the TestModel struct.
-V1_0::Request createRequest(const ::test_helper::TestModel& testModel);
+V1_0::Request createRequest(const test_helper::TestModel& testModel);
 
 // After execution, copy out output results from the output memory pool.
 std::vector<::test_helper::TestBuffer> getOutputBuffers(const V1_0::Request& request);
@@ -51,6 +53,21 @@
     return index;
 }
 
+template <typename Type>
+using Named = std::pair<std::string, Type>;
+
+template <typename Type>
+const std::string& getName(const Named<Type>& namedData) {
+    return namedData.first;
+}
+
+template <typename Type>
+const Type& getData(const Named<Type>& namedData) {
+    return namedData.second;
+}
+
+std::string gtestCompliantName(std::string name);
+
 }  // namespace android::hardware::neuralnetworks
 
 namespace android::hardware::neuralnetworks::V1_0 {