commit 4d5bb1097a34495212c09473b477dc97acb99264
author Michael Butler <butlermichael@google.com> Mon Feb 26 15:24:46 2018 -0800
committer Michael Butler <butlermichael@google.com> Mon Feb 26 15:24:46 2018 -0800
tree 9aeb2a8eb11ec889c44848573b15e64eaf234f57
parent fe606d5aee56f309368039ad2a3da75b2cc7c288

Relax NeuralNetwork's VTS positive and negative base tests

There are some NN VTS tests that assume a service is able to generate a
model consisting only of a floating point add operation. However, some
drivers do not support floating point operations. This CL relaxes the
test requirements to allow a test to be skipped if the service does not
support floating point add.

Bug: 72764145
Test: mma
Test: VtsHalNeuralnetworksV1_0TargetTest
Change-Id: I6b0644432680fc2f8098b5187795dc2953df03f9

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index f0ce938..8646a4c 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -186,35 +186,29 @@
 
     // see if service can handle model
     bool fullySupportsModel = false;
-    ErrorStatus supportedStatus;
-    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
-    ASSERT_NE(nullptr, preparedModelCallback.get());
-
     Return<void> supportedCall = device->getSupportedOperations(
-        model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
-            supportedStatus = status;
+        model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
+            ASSERT_EQ(ErrorStatus::NONE, status);
             ASSERT_NE(0ul, supported.size());
             fullySupportsModel =
                 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
         });
     ASSERT_TRUE(supportedCall.isOk());
-    ASSERT_EQ(ErrorStatus::NONE, supportedStatus);
+
+    // launch prepare model
+    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+    ASSERT_NE(nullptr, preparedModelCallback.get());
     Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback);
     ASSERT_TRUE(prepareLaunchStatus.isOk());
+    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
 
     // retrieve prepared model
     preparedModelCallback->wait();
     ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
     sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
-    if (fullySupportsModel) {
-        EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
-    } else {
-        EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||
-                    prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);
-    }
 
     // early termination if vendor service cannot fully prepare model
-    if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) {
+    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
         ASSERT_EQ(nullptr, preparedModel.get());
         LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
                      "prepare model that it does not support.";
@@ -223,6 +217,7 @@
                   << std::endl;
         return;
     }
+    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
     ASSERT_NE(nullptr, preparedModel.get());
 
     EvaluatePreparedModel(preparedModel, is_ignored, examples);
@@ -235,36 +230,30 @@
 
     // see if service can handle model
     bool fullySupportsModel = false;
-    ErrorStatus supportedStatus;
-    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
-    ASSERT_NE(nullptr, preparedModelCallback.get());
-
     Return<void> supportedCall = device->getSupportedOperations_1_1(
-        model, [&](ErrorStatus status, const hidl_vec<bool>& supported) {
-            supportedStatus = status;
+        model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
+            ASSERT_EQ(ErrorStatus::NONE, status);
             ASSERT_NE(0ul, supported.size());
             fullySupportsModel =
                 std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; });
         });
     ASSERT_TRUE(supportedCall.isOk());
-    ASSERT_EQ(ErrorStatus::NONE, supportedStatus);
+
+    // launch prepare model
+    sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
+    ASSERT_NE(nullptr, preparedModelCallback.get());
     Return<ErrorStatus> prepareLaunchStatus =
         device->prepareModel_1_1(model, preparedModelCallback);
     ASSERT_TRUE(prepareLaunchStatus.isOk());
+    ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
 
     // retrieve prepared model
     preparedModelCallback->wait();
     ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
     sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel();
-    if (fullySupportsModel) {
-        EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
-    } else {
-        EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||
-                    prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);
-    }
 
     // early termination if vendor service cannot fully prepare model
-    if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) {
+    if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
         ASSERT_EQ(nullptr, preparedModel.get());
         LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
                      "prepare model that it does not support.";
@@ -273,6 +262,7 @@
                   << std::endl;
         return;
     }
+    EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
     ASSERT_NE(nullptr, preparedModel.get());
 
     // If in relaxed mode, set the error range to be 5ULP of FP16.