commit 13cbc53a65dd84a95a96e852f2cd7a5348cf4762
author Michael Butler <butlermichael@google.com> Wed Oct 04 18:13:52 2017 -0700
committer Michael Butler <butlermichael@google.com> Wed Oct 04 18:14:45 2017 -0700
tree ace2dc2e1293ff9548e0e80332cbc326d29d8408
parent e51f34781a93a771ba3f3be8ac407154526389d3

Early test termination for unsupported models

Some service drivers won't be able to support all operations. In those
cases, try to prepare the model anyway. If it works, proceed with the
test; if it fails, terminate the test early.

For example, the hvx nn service does not support svdf, so it should
attempt to prepare the model, but the test should not fail if the hvx
driver reports an error.

Bug: 63905942
Test: mm, expected behavior when running on hvx service
Change-Id: I51aa1911145f0d17590906c21314d2bdc24cffea

neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
index 8c9a0a3..fc8cc6d 100644
--- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
+++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp
@@ -100,6 +100,17 @@
         EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||
                     prepareReturnStatus == ErrorStatus::GENERAL_FAILURE);
     }
+
+    // early termination if vendor service cannot fully prepare model
+    if (!fullySupportsModel && prepareReturnStatus == ErrorStatus::GENERAL_FAILURE) {
+        ASSERT_EQ(nullptr, preparedModel.get());
+        LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
+                     "prepare model that it does not support.";
+        std::cout << "[          ]   Early termination of test because vendor service cannot "
+                     "prepare model that it does not support."
+                  << std::endl;
+        return;
+    }
     ASSERT_NE(nullptr, preparedModel.get());
 
     int example_no = 1;
@@ -192,17 +203,6 @@
         // retrieve execution status
         executionCallback->wait();
         ErrorStatus executionReturnStatus = executionCallback->getStatus();
-        if (!fullySupportsModel &&
-            static_cast<ErrorStatus>(executionReturnStatus) == ErrorStatus::GENERAL_FAILURE) {
-            LOG(INFO) << "Ignoring execution results from model that is not supported by the "
-                         "vendor service driver";
-            std::cout << "[          ]   Ignoring execution results from model that is not "
-                         "supported by the vendor service driver"
-                      << std::endl;
-            continue;
-        }
-        LOG(INFO) << "CONTINUING TO CHECK VALUE";
-        std::cout << "[          ]   CONTINUING TO CHECK VALUE" << std::endl;
         EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus);
 
         // validate results