pdx: Rework error reporting when transfering file and channel handles

There is a lot of confusion about reporting errors when passing file
and channel handles over PDX transport between client and service.

Methods like Message::PushFileHandle return an integer which means
both a file handle reference value (if positive) and a possible error
code (if negative). But file handles could contain negative values too
(when they are empty). This is used frequently when passing buffer
fences around (when a fence is not being used, its fd is set to -1).

This results in a special case of when PushFileHandle is called with
a file handle with value of -1, the return value is actually "-errno"
which becomes dependent on a global state (errno is not set by
PushFileHandle itself in case file handle value is negative) and results
in unpredicted behavior (sometimes errno is 0, sometimes its >0).

Cleaned this all up by using Status<T> everywhere we used an int to
pass value payload along with possible error code.

Now the semantics of the calls are more clear.

Bug: 36866492
Test: `m -j32` for sailfish-eng succeeds
      Ran unit tests on device (pdx_tests, libpdx_uds_tests, bufferhub_tests,
      buffer_hub_queue-test, buffer_hub_queue_producer-test), all pass
      Ran CubeSea, NativeTreasureHunt and Ithaca on Sailfish with vrflinger
      enabled, was able to use controller and screen rendered correctly.

Change-Id: I0f40c3f356fcba8bc217d5219a0ddf9685e57fd7
diff --git a/libs/vr/libvrflinger/display_manager_service.cpp b/libs/vr/libvrflinger/display_manager_service.cpp
index e07901d..49b6f09 100644
--- a/libs/vr/libvrflinger/display_manager_service.cpp
+++ b/libs/vr/libvrflinger/display_manager_service.cpp
@@ -32,12 +32,12 @@
 namespace dvr {
 
 void DisplayManager::SetNotificationsPending(bool pending) {
-  int ret = service_->ModifyChannelEvents(channel_id_, pending ? 0 : POLLIN,
-                                          pending ? POLLIN : 0);
-  ALOGE_IF(ret < 0,
+  auto status = service_->ModifyChannelEvents(channel_id_, pending ? 0 : POLLIN,
+                                              pending ? POLLIN : 0);
+  ALOGE_IF(!status,
            "DisplayManager::SetNotificationPending: Failed to modify channel "
            "events: %s",
-           strerror(-ret));
+           status.GetErrorMessage().c_str());
 }
 
 DisplayManagerService::DisplayManagerService(
@@ -68,24 +68,24 @@
     display_manager_ = nullptr;
 }
 
-int DisplayManagerService::HandleMessage(pdx::Message& message) {
+pdx::Status<void> DisplayManagerService::HandleMessage(pdx::Message& message) {
   auto channel = std::static_pointer_cast<DisplayManager>(message.GetChannel());
 
   switch (message.GetOp()) {
     case DisplayManagerRPC::GetSurfaceList::Opcode:
       DispatchRemoteMethod<DisplayManagerRPC::GetSurfaceList>(
           *this, &DisplayManagerService::OnGetSurfaceList, message);
-      return 0;
+      return {};
 
     case DisplayManagerRPC::UpdateSurfaces::Opcode:
       DispatchRemoteMethod<DisplayManagerRPC::UpdateSurfaces>(
           *this, &DisplayManagerService::OnUpdateSurfaces, message);
-      return 0;
+      return {};
 
   case DisplayManagerRPC::SetupPoseBuffer::Opcode:
       DispatchRemoteMethod<DisplayManagerRPC::SetupPoseBuffer>(
           *this, &DisplayManagerService::OnSetupPoseBuffer, message);
-      return 0;
+      return {};
 
     default:
       return Service::DefaultHandleMessage(message);
@@ -189,7 +189,7 @@
 }
 
 pdx::BorrowedChannelHandle DisplayManagerService::OnSetupPoseBuffer(
-    pdx::Message& message, size_t extended_region_size, int usage) {
+    pdx::Message& /*message*/, size_t extended_region_size, int usage) {
   return display_service_->SetupPoseBuffer(extended_region_size, usage);
 }
 
diff --git a/libs/vr/libvrflinger/display_manager_service.h b/libs/vr/libvrflinger/display_manager_service.h
index 19098c2..80324fd 100644
--- a/libs/vr/libvrflinger/display_manager_service.h
+++ b/libs/vr/libvrflinger/display_manager_service.h
@@ -42,7 +42,7 @@
   std::shared_ptr<pdx::Channel> OnChannelOpen(pdx::Message& message) override;
   void OnChannelClose(pdx::Message& message,
                       const std::shared_ptr<pdx::Channel>& channel) override;
-  int HandleMessage(pdx::Message& message) override;
+  pdx::Status<void> HandleMessage(pdx::Message& message) override;
 
  private:
   friend BASE;
diff --git a/libs/vr/libvrflinger/display_service.cpp b/libs/vr/libvrflinger/display_service.cpp
index c079187..3750ea3 100644
--- a/libs/vr/libvrflinger/display_service.cpp
+++ b/libs/vr/libvrflinger/display_service.cpp
@@ -65,39 +65,39 @@
 // First-level dispatch for display service messages. Directly handles messages
 // that are independent of the display surface (metrics, creation) and routes
 // surface-specific messages to the per-instance handlers.
-int DisplayService::HandleMessage(pdx::Message& message) {
+pdx::Status<void> DisplayService::HandleMessage(pdx::Message& message) {
   auto channel = message.GetChannel<SurfaceChannel>();
 
   switch (message.GetOp()) {
     case DisplayRPC::GetMetrics::Opcode:
       DispatchRemoteMethod<DisplayRPC::GetMetrics>(
           *this, &DisplayService::OnGetMetrics, message);
-      return 0;
+      return {};
 
     case DisplayRPC::GetEdsCapture::Opcode:
       DispatchRemoteMethod<DisplayRPC::GetEdsCapture>(
           *this, &DisplayService::OnGetEdsCapture, message);
-      return 0;
+      return {};
 
     case DisplayRPC::CreateSurface::Opcode:
       DispatchRemoteMethod<DisplayRPC::CreateSurface>(
           *this, &DisplayService::OnCreateSurface, message);
-      return 0;
+      return {};
 
     case DisplayRPC::SetViewerParams::Opcode:
       DispatchRemoteMethod<DisplayRPC::SetViewerParams>(
           *this, &DisplayService::OnSetViewerParams, message);
-      return 0;
+      return {};
 
     case DisplayRPC::GetPoseBuffer::Opcode:
       DispatchRemoteMethod<DisplayRPC::GetPoseBuffer>(
           *this, &DisplayService::OnGetPoseBuffer, message);
-      return 0;
+      return {};
 
     case DisplayRPC::IsVrAppRunning::Opcode:
       DispatchRemoteMethod<DisplayRPC::IsVrAppRunning>(
           *this, &DisplayService::IsVrAppRunning, message);
-      return 0;
+      return {};
 
     // Direct the surface specific messages to the surface instance.
     case DisplayRPC::CreateBufferQueue::Opcode:
@@ -265,7 +265,7 @@
 
 // Calls the message handler for the DisplaySurface associated with this
 // channel.
-int DisplayService::HandleSurfaceMessage(pdx::Message& message) {
+pdx::Status<void> DisplayService::HandleSurfaceMessage(pdx::Message& message) {
   auto surface = std::static_pointer_cast<SurfaceChannel>(message.GetChannel());
   ALOGW_IF(!surface,
            "DisplayService::HandleSurfaceMessage: surface is nullptr!");
@@ -273,7 +273,7 @@
   if (surface)
     return surface->HandleMessage(message);
   else
-    REPLY_ERROR_RETURN(message, EINVAL, 0);
+    REPLY_ERROR_RETURN(message, EINVAL, {});
 }
 
 std::shared_ptr<DisplaySurface> DisplayService::GetDisplaySurface(
diff --git a/libs/vr/libvrflinger/display_service.h b/libs/vr/libvrflinger/display_service.h
index 8e96172..da80a84 100644
--- a/libs/vr/libvrflinger/display_service.h
+++ b/libs/vr/libvrflinger/display_service.h
@@ -27,7 +27,7 @@
 
   void OnChannelClose(pdx::Message& message,
                       const std::shared_ptr<pdx::Channel>& channel) override;
-  int HandleMessage(pdx::Message& message) override;
+  pdx::Status<void> HandleMessage(pdx::Message& message) override;
 
   std::shared_ptr<DisplaySurface> GetDisplaySurface(int surface_id) const;
   std::vector<std::shared_ptr<DisplaySurface>> GetDisplaySurfaces() const;
@@ -94,7 +94,7 @@
   // the display manager should be notified.
   void NotifyDisplayConfigurationUpdate();
 
-  int HandleSurfaceMessage(pdx::Message& message);
+  pdx::Status<void> HandleSurfaceMessage(pdx::Message& message);
 
   DisplayService(const DisplayService&) = delete;
   void operator=(const DisplayService&) = delete;
diff --git a/libs/vr/libvrflinger/display_surface.cpp b/libs/vr/libvrflinger/display_surface.cpp
index 66e9925..a7220fe 100644
--- a/libs/vr/libvrflinger/display_surface.cpp
+++ b/libs/vr/libvrflinger/display_surface.cpp
@@ -206,7 +206,7 @@
   return !acquired_buffers_.IsEmpty();
 }
 
-int DisplaySurface::HandleMessage(pdx::Message& message) {
+pdx::Status<void> DisplaySurface::HandleMessage(pdx::Message& message) {
   switch (message.GetOp()) {
     case DisplayRPC::SetAttributes::Opcode:
       DispatchRemoteMethod<DisplayRPC::SetAttributes>(
@@ -227,7 +227,7 @@
       return SurfaceChannel::HandleMessage(message);
   }
 
-  return 0;
+  return {};
 }
 
 int DisplaySurface::OnClientSetAttributes(
@@ -301,7 +301,7 @@
     pdx::Message& message) {
   if (flags_ & DVR_DISPLAY_SURFACE_FLAGS_DISABLE_SYSTEM_DISTORTION) {
     ALOGE(
-        "DisplaySurface::OnCreateVideoMeshSurface: system distorion is "
+        "DisplaySurface::OnCreateVideoMeshSurface: system distortion is "
         "disabled on this display surface, cannot create VideoMeshSurface on "
         "top of it.");
     REPLY_ERROR_RETURN(message, EINVAL, {});
@@ -309,22 +309,21 @@
 
   int channel_id;
   auto status = message.PushChannel(0, nullptr, &channel_id);
-
   if (!status) {
     ALOGE(
         "DisplaySurface::OnCreateVideoMeshSurface: failed to push channel: %s",
         status.GetErrorMessage().c_str());
-    REPLY_ERROR_RETURN(message, ENOMEM, {});
+    REPLY_ERROR_RETURN(message, status.error(), {});
   }
 
   auto surface = std::make_shared<VideoMeshSurface>(service(), channel_id);
-  const int ret = service()->SetChannel(channel_id, surface);
-  if (ret < 0) {
+  auto channel_status = service()->SetChannel(channel_id, surface);
+  if (!channel_status) {
     ALOGE(
         "DisplaySurface::OnCreateVideoMeshSurface: failed to set new video "
         "mesh surface channel: %s",
-        strerror(-ret));
-    REPLY_ERROR_RETURN(message, ENOMEM, {});
+        channel_status.GetErrorMessage().c_str());
+    REPLY_ERROR_RETURN(message, channel_status.error(), {});
   }
 
   {
diff --git a/libs/vr/libvrflinger/display_surface.h b/libs/vr/libvrflinger/display_surface.h
index d31a3a9..2e4cf75 100644
--- a/libs/vr/libvrflinger/display_surface.h
+++ b/libs/vr/libvrflinger/display_surface.h
@@ -106,7 +106,7 @@
 
   // Dispatches display surface messages to the appropriate handlers. This
   // handler runs on the displayd message dispatch thread.
-  int HandleMessage(pdx::Message& message) override;
+  pdx::Status<void> HandleMessage(pdx::Message& message) override;
 
   // Sets display surface's client-controlled attributes.
   int OnClientSetAttributes(pdx::Message& message,
diff --git a/libs/vr/libvrflinger/screenshot_service.cpp b/libs/vr/libvrflinger/screenshot_service.cpp
index fd1c582..d14d588 100644
--- a/libs/vr/libvrflinger/screenshot_service.cpp
+++ b/libs/vr/libvrflinger/screenshot_service.cpp
@@ -19,17 +19,17 @@
 
 ScreenshotService::~ScreenshotService() { instance_ = nullptr; }
 
-int ScreenshotService::HandleMessage(pdx::Message& message) {
+pdx::Status<void> ScreenshotService::HandleMessage(pdx::Message& message) {
   switch (message.GetOp()) {
     case DisplayScreenshotRPC::GetFormat::Opcode:
       DispatchRemoteMethod<DisplayScreenshotRPC::GetFormat>(
           *this, &ScreenshotService::OnGetFormat, message);
-      return 0;
+      return {};
 
     case DisplayScreenshotRPC::TakeScreenshot::Opcode:
       DispatchRemoteMethod<DisplayScreenshotRPC::TakeScreenshot>(
           *this, &ScreenshotService::OnTakeScreenshot, message);
-      return 0;
+      return {};
 
     default:
       return Service::HandleMessage(message);
diff --git a/libs/vr/libvrflinger/screenshot_service.h b/libs/vr/libvrflinger/screenshot_service.h
index ec4c527..f59e872 100644
--- a/libs/vr/libvrflinger/screenshot_service.h
+++ b/libs/vr/libvrflinger/screenshot_service.h
@@ -38,7 +38,7 @@
  public:
   ~ScreenshotService();
 
-  int HandleMessage(pdx::Message& message) override;
+  pdx::Status<void> HandleMessage(pdx::Message& message) override;
 
   // Returns true if there is a pending screenshot request.
   bool IsScreenshotRequestPending() const {
diff --git a/libs/vr/libvrflinger/surface_channel.cpp b/libs/vr/libvrflinger/surface_channel.cpp
index 8aa220b..263b382 100644
--- a/libs/vr/libvrflinger/surface_channel.cpp
+++ b/libs/vr/libvrflinger/surface_channel.cpp
@@ -7,7 +7,7 @@
 namespace android {
 namespace dvr {
 
-int SurfaceChannel::HandleMessage(Message& message) {
+pdx::Status<void> SurfaceChannel::HandleMessage(Message& message) {
   switch (message.GetOp()) {
     case DisplayRPC::GetMetadataBuffer::Opcode:
       DispatchRemoteMethod<DisplayRPC::GetMetadataBuffer>(
@@ -15,7 +15,7 @@
       break;
   }
 
-  return 0;
+  return {};
 }
 
 BorrowedChannelHandle SurfaceChannel::OnGetMetadataBuffer(Message& message) {
diff --git a/libs/vr/libvrflinger/surface_channel.h b/libs/vr/libvrflinger/surface_channel.h
index 870e1a4..bb6b1c5 100644
--- a/libs/vr/libvrflinger/surface_channel.h
+++ b/libs/vr/libvrflinger/surface_channel.h
@@ -33,7 +33,7 @@
 
   // Dispatches surface channel messages to the appropriate handlers. This
   // handler runs on the displayd message dispatch thread.
-  virtual int HandleMessage(pdx::Message& message);
+  virtual pdx::Status<void> HandleMessage(pdx::Message& message);
 
  protected:
   // Contains the surface metadata.
diff --git a/libs/vr/libvrflinger/video_mesh_surface.cpp b/libs/vr/libvrflinger/video_mesh_surface.cpp
index a961a3d..d915a4a 100644
--- a/libs/vr/libvrflinger/video_mesh_surface.cpp
+++ b/libs/vr/libvrflinger/video_mesh_surface.cpp
@@ -14,7 +14,7 @@
 
 VideoMeshSurface::~VideoMeshSurface() {}
 
-int VideoMeshSurface::HandleMessage(Message& message) {
+pdx::Status<void> VideoMeshSurface::HandleMessage(Message& message) {
   ATRACE_NAME("VideoMeshSurface::HandleMessage");
 
   switch (message.GetOp()) {
@@ -27,7 +27,7 @@
       return SurfaceChannel::HandleMessage(message);
   }
 
-  return 0;
+  return {};
 }
 
 std::shared_ptr<ConsumerQueue> VideoMeshSurface::GetConsumerQueue() {
diff --git a/libs/vr/libvrflinger/video_mesh_surface.h b/libs/vr/libvrflinger/video_mesh_surface.h
index 1370793..2c9f3e8 100644
--- a/libs/vr/libvrflinger/video_mesh_surface.h
+++ b/libs/vr/libvrflinger/video_mesh_surface.h
@@ -12,7 +12,7 @@
 
 // VideoMeshSurface takes three inputs: 1) buffers filled by Android system
 // components (e.g. MediaCodec or camera stack) other than applications' GL
-// context; 2) a 3D mesh choosen by application to define the shape of the
+// context; 2) a 3D mesh chosen by application to define the shape of the
 // surface; 3) a transformation matrix from application to define the rotation,
 // position, and scaling of the video surface.
 class VideoMeshSurface : public SurfaceChannel {
@@ -33,7 +33,7 @@
     }
   }
 
-  int HandleMessage(Message& message) override;
+  pdx::Status<void> HandleMessage(Message& message) override;
 
   std::shared_ptr<ConsumerQueue> GetConsumerQueue();
 
diff --git a/libs/vr/libvrflinger/vsync_service.cpp b/libs/vr/libvrflinger/vsync_service.cpp
index 48fa2c2..612b9b2 100644
--- a/libs/vr/libvrflinger/vsync_service.cpp
+++ b/libs/vr/libvrflinger/vsync_service.cpp
@@ -107,26 +107,26 @@
   }
 }
 
-int VSyncService::HandleMessage(pdx::Message& message) {
+pdx::Status<void> VSyncService::HandleMessage(pdx::Message& message) {
   switch (message.GetOp()) {
     case DisplayVSyncRPC::Wait::Opcode:
       AddWaiter(message);
-      return 0;
+      return {};
 
     case DisplayVSyncRPC::GetLastTimestamp::Opcode:
       DispatchRemoteMethod<DisplayVSyncRPC::GetLastTimestamp>(
           *this, &VSyncService::OnGetLastTimestamp, message);
-      return 0;
+      return {};
 
     case DisplayVSyncRPC::GetSchedInfo::Opcode:
       DispatchRemoteMethod<DisplayVSyncRPC::GetSchedInfo>(
           *this, &VSyncService::OnGetSchedInfo, message);
-      return 0;
+      return {};
 
     case DisplayVSyncRPC::Acknowledge::Opcode:
       DispatchRemoteMethod<DisplayVSyncRPC::Acknowledge>(
           *this, &VSyncService::OnAcknowledge, message);
-      return 0;
+      return {};
 
     default:
       return Service::HandleMessage(message);
diff --git a/libs/vr/libvrflinger/vsync_service.h b/libs/vr/libvrflinger/vsync_service.h
index ba1d4df..1c86d42 100644
--- a/libs/vr/libvrflinger/vsync_service.h
+++ b/libs/vr/libvrflinger/vsync_service.h
@@ -56,7 +56,7 @@
  public:
   ~VSyncService() override;
 
-  int HandleMessage(pdx::Message& message) override;
+  pdx::Status<void> HandleMessage(pdx::Message& message) override;
 
   std::shared_ptr<pdx::Channel> OnChannelOpen(pdx::Message& message) override;
   void OnChannelClose(pdx::Message& message,