drm_hwcomposer: Remove GL compositing support

The GL based compositing adds alot of complexity and was only ever well
tested on closed stacks. It also only supports GLES3.x and still relies
on sw_sync timeline which is now a debugfs feature. Those are just the
known issues.

Removing the GL compositor means everything related to squashing layers
and pre-compositing can be removed. The planner is left as it may be
useful when adding back support for overlay planes. With this change,
only a single plane is supported until ValidateDisplay learns to do
atomic modesetting test for overlay planes.

Tested-by: John Stultz <john.stultz@linaro.org>
Signed-off-by: Rob Herring <robh@kernel.org>
diff --git a/drmdisplaycompositor.cpp b/drmdisplaycompositor.cpp
index 9932fcb..379d80e 100644
--- a/drmdisplaycompositor.cpp
+++ b/drmdisplaycompositor.cpp
@@ -35,153 +35,15 @@
 #include "drmcrtc.h"
 #include "drmplane.h"
 #include "drmresources.h"
-#include "glworker.h"
 
 namespace android {
 
-void SquashState::Init(DrmHwcLayer *layers, size_t num_layers) {
-  generation_number_++;
-  valid_history_ = 0;
-  regions_.clear();
-  last_handles_.clear();
-
-  std::vector<DrmHwcRect<int>> in_rects;
-  for (size_t i = 0; i < num_layers; i++) {
-    DrmHwcLayer *layer = &layers[i];
-    in_rects.emplace_back(layer->display_frame);
-    last_handles_.push_back(layer->sf_handle);
-  }
-
-  std::vector<separate_rects::RectSet<uint64_t, int>> out_regions;
-  separate_rects::separate_rects_64(in_rects, &out_regions);
-
-  for (const separate_rects::RectSet<uint64_t, int> &out_region : out_regions) {
-    regions_.emplace_back();
-    Region &region = regions_.back();
-    region.rect = out_region.rect;
-    region.layer_refs = out_region.id_set.getBits();
-  }
-}
-
-void SquashState::GenerateHistory(DrmHwcLayer *layers, size_t num_layers,
-                                  std::vector<bool> &changed_regions) const {
-  changed_regions.resize(regions_.size());
-  if (num_layers != last_handles_.size()) {
-    ALOGE("SquashState::GenerateHistory expected %zu layers but got %zu layers",
-          last_handles_.size(), num_layers);
-    return;
-  }
-  std::bitset<kMaxLayers> changed_layers;
-  for (size_t i = 0; i < last_handles_.size(); i++) {
-    DrmHwcLayer *layer = &layers[i];
-    // Protected layers can't be squashed so we treat them as constantly
-    // changing.
-    if (layer->protected_usage() || last_handles_[i] != layer->sf_handle)
-      changed_layers.set(i);
-  }
-
-  for (size_t i = 0; i < regions_.size(); i++) {
-    changed_regions[i] = (regions_[i].layer_refs & changed_layers).any();
-  }
-}
-
-void SquashState::StableRegionsWithMarginalHistory(
-    const std::vector<bool> &changed_regions,
-    std::vector<bool> &stable_regions) const {
-  stable_regions.resize(regions_.size());
-  for (size_t i = 0; i < regions_.size(); i++) {
-    stable_regions[i] = !changed_regions[i] && is_stable(i);
-  }
-}
-
-void SquashState::RecordHistory(DrmHwcLayer *layers, size_t num_layers,
-                                const std::vector<bool> &changed_regions) {
-  if (num_layers != last_handles_.size()) {
-    ALOGE("SquashState::RecordHistory expected %zu layers but got %zu layers",
-          last_handles_.size(), num_layers);
-    return;
-  }
-  if (changed_regions.size() != regions_.size()) {
-    ALOGE("SquashState::RecordHistory expected %zu regions but got %zu regions",
-          regions_.size(), changed_regions.size());
-    return;
-  }
-
-  for (size_t i = 0; i < last_handles_.size(); i++) {
-    DrmHwcLayer *layer = &layers[i];
-    last_handles_[i] = layer->sf_handle;
-  }
-
-  for (size_t i = 0; i < regions_.size(); i++) {
-    regions_[i].change_history <<= 1;
-    regions_[i].change_history.set(/* LSB */ 0, changed_regions[i]);
-  }
-
-  valid_history_++;
-}
-
-bool SquashState::RecordAndCompareSquashed(
-    const std::vector<bool> &squashed_regions) {
-  if (squashed_regions.size() != regions_.size()) {
-    ALOGE(
-        "SquashState::RecordAndCompareSquashed expected %zu regions but got "
-        "%zu regions",
-        regions_.size(), squashed_regions.size());
-    return false;
-  }
-  bool changed = false;
-  for (size_t i = 0; i < regions_.size(); i++) {
-    if (regions_[i].squashed != squashed_regions[i]) {
-      regions_[i].squashed = squashed_regions[i];
-      changed = true;
-    }
-  }
-  return changed;
-}
-
-void SquashState::Dump(std::ostringstream *out) const {
-  *out << "----SquashState generation=" << generation_number_
-       << " history=" << valid_history_ << "\n"
-       << "    Regions: count=" << regions_.size() << "\n";
-  for (size_t i = 0; i < regions_.size(); i++) {
-    const Region &region = regions_[i];
-    *out << "      [" << i << "]"
-         << " history=" << region.change_history << " rect";
-    region.rect.Dump(out);
-    *out << " layers=(";
-    bool first = true;
-    for (size_t layer_index = 0; layer_index < kMaxLayers; layer_index++) {
-      if ((region.layer_refs &
-           std::bitset<kMaxLayers>((size_t)1 << layer_index))
-              .any()) {
-        if (!first)
-          *out << " ";
-        first = false;
-        *out << layer_index;
-      }
-    }
-    *out << ")";
-    if (region.squashed)
-      *out << " squashed";
-    *out << "\n";
-  }
-}
-
-static bool UsesSquash(const std::vector<DrmCompositionPlane> &comp_planes) {
-  return std::any_of(comp_planes.begin(), comp_planes.end(),
-                     [](const DrmCompositionPlane &plane) {
-    return plane.type() == DrmCompositionPlane::Type::kSquash;
-  });
-}
-
 DrmDisplayCompositor::DrmDisplayCompositor()
     : drm_(NULL),
       display_(-1),
       initialized_(false),
       active_(false),
       use_hw_overlays_(true),
-      framebuffer_index_(0),
-      squash_framebuffer_index_(0),
       dump_frames_composited_(0),
       dump_last_timestamp_ns_(0) {
   struct timespec ts;
@@ -244,112 +106,6 @@
   return std::make_tuple(mode.h_display(), mode.v_display(), 0);
 }
 
-int DrmDisplayCompositor::PrepareFramebuffer(
-    DrmFramebuffer &fb, DrmDisplayComposition *display_comp) {
-  int ret = fb.WaitReleased(-1);
-  if (ret) {
-    ALOGE("Failed to wait for framebuffer release %d", ret);
-    return ret;
-  }
-  uint32_t width, height;
-  std::tie(width, height, ret) = GetActiveModeResolution();
-  if (ret) {
-    ALOGE(
-        "Failed to allocate framebuffer because the display resolution could "
-        "not be determined %d",
-        ret);
-    return ret;
-  }
-
-  fb.set_release_fence_fd(-1);
-  if (!fb.Allocate(width, height)) {
-    ALOGE("Failed to allocate framebuffer with size %dx%d", width, height);
-    return -ENOMEM;
-  }
-
-  display_comp->layers().emplace_back();
-  DrmHwcLayer &pre_comp_layer = display_comp->layers().back();
-  pre_comp_layer.sf_handle = fb.buffer()->handle;
-  pre_comp_layer.blending = DrmHwcBlending::kPreMult;
-  pre_comp_layer.source_crop = DrmHwcRect<float>(0, 0, width, height);
-  pre_comp_layer.display_frame = DrmHwcRect<int>(0, 0, width, height);
-  ret = pre_comp_layer.buffer.ImportBuffer(fb.buffer()->handle,
-                                           display_comp->importer());
-  if (ret) {
-    ALOGE("Failed to import framebuffer for display %d", ret);
-    return ret;
-  }
-
-  return ret;
-}
-
-int DrmDisplayCompositor::ApplySquash(DrmDisplayComposition *display_comp) {
-  int ret = 0;
-
-  DrmFramebuffer &fb = squash_framebuffers_[squash_framebuffer_index_];
-  ret = PrepareFramebuffer(fb, display_comp);
-  if (ret) {
-    ALOGE("Failed to prepare framebuffer for squash %d", ret);
-    return ret;
-  }
-
-  std::vector<DrmCompositionRegion> &regions = display_comp->squash_regions();
-  ret = pre_compositor_->Composite(display_comp->layers().data(),
-                                   regions.data(), regions.size(), fb.buffer(),
-                                   display_comp->importer());
-  pre_compositor_->Finish();
-
-  if (ret) {
-    ALOGE("Failed to squash layers");
-    return ret;
-  }
-
-  ret = display_comp->CreateNextTimelineFence();
-  if (ret <= 0) {
-    ALOGE("Failed to create squash framebuffer release fence %d", ret);
-    return ret;
-  }
-
-  fb.set_release_fence_fd(ret);
-  display_comp->SignalSquashDone();
-
-  return 0;
-}
-
-int DrmDisplayCompositor::ApplyPreComposite(
-    DrmDisplayComposition *display_comp) {
-  int ret = 0;
-
-  DrmFramebuffer &fb = framebuffers_[framebuffer_index_];
-  ret = PrepareFramebuffer(fb, display_comp);
-  if (ret) {
-    ALOGE("Failed to prepare framebuffer for pre-composite %d", ret);
-    return ret;
-  }
-
-  std::vector<DrmCompositionRegion> &regions = display_comp->pre_comp_regions();
-  ret = pre_compositor_->Composite(display_comp->layers().data(),
-                                   regions.data(), regions.size(), fb.buffer(),
-                                   display_comp->importer());
-  pre_compositor_->Finish();
-
-  if (ret) {
-    ALOGE("Failed to pre-composite layers");
-    return ret;
-  }
-
-  ret = display_comp->CreateNextTimelineFence();
-  if (ret <= 0) {
-    ALOGE("Failed to create pre-composite framebuffer release fence %d", ret);
-    return ret;
-  }
-
-  fb.set_release_fence_fd(ret);
-  display_comp->SignalPreCompDone();
-
-  return 0;
-}
-
 int DrmDisplayCompositor::DisablePlanes(DrmDisplayComposition *display_comp) {
   drmModeAtomicReqPtr pset = drmModeAtomicAlloc();
   if (!pset) {
@@ -384,103 +140,6 @@
   return 0;
 }
 
-int DrmDisplayCompositor::PrepareFrame(DrmDisplayComposition *display_comp) {
-  int ret = 0;
-
-  std::vector<DrmHwcLayer> &layers = display_comp->layers();
-  std::vector<DrmCompositionPlane> &comp_planes =
-      display_comp->composition_planes();
-  std::vector<DrmCompositionRegion> &squash_regions =
-      display_comp->squash_regions();
-  std::vector<DrmCompositionRegion> &pre_comp_regions =
-      display_comp->pre_comp_regions();
-
-  if (!pre_compositor_) {
-    pre_compositor_.reset(new GLWorkerCompositor());
-    int ret = pre_compositor_->Init();
-    if (ret) {
-      ALOGE("Failed to initialize OpenGL compositor %d", ret);
-      return ret;
-    }
-  }
-
-  int squash_layer_index = -1;
-  if (squash_regions.size() > 0) {
-    squash_framebuffer_index_ = (squash_framebuffer_index_ + 1) % 2;
-    ret = ApplySquash(display_comp);
-    if (ret)
-      return ret;
-
-    squash_layer_index = layers.size() - 1;
-  } else {
-    if (UsesSquash(comp_planes)) {
-      DrmFramebuffer &fb = squash_framebuffers_[squash_framebuffer_index_];
-      layers.emplace_back();
-      squash_layer_index = layers.size() - 1;
-      DrmHwcLayer &squash_layer = layers.back();
-      ret = squash_layer.buffer.ImportBuffer(fb.buffer()->handle,
-                                             display_comp->importer());
-      if (ret) {
-        ALOGE("Failed to import old squashed framebuffer %d", ret);
-        return ret;
-      }
-      squash_layer.sf_handle = fb.buffer()->handle;
-      squash_layer.blending = DrmHwcBlending::kPreMult;
-      squash_layer.source_crop = DrmHwcRect<float>(
-          0, 0, squash_layer.buffer->width, squash_layer.buffer->height);
-      squash_layer.display_frame = DrmHwcRect<int>(
-          0, 0, squash_layer.buffer->width, squash_layer.buffer->height);
-      ret = display_comp->CreateNextTimelineFence();
-
-      if (ret <= 0) {
-        ALOGE("Failed to create squash framebuffer release fence %d", ret);
-        return ret;
-      }
-
-      fb.set_release_fence_fd(ret);
-      ret = 0;
-    }
-  }
-
-  bool do_pre_comp = pre_comp_regions.size() > 0;
-  int pre_comp_layer_index = -1;
-  if (do_pre_comp) {
-    ret = ApplyPreComposite(display_comp);
-    if (ret)
-      return ret;
-
-    pre_comp_layer_index = layers.size() - 1;
-    framebuffer_index_ = (framebuffer_index_ + 1) % DRM_DISPLAY_BUFFERS;
-  }
-
-  for (DrmCompositionPlane &comp_plane : comp_planes) {
-    std::vector<size_t> &source_layers = comp_plane.source_layers();
-    switch (comp_plane.type()) {
-      case DrmCompositionPlane::Type::kSquash:
-        if (source_layers.size())
-          ALOGE("Squash source_layers is expected to be empty (%zu/%d)",
-                source_layers[0], squash_layer_index);
-        source_layers.push_back(squash_layer_index);
-        break;
-      case DrmCompositionPlane::Type::kPrecomp:
-        if (!do_pre_comp) {
-          ALOGE(
-              "Can not use pre composite framebuffer with no pre composite "
-              "regions");
-          return -EINVAL;
-        }
-        // Replace source_layers with the output of the precomposite
-        source_layers.clear();
-        source_layers.push_back(pre_comp_layer_index);
-        break;
-      default:
-        break;
-    }
-  }
-
-  return ret;
-}
-
 int DrmDisplayCompositor::CommitFrame(DrmDisplayComposition *display_comp,
                                       bool test_only) {
   ATRACE_CALL();
@@ -778,8 +437,6 @@
   if (DisablePlanes(active_composition_.get()))
     return;
 
-  active_composition_->SignalCompositionDone();
-
   active_composition_.reset(NULL);
 }
 
@@ -799,9 +456,6 @@
   }
   ++dump_frames_composited_;
 
-  if (active_composition_)
-    active_composition_->SignalCompositionDone();
-
   ret = pthread_mutex_lock(&lock_);
   if (ret)
     ALOGE("Failed to acquire lock for active_composition swap");
@@ -819,38 +473,16 @@
   int ret = 0;
   switch (composition->type()) {
     case DRM_COMPOSITION_TYPE_FRAME:
-      ret = PrepareFrame(composition.get());
-      if (ret) {
-        ALOGE("Failed to prepare frame for display %d", display_);
-        return ret;
-      }
       if (composition->geometry_changed()) {
         // Send the composition to the kernel to ensure we can commit it. This
-        // is just a test, it won't actually commit the frame. If rejected,
-        // squash the frame into one layer and use the squashed composition
+        // is just a test, it won't actually commit the frame.
         ret = CommitFrame(composition.get(), true);
-        if (ret)
-          ALOGI("Commit test failed, squashing frame for display %d", display_);
-        use_hw_overlays_ = !ret;
-      }
-
-      // If use_hw_overlays_ is false, we can't use hardware to composite the
-      // frame. So squash all layers into a single composition and apply that
-      // instead.
-      if (!use_hw_overlays_) {
-        std::unique_ptr<DrmDisplayComposition> squashed = CreateComposition();
-        ret = SquashFrame(composition.get(), squashed.get());
-        if (!ret) {
-          composition = std::move(squashed);
-        } else {
-          ALOGE("Failed to squash frame for display %d", display_);
-          // Disable the hw used by the last active composition. This allows us
-          // to signal the release fences from that composition to avoid
-          // hanging.
-          ClearDisplay();
+        if (ret) {
+          ALOGE("Commit test failed for display %d, FIXME", display_);
           return ret;
         }
       }
+
       ApplyFrame(std::move(composition), ret);
       break;
     case DRM_COMPOSITION_TYPE_DPMS:
@@ -878,154 +510,6 @@
   return ret;
 }
 
-int DrmDisplayCompositor::SquashAll() {
-  AutoLock lock(&lock_, "compositor");
-  int ret = lock.Lock();
-  if (ret)
-    return ret;
-
-  if (!active_composition_)
-    return 0;
-
-  std::unique_ptr<DrmDisplayComposition> comp = CreateComposition();
-  ret = SquashFrame(active_composition_.get(), comp.get());
-
-  // ApplyFrame needs the lock
-  lock.Unlock();
-
-  if (!ret)
-    ApplyFrame(std::move(comp), 0);
-
-  return ret;
-}
-
-// Returns:
-//   - 0 if src is successfully squashed into dst
-//   - -EALREADY if the src is already squashed
-//   - Appropriate error if the squash fails
-int DrmDisplayCompositor::SquashFrame(DrmDisplayComposition *src,
-                                      DrmDisplayComposition *dst) {
-  if (src->type() != DRM_COMPOSITION_TYPE_FRAME)
-    return -ENOTSUP;
-
-  std::vector<DrmCompositionPlane> &src_planes = src->composition_planes();
-  std::vector<DrmHwcLayer> &src_layers = src->layers();
-
-  // Make sure there is more than one layer to squash.
-  size_t src_planes_with_layer = std::count_if(
-      src_planes.begin(), src_planes.end(), [](DrmCompositionPlane &p) {
-        return p.type() != DrmCompositionPlane::Type::kDisable;
-      });
-  if (src_planes_with_layer <= 1)
-    return -EALREADY;
-
-  int pre_comp_layer_index;
-
-  int ret = dst->Init(drm_, src->crtc(), src->importer(), src->planner(),
-                      src->frame_no());
-  if (ret) {
-    ALOGE("Failed to init squash all composition %d", ret);
-    return ret;
-  }
-
-  DrmCompositionPlane squashed_comp(DrmCompositionPlane::Type::kPrecomp, NULL,
-                                    src->crtc());
-  std::vector<DrmHwcLayer> dst_layers;
-  for (DrmCompositionPlane &comp_plane : src_planes) {
-    // Composition planes without DRM planes should never happen
-    if (comp_plane.plane() == NULL) {
-      ALOGE("Skipping squash all because of NULL plane");
-      ret = -EINVAL;
-      goto move_layers_back;
-    }
-
-    if (comp_plane.plane()->type() == DRM_PLANE_TYPE_PRIMARY)
-      squashed_comp.set_plane(comp_plane.plane());
-    else
-      dst->AddPlaneDisable(comp_plane.plane());
-
-    if (comp_plane.type() == DrmCompositionPlane::Type::kDisable)
-      continue;
-
-    for (auto i : comp_plane.source_layers()) {
-      DrmHwcLayer &layer = src_layers[i];
-
-      // Squashing protected layers is impossible.
-      if (layer.protected_usage()) {
-        ret = -ENOTSUP;
-        goto move_layers_back;
-      }
-
-      // The OutputFds point to freed memory after hwc_set returns. They are
-      // returned to the default to prevent DrmDisplayComposition::Plan from
-      // filling the OutputFds.
-      layer.release_fence = OutputFd();
-      dst_layers.emplace_back(std::move(layer));
-      squashed_comp.source_layers().push_back(
-          squashed_comp.source_layers().size());
-    }
-  }
-
-  if (squashed_comp.plane() == NULL) {
-    ALOGE("Primary plane not found for squash");
-    ret = -ENOTSUP;
-    goto move_layers_back;
-  }
-
-  ret = dst->SetLayers(dst_layers.data(), dst_layers.size(), false);
-  if (ret) {
-    ALOGE("Failed to set layers for squash all composition %d", ret);
-    goto move_layers_back;
-  }
-
-  ret = dst->AddPlaneComposition(std::move(squashed_comp));
-  if (ret) {
-    ALOGE("Failed to add squashed plane composition %d", ret);
-    goto move_layers_back;
-  }
-
-  ret = dst->FinalizeComposition();
-  if (ret) {
-    ALOGE("Failed to plan for squash all composition %d", ret);
-    goto move_layers_back;
-  }
-
-  ret = ApplyPreComposite(dst);
-  if (ret) {
-    ALOGE("Failed to pre-composite for squash all composition %d", ret);
-    goto move_layers_back;
-  }
-
-  pre_comp_layer_index = dst->layers().size() - 1;
-  framebuffer_index_ = (framebuffer_index_ + 1) % DRM_DISPLAY_BUFFERS;
-
-  for (DrmCompositionPlane &plane : dst->composition_planes()) {
-    if (plane.type() == DrmCompositionPlane::Type::kPrecomp) {
-      // Replace source_layers with the output of the precomposite
-      plane.source_layers().clear();
-      plane.source_layers().push_back(pre_comp_layer_index);
-      break;
-    }
-  }
-
-  return 0;
-
-// TODO(zachr): think of a better way to transfer ownership back to the active
-// composition.
-move_layers_back:
-  for (size_t plane_index = 0;
-       plane_index < src_planes.size() && plane_index < dst_layers.size();) {
-    if (src_planes[plane_index].source_layers().empty()) {
-      plane_index++;
-      continue;
-    }
-    for (auto i : src_planes[plane_index].source_layers())
-      src_layers[i] = std::move(dst_layers[plane_index++]);
-  }
-
-  return ret;
-}
-
 void DrmDisplayCompositor::Dump(std::ostringstream *out) const {
   int ret = pthread_mutex_lock(&lock_);
   if (ret)
@@ -1051,11 +535,6 @@
 
   dump_last_timestamp_ns_ = cur_ts;
 
-  if (active_composition_)
-    active_composition_->Dump(out);
-
-  squash_state_.Dump(out);
-
   pthread_mutex_unlock(&lock_);
 }
 }