commit da2fcf6676e8c059b6c7f5dabb68f7590d44922a
author Roman Stratiienko <r.stratiienko@gmail.com> Thu Jan 23 17:47:03 2025 +0200
committer Roman Stratiienko <r.stratiienko@gmail.com> Fri Jan 24 23:33:35 2025 +0200
tree 7dd6e8e5dc911c12be0b2d6962f8c2cd784a49f4
parent 0369cf731fb88aa9b6aa74a20b69b751351db529

drm_hwcomposer: Simplify LayerTransform

Fixes clang-analyzer-optin.core.EnumCastOutOfRange clang tidy check.

Change-Id: I0a88d1ef084848c924198e8bd3831533b6578675
Signed-off-by: Roman Stratiienko <r.stratiienko@gmail.com>

drm/DrmPlane.cpp

diff --git a/drm/DrmPlane.cpp b/drm/DrmPlane.cpp
index dbb5ad6..0010742 100644
--- a/drm/DrmPlane.cpp
+++ b/drm/DrmPlane.cpp
@@ -88,22 +88,8 @@
 
   GetPlaneProperty("zpos", zpos_property_, Presence::kOptional);
 
-  /* DRM/KMS uses counter-clockwise rotations, while HWC API uses
-   * clockwise. That's why 90 and 270 are swapped here.
-   */
   if (GetPlaneProperty("rotation", rotation_property_, Presence::kOptional)) {
-    rotation_property_.AddEnumToMap("rotate-0", LayerTransform::kIdentity,
-                                    transform_enum_map_);
-    rotation_property_.AddEnumToMap("rotate-90", LayerTransform::kRotate270,
-                                    transform_enum_map_);
-    rotation_property_.AddEnumToMap("rotate-180", LayerTransform::kRotate180,
-                                    transform_enum_map_);
-    rotation_property_.AddEnumToMap("rotate-270", LayerTransform::kRotate90,
-                                    transform_enum_map_);
-    rotation_property_.AddEnumToMap("reflect-x", LayerTransform::kFlipH,
-                                    transform_enum_map_);
-    rotation_property_.AddEnumToMap("reflect-y", LayerTransform::kFlipV,
-                                    transform_enum_map_);
+    rotation_property_.GetEnumMask(transform_enum_mask_);
   }
 
   GetPlaneProperty("alpha", alpha_property_, Presence::kOptional);
@@ -166,22 +152,40 @@
   return ((1 << crtc.GetIndexInResArray()) & plane_->possible_crtcs) != 0;
 }
 
+static uint64_t ToDrmRotation(LayerTransform transform) {
+  /* DRM/KMS uses counter-clockwise rotations, while HWC API uses
+   * clockwise. That's why 90 and 270 are swapped here.
+   */
+  uint64_t rotation = DRM_MODE_ROTATE_0;
+
+  if (transform.rotate90) {
+    rotation |= DRM_MODE_ROTATE_270;
+  }
+
+  if (transform.hflip) {
+    rotation |= DRM_MODE_REFLECT_X;
+  }
+
+  if (transform.vflip) {
+    rotation |= DRM_MODE_REFLECT_Y;
+  }
+
+  // TODO(nobody): Respect transform_enum_mask_ to find alternative rotation
+  // values
+
+  return rotation;
+}
+
 bool DrmPlane::IsValidForLayer(LayerData *layer) {
   if (layer == nullptr || !layer->bi) {
     ALOGE("%s: Invalid parameters", __func__);
     return false;
   }
 
-  if (!rotation_property_) {
-    if (layer->pi.transform != LayerTransform::kIdentity) {
-      ALOGV("No rotation property on plane %d", GetId());
-      return false;
-    }
-  } else {
-    if (transform_enum_map_.count(layer->pi.transform) == 0) {
-      ALOGV("Transform is not supported on plane %d", GetId());
-      return false;
-    }
+  uint64_t drm_rotation = ToDrmRotation(layer->pi.transform);
+  if ((drm_rotation & transform_enum_mask_) != drm_rotation) {
+    ALOGV("Transform is not supported on plane %d", GetId());
+    return false;
   }
 
   if (!alpha_property_ && layer->pi.alpha != UINT16_MAX) {
@@ -218,27 +222,6 @@
                           }) != std::end(formats_);
 }
 
-static uint64_t ToDrmRotation(LayerTransform transform) {
-  uint64_t rotation = 0;
-  /* DRM/KMS uses counter-clockwise rotations, while HWC API uses
-   * clockwise. That's why 90 and 270 are swapped here.
-   */
-  if ((transform & LayerTransform::kFlipH) != 0)
-    rotation |= DRM_MODE_REFLECT_X;
-  if ((transform & LayerTransform::kFlipV) != 0)
-    rotation |= DRM_MODE_REFLECT_Y;
-  if ((transform & LayerTransform::kRotate90) != 0)
-    rotation |= DRM_MODE_ROTATE_270;
-  else if ((transform & LayerTransform::kRotate180) != 0)
-    rotation |= DRM_MODE_ROTATE_180;
-  else if ((transform & LayerTransform::kRotate270) != 0)
-    rotation |= DRM_MODE_ROTATE_90;
-  else
-    rotation |= DRM_MODE_ROTATE_0;
-
-  return rotation;
-}
-
 /* Convert float to 16.16 fixed point */
 static int To1616FixPt(float in) {
   constexpr int kBitShift = 16;