commit 86207bac10e8ad0f0459b92cb9814b165db74896
author Nick Deakin <deakin@google.com> Mon Nov 21 16:07:36 2022 -0500
committer Nick Deakin <deakin@google.com> Mon Nov 21 16:27:52 2022 -0500
tree 49183a44bb4cf888fbee29652ee629fb69554d96
parent dee51fe8b4f8950a04b20b832c5a5a1cd041885b

Some recoverymap fixes from initial Ittiam feedback.

Add missing right shift to account for P010 data format. This is
because the 6 LSBs are ignored for each 16-bit data element.

Also fix potential out-of-bounds indexing in sampleMap() by clamping
x/y_lower/upper values to the map size.

Bug: 252835416
Test: builds
Change-Id: I2064a507a458f1d5d20f3d48eca974a712307d4a

libs/jpegrecoverymap/recoverymapmath.cpp

diff --git a/libs/jpegrecoverymap/recoverymapmath.cpp b/libs/jpegrecoverymap/recoverymapmath.cpp
index 4541f9b..6dcbca3 100644
--- a/libs/jpegrecoverymap/recoverymapmath.cpp
+++ b/libs/jpegrecoverymap/recoverymapmath.cpp
@@ -288,6 +288,10 @@
 // TODO: do we need something more clever for filtering either the map or images
 // to generate the map?
 
+static size_t clamp(const size_t& val, const size_t& low, const size_t& high) {
+  return val < low ? low : (high < val ? high : val);
+}
+
 static float mapUintToFloat(uint8_t map_uint) {
   return (static_cast<float>(map_uint) - 127.5f) / 127.5f;
 }
@@ -301,6 +305,11 @@
   size_t y_lower = static_cast<size_t>(floor(y_map));
   size_t y_upper = y_lower + 1;
 
+  x_lower = clamp(x_lower, 0, map->width - 1);
+  x_upper = clamp(x_upper, 0, map->width - 1);
+  y_lower = clamp(y_lower, 0, map->height - 1);
+  y_upper = clamp(y_upper, 0, map->height - 1);
+
   float x_influence = x_map - static_cast<float>(x_lower);
   float y_influence = y_map - static_cast<float>(y_lower);
 
@@ -338,9 +347,12 @@
   size_t pixel_y_idx = x + y * image->width;
   size_t pixel_uv_idx = x / 2 + (y / 2) * (image->width / 2);
 
-  uint16_t y_uint = reinterpret_cast<uint16_t*>(image->data)[pixel_y_idx];
-  uint16_t u_uint = reinterpret_cast<uint16_t*>(image->data)[pixel_count + pixel_uv_idx * 2];
-  uint16_t v_uint = reinterpret_cast<uint16_t*>(image->data)[pixel_count + pixel_uv_idx * 2 + 1];
+  uint16_t y_uint = reinterpret_cast<uint16_t*>(image->data)[pixel_y_idx]
+                  >> 6;
+  uint16_t u_uint = reinterpret_cast<uint16_t*>(image->data)[pixel_count + pixel_uv_idx * 2]
+                  >> 6;
+  uint16_t v_uint = reinterpret_cast<uint16_t*>(image->data)[pixel_count + pixel_uv_idx * 2 + 1]
+                  >> 6;
 
   // Conversions include taking narrow-range into account.
   return {{{ static_cast<float>(y_uint) / 940.0f,