Several fixes to recovery map math.
* Fix incorrect conversion conversion from RGB to YUV using bt.2100 when
the data is actually assumed to be srgb (although this will get
improved further soon) in map application.
* Fix incorrectly calculating luminance directly from luma in map
generation; we need to convert from YUV to RGB first. This also
involved updating the sampling functions to sample entire pixels.
* Some cleanup to structure of recoverymapmath files.
* Call all public lib functions in recovermap_test to force linking, and
therefore verify that the lib actually builds properly.
Test: builds (including the test, which verifies linking now)
Bug: 252835416
No-Typo-Check: incorrectly interpretting code as a comment
Change-Id: I13565145dc6efcdf642981a1e52420d662152737
diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h
index ef9112e..e76bc20 100644
--- a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h
+++ b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h
@@ -23,6 +23,9 @@
namespace android::recoverymap {
+////////////////////////////////////////////////////////////////////////////////
+// Framework
+
const float kSdrWhiteNits = 100.0f;
struct Color {
@@ -40,35 +43,142 @@
};
};
-/*
- * Convert from OETF'd bt.2100 RGB to YUV, according to BT.2100
- */
-Color bt2100RgbToYuv(Color e);
+inline Color operator+=(Color& lhs, const Color& rhs) {
+ lhs.r += rhs.r;
+ lhs.g += rhs.g;
+ lhs.b += rhs.b;
+ return lhs;
+}
+inline Color operator-=(Color& lhs, const Color& rhs) {
+ lhs.r -= rhs.r;
+ lhs.g -= rhs.g;
+ lhs.b -= rhs.b;
+ return lhs;
+}
+
+inline Color operator+(const Color& lhs, const Color& rhs) {
+ Color temp = lhs;
+ return temp += rhs;
+}
+inline Color operator-(const Color& lhs, const Color& rhs) {
+ Color temp = lhs;
+ return temp -= rhs;
+}
+
+inline Color operator+=(Color& lhs, const float rhs) {
+ lhs.r += rhs;
+ lhs.g += rhs;
+ lhs.b += rhs;
+ return lhs;
+}
+inline Color operator-=(Color& lhs, const float rhs) {
+ lhs.r -= rhs;
+ lhs.g -= rhs;
+ lhs.b -= rhs;
+ return lhs;
+}
+inline Color operator*=(Color& lhs, const float rhs) {
+ lhs.r *= rhs;
+ lhs.g *= rhs;
+ lhs.b *= rhs;
+ return lhs;
+}
+inline Color operator/=(Color& lhs, const float rhs) {
+ lhs.r /= rhs;
+ lhs.g /= rhs;
+ lhs.b /= rhs;
+ return lhs;
+}
+
+inline Color operator+(const Color& lhs, const float rhs) {
+ Color temp = lhs;
+ return temp += rhs;
+}
+inline Color operator-(const Color& lhs, const float rhs) {
+ Color temp = lhs;
+ return temp -= rhs;
+}
+inline Color operator*(const Color& lhs, const float rhs) {
+ Color temp = lhs;
+ return temp *= rhs;
+}
+inline Color operator/(const Color& lhs, const float rhs) {
+ Color temp = lhs;
+ return temp /= rhs;
+}
+
+
+////////////////////////////////////////////////////////////////////////////////
+// sRGB transformations
/*
- * Convert srgb YUV to RGB, according to ECMA TR/98.
+ * Calculate the luminance of a linear RGB sRGB pixel, according to IEC 61966-2-1.
*/
-Color srgbYuvToRgb(Color e);
+float srgbLuminance(Color e);
/*
- * TODO: better source for srgb transfer function
- * Convert from srgb to linear, according to https://en.wikipedia.org/wiki/SRGB.
+ * Convert from OETF'd srgb YUV to RGB, according to ECMA TR/98.
+ */
+Color srgbYuvToRgb(Color e_gamma);
+
+/*
+ * Convert from OETF'd srgb RGB to YUV, according to ECMA TR/98.
+ */
+Color srgbRgbToYuv(Color e_gamma);
+
+/*
+ * Convert from srgb to linear, according to IEC 61966-2-1.
+ *
* [0.0, 1.0] range in and out.
*/
-float srgbInvOetf(float e);
-Color srgbInvOetf(Color e);
+float srgbInvOetf(float e_gamma);
+Color srgbInvOetf(Color e_gamma);
+
+
+////////////////////////////////////////////////////////////////////////////////
+// Display-P3 transformations
+
+
+
+
+////////////////////////////////////////////////////////////////////////////////
+// BT.2100 transformations - according to ITU-R BT.2100-2
/*
- * Convert from HLG to scene luminance in nits, according to BT.2100.
+ * Calculate the luminance of a linear RGB BT.2100 pixel.
*/
-float hlgInvOetf(float e);
+float bt2100Luminance(Color e);
/*
- * Convert from scene luminance in nits to HLG, according to BT.2100.
+ * Convert from OETF'd BT.2100 RGB to YUV.
+ */
+Color bt2100RgbToYuv(Color e_gamma);
+
+/*
+ * Convert from OETF'd BT.2100 YUV to RGB.
+ */
+Color bt2100YuvToRgb(Color e_gamma);
+
+/*
+ * Convert from scene luminance in nits to HLG.
*/
Color hlgOetf(Color e);
/*
+ * Convert from HLG to scene luminance in nits.
+ */
+Color hlgInvOetf(Color e_gamma);
+
+////////////////////////////////////////////////////////////////////////////////
+// Color space conversions
+
+
+
+
+////////////////////////////////////////////////////////////////////////////////
+// Recovery map calculations
+
+/*
* Calculate the 8-bit unsigned integer recovery value for the given SDR and HDR
* luminances in linear space, and the hdr ratio to encode against.
*/
@@ -86,6 +196,11 @@
Color getYuv420Pixel(jr_uncompressed_ptr image, size_t x, size_t y);
/*
+ * Helper for sampling from images.
+ */
+Color getP010Pixel(jr_uncompressed_ptr image, size_t x, size_t y);
+
+/*
* Sample the recovery value for the map from a given x,y coordinate on a scale
* that is map scale factor larger than the map size.
*/
@@ -97,8 +212,13 @@
*
* Expect narrow-range image data for P010.
*/
-float sampleYuv420Y(jr_uncompressed_ptr map, size_t map_scale_factor, size_t x, size_t y);
-float sampleP010Y(jr_uncompressed_ptr map, size_t map_scale_factor, size_t x, size_t y);
+Color sampleYuv420(jr_uncompressed_ptr map, size_t map_scale_factor, size_t x, size_t y);
+
+/*
+ * Sample the image Y value at the provided location, with a weighting based on nearby pixels
+ * and the map scale factor. Assumes narrow-range image data for P010.
+ */
+Color sampleP010(jr_uncompressed_ptr map, size_t map_scale_factor, size_t x, size_t y);
} // namespace android::recoverymap
diff --git a/libs/jpegrecoverymap/recoverymap.cpp b/libs/jpegrecoverymap/recoverymap.cpp
index bd16a68..86eb8a8 100644
--- a/libs/jpegrecoverymap/recoverymap.cpp
+++ b/libs/jpegrecoverymap/recoverymap.cpp
@@ -301,31 +301,38 @@
std::unique_ptr<uint8_t[]> map_data;
map_data.reset(reinterpret_cast<uint8_t*>(dest->data));
- uint16_t yp_hdr_max = 0;
+ float hdr_y_nits_max = 0.0f;
for (size_t y = 0; y < image_height; ++y) {
for (size_t x = 0; x < image_width; ++x) {
- size_t pixel_idx = x + y * image_width;
- uint16_t yp_hdr = reinterpret_cast<uint8_t*>(uncompressed_yuv_420_image->data)[pixel_idx];
- if (yp_hdr > yp_hdr_max) {
- yp_hdr_max = yp_hdr;
+ Color hdr_yuv_gamma = getP010Pixel(uncompressed_p010_image, x, y);
+ Color hdr_rgb_gamma = bt2100YuvToRgb(hdr_yuv_gamma);
+ Color hdr_rgb = hlgInvOetf(hdr_rgb_gamma);
+ float hdr_y_nits = bt2100Luminance(hdr_rgb);
+
+ if (hdr_y_nits > hdr_y_nits_max) {
+ hdr_y_nits_max = hdr_y_nits;
}
}
}
- float y_hdr_max_nits = hlgInvOetf(yp_hdr_max);
- hdr_ratio = y_hdr_max_nits / kSdrWhiteNits;
+ hdr_ratio = hdr_y_nits_max / kSdrWhiteNits;
for (size_t y = 0; y < map_height; ++y) {
for (size_t x = 0; x < map_width; ++x) {
- float yp_sdr = sampleYuv420Y(uncompressed_yuv_420_image, kMapDimensionScaleFactor, x, y);
- float yp_hdr = sampleP010Y(uncompressed_p010_image, kMapDimensionScaleFactor, x, y);
+ Color sdr_yuv_gamma = sampleYuv420(uncompressed_yuv_420_image,
+ kMapDimensionScaleFactor, x, y);
+ Color sdr_rgb_gamma = srgbYuvToRgb(sdr_yuv_gamma);
+ Color sdr_rgb = srgbInvOetf(sdr_rgb_gamma);
+ float sdr_y_nits = srgbLuminance(sdr_rgb);
- float y_sdr_nits = srgbInvOetf(yp_sdr);
- float y_hdr_nits = hlgInvOetf(yp_hdr);
+ Color hdr_yuv_gamma = sampleP010(uncompressed_p010_image, kMapDimensionScaleFactor, x, y);
+ Color hdr_rgb_gamma = bt2100YuvToRgb(hdr_yuv_gamma);
+ Color hdr_rgb = hlgInvOetf(hdr_rgb_gamma);
+ float hdr_y_nits = bt2100Luminance(hdr_rgb);
size_t pixel_idx = x + y * map_width;
reinterpret_cast<uint8_t*>(dest->data)[pixel_idx] =
- encodeRecovery(y_sdr_nits, y_hdr_nits, hdr_ratio);
+ encodeRecovery(sdr_y_nits, hdr_y_nits, hdr_ratio);
}
}
@@ -367,7 +374,8 @@
Color rgb_hdr = applyRecovery(rgb_sdr, recovery, hdr_ratio);
Color rgbp_hdr = hlgOetf(rgb_hdr);
- Color ypuv_hdr = bt2100RgbToYuv(rgbp_hdr);
+ // TODO: actually just leave in RGB and convert to RGBA1010102 instead.
+ Color ypuv_hdr = srgbRgbToYuv(rgbp_hdr);
reinterpret_cast<uint16_t*>(dest->data)[pixel_y_idx] = ypuv_hdr.r;
reinterpret_cast<uint16_t*>(dest->data)[pixel_count + pixel_uv_idx] = ypuv_hdr.g;
diff --git a/libs/jpegrecoverymap/recoverymapmath.cpp b/libs/jpegrecoverymap/recoverymapmath.cpp
index 368783f..0d3319f 100644
--- a/libs/jpegrecoverymap/recoverymapmath.cpp
+++ b/libs/jpegrecoverymap/recoverymapmath.cpp
@@ -20,42 +20,102 @@
namespace android::recoverymap {
-static const float kBt2100R = 0.2627f, kBt2100G = 0.6780f, kBt2100B = 0.0593f;
-static const float kBt2100Cb = 1.8814f, kBt2100Cr = 1.4746f;
+////////////////////////////////////////////////////////////////////////////////
+// sRGB transformations
-Color bt2100RgbToYuv(Color e) {
- float yp = kBt2100R * e.r + kBt2100G * e.g + kBt2100B * e.b;
- return {{{yp, (e.b - yp) / kBt2100Cb, (e.r - yp) / kBt2100Cr }}};
+static const float kSrgbR = 0.299f, kSrgbG = 0.587f, kSrgbB = 0.114f;
+
+float srgbLuminance(Color e) {
+ return kSrgbR * e.r + kSrgbG * e.g + kSrgbB * e.b;
}
static const float kSrgbRCr = 1.402f, kSrgbGCb = 0.34414f, kSrgbGCr = 0.71414f, kSrgbBCb = 1.772f;
-Color srgbYuvToRgb(Color e) {
- return {{{ e.y + kSrgbRCr * e.v, e.y - kSrgbGCb * e.u - kSrgbGCr * e.v, e.y + kSrgbBCb * e.u }}};
+Color srgbYuvToRgb(Color e_gamma) {
+ return {{{ e_gamma.y + kSrgbRCr * e_gamma.v,
+ e_gamma.y - kSrgbGCb * e_gamma.u - kSrgbGCr * e_gamma.v,
+ e_gamma.y + kSrgbBCb * e_gamma.u }}};
}
-float srgbInvOetf(float e) {
- if (e <= 0.04045f) {
- return e / 12.92f;
+static const float kSrgbUR = -0.1687f, kSrgbUG = -0.3313f, kSrgbUB = 0.5f;
+static const float kSrgbVR = 0.5f, kSrgbVG = -0.4187f, kSrgbVB = -0.0813f;
+
+Color srgbRgbToYuv(Color e_gamma) {
+ return {{{ kSrgbR * e_gamma.r + kSrgbG * e_gamma.g + kSrgbB * e_gamma.b,
+ kSrgbUR * e_gamma.r + kSrgbUG * e_gamma.g + kSrgbUB * e_gamma.b,
+ kSrgbVR * e_gamma.r + kSrgbVG * e_gamma.g + kSrgbVB * e_gamma.b }}};
+}
+
+float srgbInvOetf(float e_gamma) {
+ if (e_gamma <= 0.04045f) {
+ return e_gamma / 12.92f;
} else {
- return pow((e + 0.055f) / 1.055f, 2.4);
+ return pow((e_gamma + 0.055f) / 1.055f, 2.4);
}
}
-Color srgbInvOetf(Color e) {
- return {{{ srgbInvOetf(e.r), srgbInvOetf(e.g), srgbInvOetf(e.b) }}};
+Color srgbInvOetf(Color e_gamma) {
+ return {{{ srgbInvOetf(e_gamma.r),
+ srgbInvOetf(e_gamma.g),
+ srgbInvOetf(e_gamma.b) }}};
+}
+
+
+////////////////////////////////////////////////////////////////////////////////
+// Display-P3 transformations
+
+
+
+////////////////////////////////////////////////////////////////////////////////
+// BT.2100 transformations - according to ITU-R BT.2100-2
+
+static const float kBt2100R = 0.2627f, kBt2100G = 0.6780f, kBt2100B = 0.0593f;
+
+float bt2100Luminance(Color e) {
+ return kBt2100R * e.r + kBt2100G * e.g + kBt2100B * e.b;
+}
+
+static const float kBt2100Cb = 1.8814f, kBt2100Cr = 1.4746f;
+
+Color bt2100RgbToYuv(Color e_gamma) {
+ float y_gamma = bt2100Luminance(e_gamma);
+ return {{{ y_gamma,
+ (e_gamma.b - y_gamma) / kBt2100Cb,
+ (e_gamma.r - y_gamma) / kBt2100Cr }}};
+}
+
+// Derived from the reverse of bt2100RgbToYuv. The derivation for R and B are
+// pretty straight forward; we just reverse the formulas for U and V above. But
+// deriving the formula for G is a bit more complicated:
+//
+// Start with equation for luminance:
+// Y = kBt2100R * R + kBt2100G * G + kBt2100B * B
+// Solve for G:
+// G = (Y - kBt2100R * R - kBt2100B * B) / kBt2100B
+// Substitute equations for R and B in terms YUV:
+// G = (Y - kBt2100R * (Y + kBt2100Cr * V) - kBt2100B * (Y + kBt2100Cb * U)) / kBt2100B
+// Simplify:
+// G = Y * ((1 - kBt2100R - kBt2100B) / kBt2100G)
+// + U * (kBt2100B * kBt2100Cb / kBt2100G)
+// + V * (kBt2100R * kBt2100Cr / kBt2100G)
+//
+// We then get the following coeficients for calculating G from YUV:
+//
+// Coef for Y = (1 - kBt2100R - kBt2100B) / kBt2100G = 1
+// Coef for U = kBt2100B * kBt2100Cb / kBt2100G = kBt2100GCb = ~0.1645
+// Coef for V = kBt2100R * kBt2100Cr / kBt2100G = kBt2100GCr = ~0.5713
+
+static const float kBt2100GCb = kBt2100B * kBt2100Cb / kBt2100G;
+static const float kBt2100GCr = kBt2100R * kBt2100Cr / kBt2100G;
+
+Color bt2100YuvToRgb(Color e_gamma) {
+ return {{{ e_gamma.y + kBt2100Cr * e_gamma.v,
+ e_gamma.y - kBt2100GCb * e_gamma.u - kBt2100GCr * e_gamma.v,
+ e_gamma.y + kBt2100Cb * e_gamma.u }}};
}
static const float kHlgA = 0.17883277f, kHlgB = 0.28466892f, kHlgC = 0.55991073;
-float hlgInvOetf(float e) {
- if (e <= 0.5f) {
- return pow(e, 2.0f) / 3.0f;
- } else {
- return (exp((e - kHlgC) / kHlgA) + kHlgB) / 12.0f;
- }
-}
-
static float hlgOetf(float e) {
if (e <= 1.0f/12.0f) {
return sqrt(3.0f * e);
@@ -68,6 +128,28 @@
return {{{ hlgOetf(e.r), hlgOetf(e.g), hlgOetf(e.b) }}};
}
+float hlgInvOetf(float e_gamma) {
+ if (e_gamma <= 0.5f) {
+ return pow(e_gamma, 2.0f) / 3.0f;
+ } else {
+ return (exp((e_gamma - kHlgC) / kHlgA) + kHlgB) / 12.0f;
+ }
+}
+
+Color hlgInvOetf(Color e_gamma) {
+ return {{{ hlgInvOetf(e_gamma.r),
+ hlgInvOetf(e_gamma.g),
+ hlgInvOetf(e_gamma.b) }}};
+}
+
+
+////////////////////////////////////////////////////////////////////////////////
+// Color conversions
+
+
+////////////////////////////////////////////////////////////////////////////////
+// Recovery map calculations
+
uint8_t encodeRecovery(float y_sdr, float y_hdr, float hdr_ratio) {
float gain = 1.0f;
if (y_sdr > 0.0f) {
@@ -137,40 +219,42 @@
(static_cast<float>(v_uint) - 128.0f) / 255.0f }}};
}
-typedef float (*sampleComponentFn)(jr_uncompressed_ptr, size_t, size_t);
+Color getP010Pixel(jr_uncompressed_ptr image, size_t x, size_t y) {
+ size_t pixel_count = image->width * image->height;
-static float sampleComponent(jr_uncompressed_ptr image, size_t map_scale_factor, size_t x, size_t y,
- sampleComponentFn sample_fn) {
- float e = 0.0f;
+ size_t pixel_y_idx = x + y * image->width;
+ size_t pixel_uv_idx = x / 2 + (y / 2) * (image->width / 2);
+
+ uint16_t y_uint = reinterpret_cast<uint16_t*>(image->data)[pixel_y_idx];
+ uint16_t u_uint = reinterpret_cast<uint16_t*>(image->data)[pixel_count + pixel_uv_idx * 2];
+ uint16_t v_uint = reinterpret_cast<uint16_t*>(image->data)[pixel_count + pixel_uv_idx * 2 + 1];
+
+ // Conversions include taking narrow-range into account.
+ return {{{ static_cast<float>(y_uint) / 940.0f,
+ (static_cast<float>(u_uint) - 64.0f) / 940.0f - 0.5f,
+ (static_cast<float>(v_uint) - 64.0f) / 940.0f - 0.5f }}};
+}
+
+typedef Color (*getPixelFn)(jr_uncompressed_ptr, size_t, size_t);
+
+static Color samplePixels(jr_uncompressed_ptr image, size_t map_scale_factor, size_t x, size_t y,
+ getPixelFn get_pixel_fn) {
+ Color e = {{{ 0.0f, 0.0f, 0.0f }}};
for (size_t dy = 0; dy < map_scale_factor; ++dy) {
for (size_t dx = 0; dx < map_scale_factor; ++dx) {
- e += sample_fn(image, x * map_scale_factor + dx, y * map_scale_factor + dy);
+ e += get_pixel_fn(image, x * map_scale_factor + dx, y * map_scale_factor + dy);
}
}
return e / static_cast<float>(map_scale_factor * map_scale_factor);
}
-static float getYuv420Y(jr_uncompressed_ptr image, size_t x, size_t y) {
- size_t pixel_idx = x + y * image->width;
- uint8_t y_uint = reinterpret_cast<uint8_t*>(image->data)[pixel_idx];
- return static_cast<float>(y_uint) / 255.0f;
+Color sampleYuv420(jr_uncompressed_ptr image, size_t map_scale_factor, size_t x, size_t y) {
+ return samplePixels(image, map_scale_factor, x, y, getYuv420Pixel);
}
-
-float sampleYuv420Y(jr_uncompressed_ptr image, size_t map_scale_factor, size_t x, size_t y) {
- return sampleComponent(image, map_scale_factor, x, y, getYuv420Y);
-}
-
-static float getP010Y(jr_uncompressed_ptr image, size_t x, size_t y) {
- size_t pixel_idx = x + y * image->width;
- uint8_t y_uint = reinterpret_cast<uint16_t*>(image->data)[pixel_idx];
- // Expecting narrow range input
- return (static_cast<float>(y_uint) - 64.0f) / 960.0f;
-}
-
-float sampleP010Y(jr_uncompressed_ptr image, size_t map_scale_factor, size_t x, size_t y) {
- return sampleComponent(image, map_scale_factor, x, y, getP010Y);
+Color sampleP010(jr_uncompressed_ptr image, size_t map_scale_factor, size_t x, size_t y) {
+ return samplePixels(image, map_scale_factor, x, y, getP010Pixel);
}
} // namespace android::recoverymap
diff --git a/libs/jpegrecoverymap/tests/Android.bp b/libs/jpegrecoverymap/tests/Android.bp
index 41af991..8f37954 100644
--- a/libs/jpegrecoverymap/tests/Android.bp
+++ b/libs/jpegrecoverymap/tests/Android.bp
@@ -27,7 +27,15 @@
srcs: [
"recoverymap_test.cpp",
],
+ shared_libs: [
+ "libimage_io",
+ "libjpeg",
+ "liblog",
+ ],
static_libs: [
+ "libgtest",
+ "libjpegdecoder",
+ "libjpegencoder",
"libjpegrecoverymap",
],
}
diff --git a/libs/jpegrecoverymap/tests/recoverymap_test.cpp b/libs/jpegrecoverymap/tests/recoverymap_test.cpp
index c436138..226056b 100644
--- a/libs/jpegrecoverymap/tests/recoverymap_test.cpp
+++ b/libs/jpegrecoverymap/tests/recoverymap_test.cpp
@@ -14,9 +14,33 @@
* limitations under the License.
*/
+#include <gtest/gtest.h>
#include <jpegrecoverymap/recoverymap.h>
-namespace android {
+namespace android::recoverymap {
-// Add new tests here.
-} // namespace android
+class RecoveryMapTest : public testing::Test {
+public:
+ RecoveryMapTest();
+ ~RecoveryMapTest();
+protected:
+ virtual void SetUp();
+ virtual void TearDown();
+};
+
+RecoveryMapTest::RecoveryMapTest() {}
+RecoveryMapTest::~RecoveryMapTest() {}
+
+void RecoveryMapTest::SetUp() {}
+void RecoveryMapTest::TearDown() {}
+
+TEST_F(RecoveryMapTest, build) {
+ // Force all of the recovery map lib to be linked by calling all public functions.
+ RecoveryMap recovery_map;
+ recovery_map.encodeJPEGR(nullptr, nullptr, nullptr, 0, nullptr);
+ recovery_map.encodeJPEGR(nullptr, nullptr, nullptr, nullptr);
+ recovery_map.encodeJPEGR(nullptr, nullptr, nullptr);
+ recovery_map.decodeJPEGR(nullptr, nullptr, nullptr, false);
+}
+
+} // namespace android::recoverymap