A bunch of improvements to recoverymap calculations.
* Add proper color space conversions for map generation
* Add proper luminance calculation for map generation
* Add PQ encode/decode support
* Add structs for handling JPEG/R metadata, including HDR10 stuff
* Fill in HDR10 metadata, except for ST2086 info
* Update decode output from P010 to RGBA 1010102
* Update "hdr ratio" to "range scaling factor"
Bug: 252835416
Test: builds
No-Typo-Check: incorrectly interpretting code as a comment
Change-Id: I67d7be6d2a7821108d5f7ed26dd65285684e80f4
diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/jpegrerrorcode.h b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegrerrorcode.h
index 194cd2f..9f53a57 100644
--- a/libs/jpegrecoverymap/include/jpegrecoverymap/jpegrerrorcode.h
+++ b/libs/jpegrecoverymap/include/jpegrecoverymap/jpegrerrorcode.h
@@ -37,6 +37,7 @@
ERROR_JPEGR_INVALID_NULL_PTR = JPEGR_IO_ERROR_BASE - 2,
ERROR_JPEGR_RESOLUTION_MISMATCH = JPEGR_IO_ERROR_BASE - 3,
ERROR_JPEGR_BUFFER_TOO_SMALL = JPEGR_IO_ERROR_BASE - 4,
+ ERROR_JPEGR_INVALID_COLORGAMUT = JPEGR_IO_ERROR_BASE - 5,
JPEGR_RUNTIME_ERROR_BASE = -20000,
ERROR_JPEGR_ENCODE_ERROR = JPEGR_RUNTIME_ERROR_BASE - 1,
diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h
index b2ca481..d258f80 100644
--- a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h
+++ b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h
@@ -22,11 +22,17 @@
namespace android::recoverymap {
typedef enum {
- JPEGR_COLORSPACE_UNSPECIFIED,
- JPEGR_COLORSPACE_BT709,
- JPEGR_COLORSPACE_P3,
- JPEGR_COLORSPACE_BT2100,
-} jpegr_color_space;
+ JPEGR_COLORGAMUT_UNSPECIFIED,
+ JPEGR_COLORGAMUT_BT709,
+ JPEGR_COLORGAMUT_P3,
+ JPEGR_COLORGAMUT_BT2100,
+} jpegr_color_gamut;
+
+// Transfer functions as defined for XMP metadata
+typedef enum {
+ JPEGR_TF_HLG = 0,
+ JPEGR_TF_PQ = 1,
+} jpegr_transfer_function;
/*
* Holds information for uncompressed image or recovery map.
@@ -38,8 +44,8 @@
int width;
// Height of the recovery map or image in pixels.
int height;
- // Color space.
- jpegr_color_space colorSpace;
+ // Color gamut.
+ jpegr_color_gamut colorGamut;
};
/*
@@ -50,8 +56,8 @@
void* data;
// Data length.
int length;
- // Color space.
- jpegr_color_space colorSpace;
+ // Color gamut.
+ jpegr_color_gamut colorGamut;
};
/*
@@ -64,9 +70,51 @@
int length;
};
+struct chromaticity_coord {
+ float x;
+ float y;
+};
+
+
+struct st2086_metadata {
+ // xy chromaticity coordinate of the red primary of the mastering display
+ chromaticity_coord redPrimary;
+ // xy chromaticity coordinate of the green primary of the mastering display
+ chromaticity_coord greenPrimary;
+ // xy chromaticity coordinate of the blue primary of the mastering display
+ chromaticity_coord bluePrimary;
+ // xy chromaticity coordinate of the white point of the mastering display
+ chromaticity_coord whitePoint;
+ // Maximum luminance in nits of the mastering display
+ uint32_t maxLuminance;
+ // Minimum luminance in nits of the mastering display
+ float minLuminance;
+};
+
+struct hdr10_metadata {
+ // Mastering display color volume
+ st2086_metadata st2086Metadata;
+ // Max frame average light level in nits
+ float maxFALL;
+ // Max content light level in nits
+ float maxCLL;
+};
+
+struct jpegr_metadata {
+ // JPEG/R version
+ uint32_t version;
+ // Range scaling factor for the map
+ float rangeScalingFactor;
+ // The transfer function for decoding the HDR representation of the image
+ jpegr_transfer_function transferFunction;
+ // HDR10 metadata, only applicable for transferFunction of JPEGR_TF_PQ
+ hdr10_metadata hdr10Metadata;
+};
+
typedef struct jpegr_uncompressed_struct* jr_uncompressed_ptr;
typedef struct jpegr_compressed_struct* jr_compressed_ptr;
typedef struct jpegr_exif_struct* jr_exif_ptr;
+typedef struct jpegr_metadata* jr_metadata_ptr;
class RecoveryMap {
public:
@@ -75,9 +123,10 @@
*
* Generate recovery map from the HDR and SDR inputs, compress SDR YUV to 8-bit JPEG and append
* the recovery map to the end of the compressed JPEG. HDR and SDR inputs must be the same
- * resolution and color space.
+ * resolution.
* @param uncompressed_p010_image uncompressed HDR image in P010 color format
* @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format
+ * @param hdr_tf transfer function of the HDR image
* @param dest destination of the compressed JPEGR image
* @param quality target quality of the JPEG encoding, must be in range of 0-100 where 100 is
* the highest quality
@@ -86,6 +135,7 @@
*/
status_t encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,
jr_uncompressed_ptr uncompressed_yuv_420_image,
+ jpegr_transfer_function hdr_tf,
jr_compressed_ptr dest,
int quality,
jr_exif_ptr exif);
@@ -100,12 +150,14 @@
* @param uncompressed_p010_image uncompressed HDR image in P010 color format
* @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format
* @param compressed_jpeg_image compressed 8-bit JPEG image
+ * @param hdr_tf transfer function of the HDR image
* @param dest destination of the compressed JPEGR image
* @return NO_ERROR if encoding succeeds, error code if error occurs.
*/
status_t encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,
jr_uncompressed_ptr uncompressed_yuv_420_image,
jr_compressed_ptr compressed_jpeg_image,
+ jpegr_transfer_function hdr_tf,
jr_compressed_ptr dest);
/*
@@ -115,27 +167,28 @@
*
* Decode the compressed 8-bit JPEG image to YUV SDR, generate recovery map from the HDR input
* and the decoded SDR result, append the recovery map to the end of the compressed JPEG. HDR
- * and SDR inputs must be the same resolution and color space.
+ * and SDR inputs must be the same resolution.
* @param uncompressed_p010_image uncompressed HDR image in P010 color format
* @param compressed_jpeg_image compressed 8-bit JPEG image
+ * @param hdr_tf transfer function of the HDR image
* @param dest destination of the compressed JPEGR image
* @return NO_ERROR if encoding succeeds, error code if error occurs.
*/
status_t encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,
jr_compressed_ptr compressed_jpeg_image,
+ jpegr_transfer_function hdr_tf,
jr_compressed_ptr dest);
/*
* Decompress JPEGR image.
*
+ * The output JPEGR image is in RGBA_1010102 data format if decoding to HDR.
* @param compressed_jpegr_image compressed JPEGR image
* @param dest destination of the uncompressed JPEGR image
- * @param exif destination of the decoded EXIF metadata. Default value is nullptr where EXIF
- * metadata will not be decoded.
- * @param request_sdr flag that request SDR output, default to false (request HDR output). If
- * set to true, decoder will only decode the primary image which is SDR.
- * Setting of request_sdr and input source (HDR or SDR) can be found in
- * the table below:
+ * @param exif destination of the decoded EXIF metadata.
+ * @param request_sdr flag that request SDR output. If set to true, decoder will only decode
+ * the primary image which is SDR. Setting of request_sdr and input source
+ * (HDR or SDR) can be found in the table below:
* | input source | request_sdr | output of decoding |
* | HDR | true | SDR |
* | HDR | false | HDR |
@@ -145,8 +198,8 @@
*/
status_t decodeJPEGR(jr_compressed_ptr compressed_jpegr_image,
jr_uncompressed_ptr dest,
- jr_exif_ptr exif = nullptr,
- bool request_sdr = false);
+ jr_exif_ptr exif,
+ bool request_sdr);
private:
/*
* This method is called in the decoding pipeline. It will decode the recovery map.
@@ -176,26 +229,32 @@
* @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format
* @param uncompressed_p010_image uncompressed HDR image in P010 color format
* @param dest recovery map; caller responsible for memory of data
- * @param hdr_ratio HDR ratio will be updated in this method
+ * @param metadata metadata provides the transfer function for the HDR
+ * image; range_scaling_factor and hdr10 FALL and CLL will
+ * be updated.
* @return NO_ERROR if calculation succeeds, error code if error occurs.
*/
status_t generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image,
jr_uncompressed_ptr uncompressed_p010_image,
- jr_uncompressed_ptr dest,
- float &hdr_ratio);
+ jr_metadata_ptr metadata,
+ jr_uncompressed_ptr dest);
/*
* This method is called in the decoding pipeline. It will take the uncompressed (decoded)
- * 8-bit yuv image and the uncompressed (decoded) recovery map as input, and calculate the
- * 10-bit recovered image (in p010 color format).
+ * 8-bit yuv image, the uncompressed (decoded) recovery map, and extracted JPEG/R metadata as
+ * input, and calculate the 10-bit recovered image. The recovered output image is the same
+ * color gamut as the SDR image, with the transfer function specified in the JPEG/R metadata,
+ * and is in RGBA1010102 data format.
*
* @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format
* @param uncompressed_recovery_map uncompressed recovery map
+ * @param metadata JPEG/R metadata extracted from XMP.
* @param dest reconstructed HDR image
* @return NO_ERROR if calculation succeeds, error code if error occurs.
*/
status_t applyRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image,
jr_uncompressed_ptr uncompressed_recovery_map,
+ jr_metadata_ptr metadata,
jr_uncompressed_ptr dest);
/*
@@ -204,9 +263,12 @@
*
* @param compressed_jpegr_image compressed JPEGR image
* @param dest destination of compressed recovery map
+ * @param metadata destination of the recovery map metadata
* @return NO_ERROR if calculation succeeds, error code if error occurs.
*/
- status_t extractRecoveryMap(jr_compressed_ptr compressed_jpegr_image, jr_compressed_ptr dest);
+ status_t extractRecoveryMap(jr_compressed_ptr compressed_jpegr_image,
+ jr_compressed_ptr dest,
+ jr_metadata_ptr metadata);
/*
* This method is called in the encoding pipeline. It will take the standard 8-bit JPEG image
@@ -215,13 +277,13 @@
*
* @param compressed_jpeg_image compressed 8-bit JPEG image
* @param compress_recovery_map compressed recover map
- * @param hdr_ratio HDR ratio
+ * @param metadata JPEG/R metadata to encode in XMP of the jpeg
* @param dest compressed JPEGR image
* @return NO_ERROR if calculation succeeds, error code if error occurs.
*/
status_t appendRecoveryMap(jr_compressed_ptr compressed_jpeg_image,
jr_compressed_ptr compressed_recovery_map,
- float hdr_ratio,
+ jr_metadata_ptr metadata,
jr_compressed_ptr dest);
/*
@@ -229,7 +291,7 @@
*
* below is an example of the XMP metadata that this function generates where
* secondary_image_length = 1000
- * hdr_ratio = 1.25
+ * range_scaling_factor = 1.25
*
* <x:xmpmeta
* xmlns:x="adobe:ns:meta/"
@@ -239,7 +301,7 @@
* <rdf:Description
* xmlns:GContainer="http://ns.google.com/photos/1.0/container/">
* <GContainer:Version>1</GContainer:Version>
- * <GContainer:HdrRatio>1.25</GContainer:HdrRatio>
+ * <GContainer:RangeScalingFactor>1.25</GContainer:RangeScalingFactor>
* <GContainer:Directory>
* <rdf:Seq>
* <rdf:li>
@@ -260,10 +322,10 @@
* </x:xmpmeta>
*
* @param secondary_image_length length of secondary image
- * @param hdr_ratio hdr ratio
+ * @param metadata JPEG/R metadata to encode as XMP
* @return XMP metadata in type of string
*/
- std::string generateXmp(int secondary_image_length, float hdr_ratio);
+ std::string generateXmp(int secondary_image_length, jpegr_metadata& metadata);
};
} // namespace android::recoverymap
diff --git a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h
index e76bc20..fe7a651 100644
--- a/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h
+++ b/libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h
@@ -43,6 +43,9 @@
};
};
+typedef Color (*ColorTransformFn)(Color);
+typedef float (*ColorCalculationFn)(Color);
+
inline Color operator+=(Color& lhs, const Color& rhs) {
lhs.r += rhs.r;
lhs.g += rhs.g;
@@ -138,7 +141,10 @@
////////////////////////////////////////////////////////////////////////////////
// Display-P3 transformations
-
+/*
+ * Calculated the luminance of a linear RGB P3 pixel, according to EG 432-1.
+ */
+float p3Luminance(Color e);
////////////////////////////////////////////////////////////////////////////////
@@ -169,10 +175,43 @@
*/
Color hlgInvOetf(Color e_gamma);
+/*
+ * Convert from scene luminance in nits to PQ.
+ */
+Color pqOetf(Color e);
+
+/*
+ * Convert from PQ to scene luminance in nits.
+ */
+Color pqInvOetf(Color e_gamma);
+
+
////////////////////////////////////////////////////////////////////////////////
// Color space conversions
+/*
+ * Convert between color spaces with linear RGB data, according to ITU-R BT.2407 and EG 432-1.
+ *
+ * All conversions are derived from multiplying the matrix for XYZ to output RGB color gamut by the
+ * matrix for input RGB color gamut to XYZ. The matrix for converting from XYZ to an RGB gamut is
+ * always the inverse of the RGB gamut to XYZ matrix.
+ */
+Color bt709ToP3(Color e);
+Color bt709ToBt2100(Color e);
+Color p3ToBt709(Color e);
+Color p3ToBt2100(Color e);
+Color bt2100ToBt709(Color e);
+Color bt2100ToP3(Color e);
+/*
+ * Identity conversion.
+ */
+inline Color identityConversion(Color e) { return e; }
+
+/*
+ * Get the conversion to apply to the HDR image for recovery map generation
+ */
+ColorTransformFn getHdrConversionFn(jpegr_color_gamut sdr_gamut, jpegr_color_gamut hdr_gamut);
////////////////////////////////////////////////////////////////////////////////
@@ -220,6 +259,13 @@
*/
Color sampleP010(jr_uncompressed_ptr map, size_t map_scale_factor, size_t x, size_t y);
+/*
+ * Convert from Color to RGBA1010102.
+ *
+ * Alpha always set to 1.0.
+ */
+uint32_t colorToRgba1010102(Color e_gamma);
+
} // namespace android::recoverymap
#endif // ANDROID_JPEGRECOVERYMAP_RECOVERYMAPMATH_H
diff --git a/libs/jpegrecoverymap/recoverymap.cpp b/libs/jpegrecoverymap/recoverymap.cpp
index 86eb8a8..64021b7 100644
--- a/libs/jpegrecoverymap/recoverymap.cpp
+++ b/libs/jpegrecoverymap/recoverymap.cpp
@@ -14,11 +14,6 @@
* limitations under the License.
*/
-// TODO: need to clean up handling around hdr_ratio and passing it around
-// TODO: need to handle color space information; currently we assume everything
-// is srgb in.
-// TODO: handle PQ encode/decode (currently only HLG)
-
#include <jpegrecoverymap/recoverymap.h>
#include <jpegrecoverymap/jpegencoder.h>
#include <jpegrecoverymap/jpegdecoder.h>
@@ -43,9 +38,21 @@
} \
}
+// The current JPEGR version that we encode to
+static const uint32_t kJpegrVersion = 1;
+
// Map is quarter res / sixteenth size
static const size_t kMapDimensionScaleFactor = 4;
+// TODO: fill in st2086 metadata
+static const st2086_metadata kSt2086Metadata = {
+ {0.0f, 0.0f},
+ {0.0f, 0.0f},
+ {0.0f, 0.0f},
+ {0.0f, 0.0f},
+ 0,
+ 1.0f,
+};
/*
* Helper function used for generating XMP metadata.
@@ -81,6 +88,7 @@
status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,
jr_uncompressed_ptr uncompressed_yuv_420_image,
+ jpegr_transfer_function hdr_tf,
jr_compressed_ptr dest,
int quality,
jr_exif_ptr /* exif */) {
@@ -99,10 +107,16 @@
return ERROR_JPEGR_RESOLUTION_MISMATCH;
}
+ jpegr_metadata metadata;
+ metadata.version = kJpegrVersion;
+ metadata.transferFunction = hdr_tf;
+ if (hdr_tf == JPEGR_TF_PQ) {
+ metadata.hdr10Metadata.st2086Metadata = kSt2086Metadata;
+ }
+
jpegr_uncompressed_struct map;
- float hdr_ratio = 0.0f;
JPEGR_CHECK(generateRecoveryMap(
- uncompressed_yuv_420_image, uncompressed_p010_image, &map, hdr_ratio));
+ uncompressed_yuv_420_image, uncompressed_p010_image, &metadata, &map));
std::unique_ptr<uint8_t[]> map_data;
map_data.reset(reinterpret_cast<uint8_t*>(map.data));
@@ -113,7 +127,7 @@
JPEGR_CHECK(compressRecoveryMap(&map, &compressed_map));
JpegEncoder jpeg_encoder;
- // TODO: ICC data - need color space information
+ // TODO: determine ICC data based on color gamut information
if (!jpeg_encoder.compressImage(uncompressed_yuv_420_image->data,
uncompressed_yuv_420_image->width,
uncompressed_yuv_420_image->height, quality, nullptr, 0)) {
@@ -123,7 +137,7 @@
jpeg.data = jpeg_encoder.getCompressedImagePtr();
jpeg.length = jpeg_encoder.getCompressedImageSize();
- JPEGR_CHECK(appendRecoveryMap(&jpeg, &compressed_map, hdr_ratio, dest));
+ JPEGR_CHECK(appendRecoveryMap(&jpeg, &compressed_map, &metadata, dest));
return NO_ERROR;
}
@@ -131,6 +145,7 @@
status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,
jr_uncompressed_ptr uncompressed_yuv_420_image,
jr_compressed_ptr compressed_jpeg_image,
+ jpegr_transfer_function hdr_tf,
jr_compressed_ptr dest) {
if (uncompressed_p010_image == nullptr
|| uncompressed_yuv_420_image == nullptr
@@ -144,10 +159,16 @@
return ERROR_JPEGR_RESOLUTION_MISMATCH;
}
+ jpegr_metadata metadata;
+ metadata.version = kJpegrVersion;
+ metadata.transferFunction = hdr_tf;
+ if (hdr_tf == JPEGR_TF_PQ) {
+ metadata.hdr10Metadata.st2086Metadata = kSt2086Metadata;
+ }
+
jpegr_uncompressed_struct map;
- float hdr_ratio = 0.0f;
JPEGR_CHECK(generateRecoveryMap(
- uncompressed_yuv_420_image, uncompressed_p010_image, &map, hdr_ratio));
+ uncompressed_yuv_420_image, uncompressed_p010_image, &metadata, &map));
std::unique_ptr<uint8_t[]> map_data;
map_data.reset(reinterpret_cast<uint8_t*>(map.data));
@@ -157,13 +178,14 @@
compressed_map.data = compressed_map_data.get();
JPEGR_CHECK(compressRecoveryMap(&map, &compressed_map));
- JPEGR_CHECK(appendRecoveryMap(compressed_jpeg_image, &compressed_map, hdr_ratio, dest));
+ JPEGR_CHECK(appendRecoveryMap(compressed_jpeg_image, &compressed_map, &metadata, dest));
return NO_ERROR;
}
status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,
jr_compressed_ptr compressed_jpeg_image,
+ jpegr_transfer_function hdr_tf,
jr_compressed_ptr dest) {
if (uncompressed_p010_image == nullptr
|| compressed_jpeg_image == nullptr
@@ -179,16 +201,23 @@
uncompressed_yuv_420_image.data = jpeg_decoder.getDecompressedImagePtr();
uncompressed_yuv_420_image.width = jpeg_decoder.getDecompressedImageWidth();
uncompressed_yuv_420_image.height = jpeg_decoder.getDecompressedImageHeight();
+ uncompressed_yuv_420_image.colorGamut = compressed_jpeg_image->colorGamut;
if (uncompressed_p010_image->width != uncompressed_yuv_420_image.width
|| uncompressed_p010_image->height != uncompressed_yuv_420_image.height) {
return ERROR_JPEGR_RESOLUTION_MISMATCH;
}
+ jpegr_metadata metadata;
+ metadata.version = kJpegrVersion;
+ metadata.transferFunction = hdr_tf;
+ if (hdr_tf == JPEGR_TF_PQ) {
+ metadata.hdr10Metadata.st2086Metadata = kSt2086Metadata;
+ }
+
jpegr_uncompressed_struct map;
- float hdr_ratio = 0.0f;
JPEGR_CHECK(generateRecoveryMap(
- &uncompressed_yuv_420_image, uncompressed_p010_image, &map, hdr_ratio));
+ &uncompressed_yuv_420_image, uncompressed_p010_image, &metadata, &map));
std::unique_ptr<uint8_t[]> map_data;
map_data.reset(reinterpret_cast<uint8_t*>(map.data));
@@ -198,7 +227,7 @@
compressed_map.data = compressed_map_data.get();
JPEGR_CHECK(compressRecoveryMap(&map, &compressed_map));
- JPEGR_CHECK(appendRecoveryMap(compressed_jpeg_image, &compressed_map, hdr_ratio, dest));
+ JPEGR_CHECK(appendRecoveryMap(compressed_jpeg_image, &compressed_map, &metadata, dest));
return NO_ERROR;
}
@@ -212,7 +241,8 @@
}
jpegr_compressed_struct compressed_map;
- JPEGR_CHECK(extractRecoveryMap(compressed_jpegr_image, &compressed_map));
+ jpegr_metadata metadata;
+ JPEGR_CHECK(extractRecoveryMap(compressed_jpegr_image, &compressed_map, &metadata));
jpegr_uncompressed_struct map;
JPEGR_CHECK(decompressRecoveryMap(&compressed_map, &map));
@@ -227,7 +257,7 @@
uncompressed_yuv_420_image.width = jpeg_decoder.getDecompressedImageWidth();
uncompressed_yuv_420_image.height = jpeg_decoder.getDecompressedImageHeight();
- JPEGR_CHECK(applyRecoveryMap(&uncompressed_yuv_420_image, &map, dest));
+ JPEGR_CHECK(applyRecoveryMap(&uncompressed_yuv_420_image, &map, &metadata, dest));
return NO_ERROR;
}
@@ -257,7 +287,7 @@
return ERROR_JPEGR_INVALID_NULL_PTR;
}
- // TODO: should we have ICC data?
+ // TODO: should we have ICC data for the map?
JpegEncoder jpeg_encoder;
if (!jpeg_encoder.compressImage(uncompressed_recovery_map->data, uncompressed_recovery_map->width,
uncompressed_recovery_map->height, 85, nullptr, 0,
@@ -271,16 +301,18 @@
memcpy(dest->data, jpeg_encoder.getCompressedImagePtr(), jpeg_encoder.getCompressedImageSize());
dest->length = jpeg_encoder.getCompressedImageSize();
+ dest->colorGamut = JPEGR_COLORGAMUT_UNSPECIFIED;
return NO_ERROR;
}
status_t RecoveryMap::generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image,
jr_uncompressed_ptr uncompressed_p010_image,
- jr_uncompressed_ptr dest,
- float &hdr_ratio) {
+ jr_metadata_ptr metadata,
+ jr_uncompressed_ptr dest) {
if (uncompressed_yuv_420_image == nullptr
|| uncompressed_p010_image == nullptr
+ || metadata == nullptr
|| dest == nullptr) {
return ERROR_JPEGR_INVALID_NULL_PTR;
}
@@ -290,6 +322,11 @@
return ERROR_JPEGR_RESOLUTION_MISMATCH;
}
+ if (uncompressed_yuv_420_image->colorGamut == JPEGR_COLORGAMUT_UNSPECIFIED
+ || uncompressed_p010_image->colorGamut == JPEGR_COLORGAMUT_UNSPECIFIED) {
+ return ERROR_JPEGR_INVALID_COLORGAMUT;
+ }
+
size_t image_width = uncompressed_yuv_420_image->width;
size_t image_height = uncompressed_yuv_420_image->height;
size_t map_width = image_width / kMapDimensionScaleFactor;
@@ -297,25 +334,63 @@
dest->width = map_width;
dest->height = map_height;
+ dest->colorGamut = JPEGR_COLORGAMUT_UNSPECIFIED;
dest->data = new uint8_t[map_width * map_height];
std::unique_ptr<uint8_t[]> map_data;
map_data.reset(reinterpret_cast<uint8_t*>(dest->data));
+ ColorTransformFn hdrInvOetf = nullptr;
+ switch (metadata->transferFunction) {
+ case JPEGR_TF_HLG:
+ hdrInvOetf = hlgInvOetf;
+ break;
+ case JPEGR_TF_PQ:
+ hdrInvOetf = pqInvOetf;
+ break;
+ }
+
+ ColorTransformFn hdrGamutConversionFn = getHdrConversionFn(
+ uncompressed_yuv_420_image->colorGamut, uncompressed_p010_image->colorGamut);
+
+ ColorCalculationFn luminanceFn = nullptr;
+ switch (uncompressed_yuv_420_image->colorGamut) {
+ case JPEGR_COLORGAMUT_BT709:
+ luminanceFn = srgbLuminance;
+ break;
+ case JPEGR_COLORGAMUT_P3:
+ luminanceFn = p3Luminance;
+ break;
+ case JPEGR_COLORGAMUT_BT2100:
+ luminanceFn = bt2100Luminance;
+ break;
+ case JPEGR_COLORGAMUT_UNSPECIFIED:
+ // Should be impossible to hit after input validation.
+ return ERROR_JPEGR_INVALID_COLORGAMUT;
+ }
+
float hdr_y_nits_max = 0.0f;
+ double hdr_y_nits_avg = 0.0f;
for (size_t y = 0; y < image_height; ++y) {
for (size_t x = 0; x < image_width; ++x) {
Color hdr_yuv_gamma = getP010Pixel(uncompressed_p010_image, x, y);
Color hdr_rgb_gamma = bt2100YuvToRgb(hdr_yuv_gamma);
- Color hdr_rgb = hlgInvOetf(hdr_rgb_gamma);
- float hdr_y_nits = bt2100Luminance(hdr_rgb);
+ Color hdr_rgb = hdrInvOetf(hdr_rgb_gamma);
+ hdr_rgb = hdrGamutConversionFn(hdr_rgb);
+ float hdr_y_nits = luminanceFn(hdr_rgb);
+ hdr_y_nits_avg += hdr_y_nits;
if (hdr_y_nits > hdr_y_nits_max) {
hdr_y_nits_max = hdr_y_nits;
}
}
}
+ hdr_y_nits_avg /= image_width * image_height;
- hdr_ratio = hdr_y_nits_max / kSdrWhiteNits;
+ metadata->rangeScalingFactor = hdr_y_nits_max / kSdrWhiteNits;
+ if (metadata->transferFunction == JPEGR_TF_PQ) {
+ metadata->hdr10Metadata.maxFALL = hdr_y_nits_avg;
+ metadata->hdr10Metadata.maxCLL = hdr_y_nits_max;
+ }
for (size_t y = 0; y < map_height; ++y) {
for (size_t x = 0; x < map_width; ++x) {
@@ -323,16 +398,17 @@
kMapDimensionScaleFactor, x, y);
Color sdr_rgb_gamma = srgbYuvToRgb(sdr_yuv_gamma);
Color sdr_rgb = srgbInvOetf(sdr_rgb_gamma);
- float sdr_y_nits = srgbLuminance(sdr_rgb);
+ float sdr_y_nits = luminanceFn(sdr_rgb);
Color hdr_yuv_gamma = sampleP010(uncompressed_p010_image, kMapDimensionScaleFactor, x, y);
Color hdr_rgb_gamma = bt2100YuvToRgb(hdr_yuv_gamma);
- Color hdr_rgb = hlgInvOetf(hdr_rgb_gamma);
- float hdr_y_nits = bt2100Luminance(hdr_rgb);
+ Color hdr_rgb = hdrInvOetf(hdr_rgb_gamma);
+ hdr_rgb = hdrGamutConversionFn(hdr_rgb);
+ float hdr_y_nits = luminanceFn(hdr_rgb);
size_t pixel_idx = x + y * map_width;
reinterpret_cast<uint8_t*>(dest->data)[pixel_idx] =
- encodeRecovery(sdr_y_nits, hdr_y_nits, hdr_ratio);
+ encodeRecovery(sdr_y_nits, hdr_y_nits, metadata->rangeScalingFactor);
}
}
@@ -342,17 +418,15 @@
status_t RecoveryMap::applyRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image,
jr_uncompressed_ptr uncompressed_recovery_map,
+ jr_metadata_ptr metadata,
jr_uncompressed_ptr dest) {
if (uncompressed_yuv_420_image == nullptr
|| uncompressed_recovery_map == nullptr
+ || metadata == nullptr
|| dest == nullptr) {
return ERROR_JPEGR_INVALID_NULL_PTR;
}
- // TODO: need to get this from the XMP; should probably be a function
- // parameter
- float hdr_ratio = 4.0f;
-
size_t width = uncompressed_yuv_420_image->width;
size_t height = uncompressed_yuv_420_image->height;
@@ -360,26 +434,31 @@
dest->height = height;
size_t pixel_count = width * height;
+ ColorTransformFn hdrOetf = nullptr;
+ switch (metadata->transferFunction) {
+ case JPEGR_TF_HLG:
+ hdrOetf = hlgOetf;
+ break;
+ case JPEGR_TF_PQ:
+ hdrOetf = pqOetf;
+ break;
+ }
+
for (size_t y = 0; y < height; ++y) {
for (size_t x = 0; x < width; ++x) {
- size_t pixel_y_idx = x + y * width;
+ Color yuv_gamma_sdr = getYuv420Pixel(uncompressed_yuv_420_image, x, y);
+ Color rgb_gamma_sdr = srgbYuvToRgb(yuv_gamma_sdr);
+ Color rgb_sdr = srgbInvOetf(rgb_gamma_sdr);
- size_t pixel_uv_idx = x / 2 + (y / 2) * (width / 2);
-
- Color ypuv_sdr = getYuv420Pixel(uncompressed_yuv_420_image, x, y);
- Color rgbp_sdr = srgbYuvToRgb(ypuv_sdr);
- Color rgb_sdr = srgbInvOetf(rgbp_sdr);
-
+ // TODO: determine map scaling factor based on actual map dims
float recovery = sampleMap(uncompressed_recovery_map, kMapDimensionScaleFactor, x, y);
- Color rgb_hdr = applyRecovery(rgb_sdr, recovery, hdr_ratio);
+ Color rgb_hdr = applyRecovery(rgb_sdr, recovery, metadata->rangeScalingFactor);
- Color rgbp_hdr = hlgOetf(rgb_hdr);
- // TODO: actually just leave in RGB and convert to RGBA1010102 instead.
- Color ypuv_hdr = srgbRgbToYuv(rgbp_hdr);
+ Color rgb_gamma_hdr = hdrOetf(rgb_hdr);
+ uint32_t rgba1010102 = colorToRgba1010102(rgb_gamma_hdr);
- reinterpret_cast<uint16_t*>(dest->data)[pixel_y_idx] = ypuv_hdr.r;
- reinterpret_cast<uint16_t*>(dest->data)[pixel_count + pixel_uv_idx] = ypuv_hdr.g;
- reinterpret_cast<uint16_t*>(dest->data)[pixel_count + pixel_uv_idx + 1] = ypuv_hdr.b;
+ size_t pixel_idx = x + y * width;
+ reinterpret_cast<uint32_t*>(dest->data)[pixel_idx] = rgba1010102;
}
}
@@ -387,8 +466,9 @@
}
status_t RecoveryMap::extractRecoveryMap(jr_compressed_ptr compressed_jpegr_image,
- jr_compressed_ptr dest) {
- if (compressed_jpegr_image == nullptr || dest == nullptr) {
+ jr_compressed_ptr dest,
+ jr_metadata_ptr metadata) {
+ if (compressed_jpegr_image == nullptr || dest == nullptr || metadata == nullptr) {
return ERROR_JPEGR_INVALID_NULL_PTR;
}
@@ -398,15 +478,16 @@
status_t RecoveryMap::appendRecoveryMap(jr_compressed_ptr compressed_jpeg_image,
jr_compressed_ptr compressed_recovery_map,
- float hdr_ratio,
+ jr_metadata_ptr metadata,
jr_compressed_ptr dest) {
if (compressed_jpeg_image == nullptr
|| compressed_recovery_map == nullptr
+ || metadata == nullptr
|| dest == nullptr) {
return ERROR_JPEGR_INVALID_NULL_PTR;
}
- string xmp = generateXmp(compressed_recovery_map->length, hdr_ratio);
+ string xmp = generateXmp(compressed_recovery_map->length, *metadata);
string nameSpace = "http://ns.adobe.com/xap/1.0/\0";
// 2 bytes: APP1 sign (ff e1)
@@ -442,7 +523,7 @@
return NO_ERROR;
}
-string RecoveryMap::generateXmp(int secondary_image_length, float hdr_ratio) {
+string RecoveryMap::generateXmp(int secondary_image_length, jpegr_metadata& metadata) {
const string kContainerPrefix = "GContainer";
const string kContainerUri = "http://ns.google.com/photos/1.0/container/";
const string kItemPrefix = "Item";
@@ -455,7 +536,6 @@
const string kPrimary = "Primary";
const string kSemantic = "Semantic";
const string kVersion = "Version";
- const int kVersionValue = 1;
const string kConDir = Name(kContainerPrefix, kDirectory);
const string kContainerItem = Name(kContainerPrefix, kItem);
@@ -475,8 +555,11 @@
writer.WriteXmlns("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#");
writer.StartWritingElement("rdf:Description");
writer.WriteXmlns(kContainerPrefix, kContainerUri);
- writer.WriteElementAndContent(Name(kContainerPrefix, kVersion), kVersionValue);
- writer.WriteElementAndContent(Name(kContainerPrefix, "HdrRatio"), hdr_ratio);
+ writer.WriteElementAndContent(Name(kContainerPrefix, kVersion), metadata.version);
+ writer.WriteElementAndContent(Name(kContainerPrefix, "rangeScalingFactor"),
+ metadata.rangeScalingFactor);
+ // TODO: determine structure for hdr10 metadata
+ // TODO: write rest of metadata
writer.StartWritingElements(kConDirSeq);
size_t item_depth = writer.StartWritingElements(kLiItem);
writer.WriteAttributeNameAndValue(kItemSemantic, kPrimary);
diff --git a/libs/jpegrecoverymap/recoverymapmath.cpp b/libs/jpegrecoverymap/recoverymapmath.cpp
index 0d3319f..4541f9b 100644
--- a/libs/jpegrecoverymap/recoverymapmath.cpp
+++ b/libs/jpegrecoverymap/recoverymapmath.cpp
@@ -64,6 +64,11 @@
////////////////////////////////////////////////////////////////////////////////
// Display-P3 transformations
+static const float kP3R = 0.22897f, kP3G = 0.69174f, kP3B = 0.07929f;
+
+float p3Luminance(Color e) {
+ return kP3R * e.r + kP3G * e.g + kP3B * e.b;
+}
////////////////////////////////////////////////////////////////////////////////
@@ -128,7 +133,7 @@
return {{{ hlgOetf(e.r), hlgOetf(e.g), hlgOetf(e.b) }}};
}
-float hlgInvOetf(float e_gamma) {
+static float hlgInvOetf(float e_gamma) {
if (e_gamma <= 0.5f) {
return pow(e_gamma, 2.0f) / 3.0f;
} else {
@@ -142,10 +147,118 @@
hlgInvOetf(e_gamma.b) }}};
}
+static const float kPqM1 = 2610.0f / 16384.0f, kPqM2 = 2523.0f / 4096.0f * 128.0f;
+static const float kPqC1 = 3424.0f / 4096.0f, kPqC2 = 2413.0f / 4096.0f * 32.0f,
+ kPqC3 = 2392.0f / 4096.0f * 32.0f;
+
+static float pqOetf(float e) {
+ if (e < 0.0f) e = 0.0f;
+ return pow((kPqC1 + kPqC2 * pow(e / 10000.0f, kPqM1)) / (1 + kPqC3 * pow(e / 10000.0f, kPqM1)),
+ kPqM2);
+}
+
+Color pqOetf(Color e) {
+ return {{{ pqOetf(e.r), pqOetf(e.g), pqOetf(e.b) }}};
+}
+
+static float pqInvOetf(float e_gamma) {
+ static const float kPqInvOetfCoef = log2(-(pow(kPqM1, 1.0f / kPqM2) - kPqC1)
+ / (kPqC3 * pow(kPqM1, 1.0f / kPqM2) - kPqC2));
+ return kPqInvOetfCoef / log2(e_gamma * 10000.0f);
+}
+
+Color pqInvOetf(Color e_gamma) {
+ return {{{ pqInvOetf(e_gamma.r),
+ pqInvOetf(e_gamma.g),
+ pqInvOetf(e_gamma.b) }}};
+}
+
////////////////////////////////////////////////////////////////////////////////
// Color conversions
+Color bt709ToP3(Color e) {
+ return {{{ 0.82254f * e.r + 0.17755f * e.g + 0.00006f * e.b,
+ 0.03312f * e.r + 0.96684f * e.g + -0.00001f * e.b,
+ 0.01706f * e.r + 0.07240f * e.g + 0.91049f * e.b }}};
+}
+
+Color bt709ToBt2100(Color e) {
+ return {{{ 0.62740f * e.r + 0.32930f * e.g + 0.04332f * e.b,
+ 0.06904f * e.r + 0.91958f * e.g + 0.01138f * e.b,
+ 0.01636f * e.r + 0.08799f * e.g + 0.89555f * e.b }}};
+}
+
+Color p3ToBt709(Color e) {
+ return {{{ 1.22482f * e.r + -0.22490f * e.g + -0.00007f * e.b,
+ -0.04196f * e.r + 1.04199f * e.g + 0.00001f * e.b,
+ -0.01961f * e.r + -0.07865f * e.g + 1.09831f * e.b }}};
+}
+
+Color p3ToBt2100(Color e) {
+ return {{{ 0.75378f * e.r + 0.19862f * e.g + 0.04754f * e.b,
+ 0.04576f * e.r + 0.94177f * e.g + 0.01250f * e.b,
+ -0.00121f * e.r + 0.01757f * e.g + 0.98359f * e.b }}};
+}
+
+Color bt2100ToBt709(Color e) {
+ return {{{ 1.66045f * e.r + -0.58764f * e.g + -0.07286f * e.b,
+ -0.12445f * e.r + 1.13282f * e.g + -0.00837f * e.b,
+ -0.01811f * e.r + -0.10057f * e.g + 1.11878f * e.b }}};
+}
+
+Color bt2100ToP3(Color e) {
+ return {{{ 1.34369f * e.r + -0.28223f * e.g + -0.06135f * e.b,
+ -0.06533f * e.r + 1.07580f * e.g + -0.01051f * e.b,
+ 0.00283f * e.r + -0.01957f * e.g + 1.01679f * e.b
+ }}};
+}
+
+// TODO: confirm we always want to convert like this before calculating
+// luminance.
+ColorTransformFn getHdrConversionFn(jpegr_color_gamut sdr_gamut, jpegr_color_gamut hdr_gamut) {
+ switch (sdr_gamut) {
+ case JPEGR_COLORGAMUT_BT709:
+ switch (hdr_gamut) {
+ case JPEGR_COLORGAMUT_BT709:
+ return identityConversion;
+ case JPEGR_COLORGAMUT_P3:
+ return p3ToBt709;
+ case JPEGR_COLORGAMUT_BT2100:
+ return bt2100ToBt709;
+ case JPEGR_COLORGAMUT_UNSPECIFIED:
+ return nullptr;
+ }
+ break;
+ case JPEGR_COLORGAMUT_P3:
+ switch (hdr_gamut) {
+ case JPEGR_COLORGAMUT_BT709:
+ return bt709ToP3;
+ case JPEGR_COLORGAMUT_P3:
+ return identityConversion;
+ case JPEGR_COLORGAMUT_BT2100:
+ return bt2100ToP3;
+ case JPEGR_COLORGAMUT_UNSPECIFIED:
+ return nullptr;
+ }
+ break;
+ case JPEGR_COLORGAMUT_BT2100:
+ switch (hdr_gamut) {
+ case JPEGR_COLORGAMUT_BT709:
+ return bt709ToBt2100;
+ case JPEGR_COLORGAMUT_P3:
+ return p3ToBt2100;
+ case JPEGR_COLORGAMUT_BT2100:
+ return identityConversion;
+ case JPEGR_COLORGAMUT_UNSPECIFIED:
+ return nullptr;
+ }
+ break;
+ case JPEGR_COLORGAMUT_UNSPECIFIED:
+ return nullptr;
+ }
+}
+
////////////////////////////////////////////////////////////////////////////////
// Recovery map calculations
@@ -257,4 +370,11 @@
return samplePixels(image, map_scale_factor, x, y, getP010Pixel);
}
+uint32_t colorToRgba1010102(Color e_gamma) {
+ return (0x3ff & static_cast<uint32_t>(e_gamma.r * 1023.0f))
+ | ((0x3ff & static_cast<uint32_t>(e_gamma.g * 1023.0f)) << 10)
+ | ((0x3ff & static_cast<uint32_t>(e_gamma.b * 1023.0f)) << 20)
+ | (0x3 << 30); // Set alpha to 1.0
+}
+
} // namespace android::recoverymap
diff --git a/libs/jpegrecoverymap/tests/recoverymap_test.cpp b/libs/jpegrecoverymap/tests/recoverymap_test.cpp
index 226056b..b3cd37e 100644
--- a/libs/jpegrecoverymap/tests/recoverymap_test.cpp
+++ b/libs/jpegrecoverymap/tests/recoverymap_test.cpp
@@ -37,9 +37,11 @@
TEST_F(RecoveryMapTest, build) {
// Force all of the recovery map lib to be linked by calling all public functions.
RecoveryMap recovery_map;
- recovery_map.encodeJPEGR(nullptr, nullptr, nullptr, 0, nullptr);
- recovery_map.encodeJPEGR(nullptr, nullptr, nullptr, nullptr);
- recovery_map.encodeJPEGR(nullptr, nullptr, nullptr);
+ recovery_map.encodeJPEGR(nullptr, nullptr, static_cast<jpegr_transfer_function>(0),
+ nullptr, 0, nullptr);
+ recovery_map.encodeJPEGR(nullptr, nullptr, nullptr, static_cast<jpegr_transfer_function>(0),
+ nullptr);
+ recovery_map.encodeJPEGR(nullptr, nullptr, static_cast<jpegr_transfer_function>(0), nullptr);
recovery_map.decodeJPEGR(nullptr, nullptr, nullptr, false);
}