blob: a898f1e4429ec20d7f876b6b0b26bbcc522658a1 [file] [log] [blame]
Dichen Zhang85b37562022-10-11 11:08:28 -07001/*
2 * Copyright 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <jpegrecoverymap/recoverymap.h>
Nick Deakinf6bca5a2022-11-04 10:43:43 -040018#include <jpegrecoverymap/jpegencoder.h>
19#include <jpegrecoverymap/jpegdecoder.h>
20#include <jpegrecoverymap/recoverymapmath.h>
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +000021#include <jpegrecoverymap/recoverymaputils.h>
Nick Deakinf6bca5a2022-11-04 10:43:43 -040022
Dichen Zhanga8766262022-11-07 23:48:24 +000023#include <image_io/jpeg/jpeg_marker.h>
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +000024#include <image_io/jpeg/jpeg_info.h>
25#include <image_io/jpeg/jpeg_scanner.h>
26#include <image_io/jpeg/jpeg_info_builder.h>
27#include <image_io/base/data_segment_data_source.h>
Nick Deakinf6bca5a2022-11-04 10:43:43 -040028
29#include <memory>
Dichen Zhang72fd2b12022-11-01 06:11:50 +000030#include <sstream>
31#include <string>
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +000032#include <cmath>
Harish Mahendrakar72b6f302022-12-16 10:39:15 -080033#include <condition_variable>
34#include <deque>
35#include <mutex>
36#include <thread>
37#include <unistd.h>
Dichen Zhang72fd2b12022-11-01 06:11:50 +000038
39using namespace std;
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +000040using namespace photos_editing_formats::image_io;
Dichen Zhang85b37562022-10-11 11:08:28 -070041
42namespace android::recoverymap {
43
Harish Mahendrakar555a06b2022-12-14 09:37:27 -080044#define USE_SRGB_INVOETF_LUT 1
45#define USE_HLG_OETF_LUT 1
46#define USE_PQ_OETF_LUT 1
47#define USE_HLG_INVOETF_LUT 1
48#define USE_PQ_INVOETF_LUT 1
49
Nick Deakinf6bca5a2022-11-04 10:43:43 -040050#define JPEGR_CHECK(x) \
51 { \
52 status_t status = (x); \
53 if ((status) != NO_ERROR) { \
54 return status; \
55 } \
56 }
57
Nick Deakin6bd90432022-11-20 16:26:37 -050058// The current JPEGR version that we encode to
59static const uint32_t kJpegrVersion = 1;
60
Nick Deakinf6bca5a2022-11-04 10:43:43 -040061// Map is quarter res / sixteenth size
62static const size_t kMapDimensionScaleFactor = 4;
Dichen Zhang0b9f7de2022-11-18 06:52:46 +000063// JPEG compress quality (0 ~ 100) for recovery map
64static const int kMapCompressQuality = 85;
Nick Deakinf6bca5a2022-11-04 10:43:43 -040065
Nick Deakin6bd90432022-11-20 16:26:37 -050066// TODO: fill in st2086 metadata
67static const st2086_metadata kSt2086Metadata = {
68 {0.0f, 0.0f},
69 {0.0f, 0.0f},
70 {0.0f, 0.0f},
71 {0.0f, 0.0f},
72 0,
73 1.0f,
74};
Nick Deakinf6bca5a2022-11-04 10:43:43 -040075
Harish Mahendrakar72b6f302022-12-16 10:39:15 -080076#define CONFIG_MULTITHREAD 1
77int GetCPUCoreCount() {
78 int cpuCoreCount = 1;
79#if CONFIG_MULTITHREAD
80#if defined(_SC_NPROCESSORS_ONLN)
81 cpuCoreCount = sysconf(_SC_NPROCESSORS_ONLN);
82#else
83 // _SC_NPROC_ONLN must be defined...
84 cpuCoreCount = sysconf(_SC_NPROC_ONLN);
85#endif
86#endif
87 return cpuCoreCount;
88}
89
Dichen Zhang72fd2b12022-11-01 06:11:50 +000090/*
Dichen Zhanga8766262022-11-07 23:48:24 +000091 * Helper function used for writing data to destination.
92 *
93 * @param destination destination of the data to be written.
94 * @param source source of data being written.
95 * @param length length of the data to be written.
96 * @param position cursor in desitination where the data is to be written.
97 * @return status of succeed or error code.
98 */
99status_t Write(jr_compressed_ptr destination, const void* source, size_t length, int &position) {
Dichen Zhang0b9f7de2022-11-18 06:52:46 +0000100 if (position + length > destination->maxLength) {
Dichen Zhanga8766262022-11-07 23:48:24 +0000101 return ERROR_JPEGR_BUFFER_TOO_SMALL;
102 }
103
104 memcpy((uint8_t*)destination->data + sizeof(uint8_t) * position, source, length);
105 position += length;
106 return NO_ERROR;
107}
108
Dichen Zhangd18bc302022-12-16 20:55:24 +0000109status_t Write(jr_exif_ptr destination, const void* source, size_t length, int &position) {
110 memcpy((uint8_t*)destination->data + sizeof(uint8_t) * position, source, length);
111 position += length;
112 return NO_ERROR;
113}
114
115// If the EXIF package doesn't exist in the input JPEG, we'll create one with one entry
116// where the length is represented by this value.
117const size_t PSEUDO_EXIF_PACKAGE_LENGTH = 28;
118// If the EXIF package exists in the input JPEG, we'll add an "JR" entry where the length is
119// represented by this value.
120const size_t EXIF_J_R_ENTRY_LENGTH = 12;
121
122/*
123 * Helper function
124 * Add J R entry to existing exif, or create a new one with J R entry if it's null.
125 * EXIF syntax / change:
126 * ori:
127 * FF E1 - APP1
128 * 01 FC - size of APP1 (to be calculated)
129 * -----------------------------------------------------
130 * 45 78 69 66 00 00 - Exif\0\0 "Exif header"
131 * 49 49 2A 00 - TIFF Header
132 * 08 00 00 00 - offset to the IFD (image file directory)
133 * 06 00 - 6 entries
134 * 00 01 - Width Tag
135 * 03 00 - 'Short' type
136 * 01 00 00 00 - one entry
137 * 00 05 00 00 - image with 0x500
138 *--------------------------------------------------------------------------
139 * new:
140 * FF E1 - APP1
141 * 02 08 - new size, equals to old size + EXIF_J_R_ENTRY_LENGTH (12)
142 *-----------------------------------------------------
143 * 45 78 69 66 00 00 - Exif\0\0 "Exif header"
144 * 49 49 2A 00 - TIFF Header
145 * 08 00 00 00 - offset to the IFD (image file directory)
146 * 07 00 - +1 entry
147 * 4A 52 Custom ('J''R') Tag
148 * 07 00 - Unknown type
149 * 01 00 00 00 - one element
150 * 00 00 00 00 - empty data
151 * 00 01 - Width Tag
152 * 03 00 - 'Short' type
153 * 01 00 00 00 - one entry
154 * 00 05 00 00 - image with 0x500
155 */
156status_t updateExif(jr_exif_ptr exif, jr_exif_ptr dest) {
157 if (exif == nullptr || exif->data == nullptr) {
158 uint8_t data[PSEUDO_EXIF_PACKAGE_LENGTH] = {
159 0x45, 0x78, 0x69, 0x66, 0x00, 0x00,
160 0x49, 0x49, 0x2A, 0x00,
161 0x08, 0x00, 0x00, 0x00,
162 0x01, 0x00,
163 0x4A, 0x52,
164 0x07, 0x00,
165 0x01, 0x00, 0x00, 0x00,
166 0x00, 0x00, 0x00, 0x00};
167 int pos = 0;
168 Write(dest, data, PSEUDO_EXIF_PACKAGE_LENGTH, pos);
169 return NO_ERROR;
170 }
171
172 int num_entry = 0;
173 uint8_t num_entry_low = 0;
174 uint8_t num_entry_high = 0;
175 bool use_big_endian = false;
176 if (reinterpret_cast<uint16_t*>(exif->data)[3] == 0x4949) {
177 num_entry_low = reinterpret_cast<uint8_t*>(exif->data)[14];
178 num_entry_high = reinterpret_cast<uint8_t*>(exif->data)[15];
179 } else if (reinterpret_cast<uint16_t*>(exif->data)[3] == 0x4d4d) {
180 use_big_endian = true;
181 num_entry_high = reinterpret_cast<uint8_t*>(exif->data)[14];
182 num_entry_low = reinterpret_cast<uint8_t*>(exif->data)[15];
183 } else {
184 return ERROR_JPEGR_METADATA_ERROR;
185 }
186 num_entry = (num_entry_high << 8) | num_entry_low;
187 num_entry += 1;
188 num_entry_low = num_entry & 0xff;
189 num_entry_high = (num_entry << 8) & 0xff;
190
191 int pos = 0;
192 Write(dest, (uint8_t*)exif->data, 14, pos);
193
194 if (use_big_endian) {
195 Write(dest, &num_entry_high, 1, pos);
196 Write(dest, &num_entry_low, 1, pos);
197 uint8_t data[EXIF_J_R_ENTRY_LENGTH] = {
198 0x4A, 0x52,
199 0x07, 0x00,
200 0x01, 0x00, 0x00, 0x00,
201 0x00, 0x00, 0x00, 0x00};
202 Write(dest, data, EXIF_J_R_ENTRY_LENGTH, pos);
203 } else {
204 Write(dest, &num_entry_low, 1, pos);
205 Write(dest, &num_entry_high, 1, pos);
206 uint8_t data[EXIF_J_R_ENTRY_LENGTH] = {
207 0x4A, 0x52,
208 0x00, 0x07,
209 0x00, 0x00, 0x00, 0x01,
210 0x00, 0x00, 0x00, 0x00};
211 Write(dest, data, EXIF_J_R_ENTRY_LENGTH, pos);
212 }
213
214 Write(dest, (uint8_t*)exif->data + 16, exif->length - 16, pos);
215
216 return NO_ERROR;
217}
218
Dichen Zhang63d92512023-01-04 12:01:16 -0800219/*
220 * Helper function copies the JPEG image from without EXIF.
221 *
222 * @param dest destination of the data to be written.
223 * @param source source of data being written.
224 * @param exif_pos position of the EXIF package, which is aligned with jpegdecoder.getEXIFPos().
225 * (4 bypes offset to FF sign, the byte after FF E1 XX XX <this byte>).
226 * @param exif_size exif size without the initial 4 bytes, aligned with jpegdecoder.getEXIFSize().
227 */
228void copyJpegWithoutExif(jr_compressed_ptr dest,
229 jr_compressed_ptr source,
230 size_t exif_pos,
231 size_t exif_size) {
232 memcpy(dest, source, sizeof(jpegr_compressed_struct));
233
234 const size_t exif_offset = 4; //exif_pos has 4 bypes offset to the FF sign
235 dest->length = source->length - exif_size - exif_offset;
236 dest->data = malloc(dest->length);
237
238 memcpy(dest->data, source->data, exif_pos - exif_offset);
239 memcpy((uint8_t*)dest->data + exif_pos - exif_offset,
240 (uint8_t*)source->data + exif_pos + exif_size,
241 source->length - exif_pos - exif_size);
242}
243
Dichen Zhang636f5242022-12-07 20:25:44 +0000244/* Encode API-0 */
245status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,
246 jpegr_transfer_function hdr_tf,
247 jr_compressed_ptr dest,
248 int quality,
Dichen Zhangd18bc302022-12-16 20:55:24 +0000249 jr_exif_ptr exif) {
Dichen Zhang636f5242022-12-07 20:25:44 +0000250 if (uncompressed_p010_image == nullptr || dest == nullptr) {
251 return ERROR_JPEGR_INVALID_NULL_PTR;
252 }
253
254 if (quality < 0 || quality > 100) {
255 return ERROR_JPEGR_INVALID_INPUT_TYPE;
256 }
257
258 jpegr_metadata metadata;
259 metadata.version = kJpegrVersion;
260 metadata.transferFunction = hdr_tf;
261 if (hdr_tf == JPEGR_TF_PQ) {
262 metadata.hdr10Metadata.st2086Metadata = kSt2086Metadata;
263 }
264
265 jpegr_uncompressed_struct uncompressed_yuv_420_image;
Dichen Zhangc3437ca2023-01-04 14:00:08 -0800266 unique_ptr<uint8_t[]> uncompressed_yuv_420_image_data = make_unique<uint8_t[]>(
267 uncompressed_p010_image->width * uncompressed_p010_image->height * 3 / 2);
268 uncompressed_yuv_420_image.data = uncompressed_yuv_420_image_data.get();
Dichen Zhang636f5242022-12-07 20:25:44 +0000269 JPEGR_CHECK(toneMap(uncompressed_p010_image, &uncompressed_yuv_420_image));
270
271 jpegr_uncompressed_struct map;
272 JPEGR_CHECK(generateRecoveryMap(
273 &uncompressed_yuv_420_image, uncompressed_p010_image, &metadata, &map));
274 std::unique_ptr<uint8_t[]> map_data;
275 map_data.reset(reinterpret_cast<uint8_t*>(map.data));
276
277 jpegr_compressed_struct compressed_map;
278 compressed_map.maxLength = map.width * map.height;
279 unique_ptr<uint8_t[]> compressed_map_data = make_unique<uint8_t[]>(compressed_map.maxLength);
280 compressed_map.data = compressed_map_data.get();
281 JPEGR_CHECK(compressRecoveryMap(&map, &compressed_map));
282
283 JpegEncoder jpeg_encoder;
284 // TODO: determine ICC data based on color gamut information
285 if (!jpeg_encoder.compressImage(uncompressed_yuv_420_image.data,
286 uncompressed_yuv_420_image.width,
287 uncompressed_yuv_420_image.height, quality, nullptr, 0)) {
288 return ERROR_JPEGR_ENCODE_ERROR;
289 }
290 jpegr_compressed_struct jpeg;
291 jpeg.data = jpeg_encoder.getCompressedImagePtr();
292 jpeg.length = jpeg_encoder.getCompressedImageSize();
293
Dichen Zhangd18bc302022-12-16 20:55:24 +0000294 jpegr_exif_struct new_exif;
Dichen Zhangc3437ca2023-01-04 14:00:08 -0800295 if (exif == nullptr || exif->data == nullptr) {
Dichen Zhangd18bc302022-12-16 20:55:24 +0000296 new_exif.length = PSEUDO_EXIF_PACKAGE_LENGTH;
297 } else {
298 new_exif.length = exif->length + EXIF_J_R_ENTRY_LENGTH;
299 }
300 new_exif.data = new uint8_t[new_exif.length];
301 std::unique_ptr<uint8_t[]> new_exif_data;
302 new_exif_data.reset(reinterpret_cast<uint8_t*>(new_exif.data));
303 JPEGR_CHECK(updateExif(exif, &new_exif));
304
305 JPEGR_CHECK(appendRecoveryMap(&jpeg, &compressed_map, &new_exif, &metadata, dest));
Dichen Zhang636f5242022-12-07 20:25:44 +0000306
307 return NO_ERROR;
308}
309
310/* Encode API-1 */
Dichen Zhang6947d532022-10-22 02:16:21 +0000311status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,
312 jr_uncompressed_ptr uncompressed_yuv_420_image,
Nick Deakin6bd90432022-11-20 16:26:37 -0500313 jpegr_transfer_function hdr_tf,
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400314 jr_compressed_ptr dest,
Dichen Zhangffa34012022-11-03 23:21:13 +0000315 int quality,
Dichen Zhangd18bc302022-12-16 20:55:24 +0000316 jr_exif_ptr exif) {
Dichen Zhang6947d532022-10-22 02:16:21 +0000317 if (uncompressed_p010_image == nullptr
318 || uncompressed_yuv_420_image == nullptr
319 || dest == nullptr) {
Dichen Zhang80b72482022-11-02 01:55:35 +0000320 return ERROR_JPEGR_INVALID_NULL_PTR;
Dichen Zhang6947d532022-10-22 02:16:21 +0000321 }
322
Dichen Zhangffa34012022-11-03 23:21:13 +0000323 if (quality < 0 || quality > 100) {
324 return ERROR_JPEGR_INVALID_INPUT_TYPE;
325 }
326
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400327 if (uncompressed_p010_image->width != uncompressed_yuv_420_image->width
328 || uncompressed_p010_image->height != uncompressed_yuv_420_image->height) {
329 return ERROR_JPEGR_RESOLUTION_MISMATCH;
330 }
331
Nick Deakin6bd90432022-11-20 16:26:37 -0500332 jpegr_metadata metadata;
333 metadata.version = kJpegrVersion;
334 metadata.transferFunction = hdr_tf;
335 if (hdr_tf == JPEGR_TF_PQ) {
336 metadata.hdr10Metadata.st2086Metadata = kSt2086Metadata;
337 }
338
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400339 jpegr_uncompressed_struct map;
Dichen Zhanga8766262022-11-07 23:48:24 +0000340 JPEGR_CHECK(generateRecoveryMap(
Nick Deakin6bd90432022-11-20 16:26:37 -0500341 uncompressed_yuv_420_image, uncompressed_p010_image, &metadata, &map));
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400342 std::unique_ptr<uint8_t[]> map_data;
343 map_data.reset(reinterpret_cast<uint8_t*>(map.data));
344
345 jpegr_compressed_struct compressed_map;
Dichen Zhang0b9f7de2022-11-18 06:52:46 +0000346 compressed_map.maxLength = map.width * map.height;
347 unique_ptr<uint8_t[]> compressed_map_data = make_unique<uint8_t[]>(compressed_map.maxLength);
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400348 compressed_map.data = compressed_map_data.get();
349 JPEGR_CHECK(compressRecoveryMap(&map, &compressed_map));
350
351 JpegEncoder jpeg_encoder;
Nick Deakin6bd90432022-11-20 16:26:37 -0500352 // TODO: determine ICC data based on color gamut information
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400353 if (!jpeg_encoder.compressImage(uncompressed_yuv_420_image->data,
354 uncompressed_yuv_420_image->width,
Dichen Zhang95cbb9f2022-11-07 18:32:05 +0000355 uncompressed_yuv_420_image->height, quality, nullptr, 0)) {
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400356 return ERROR_JPEGR_ENCODE_ERROR;
357 }
358 jpegr_compressed_struct jpeg;
359 jpeg.data = jpeg_encoder.getCompressedImagePtr();
360 jpeg.length = jpeg_encoder.getCompressedImageSize();
361
Dichen Zhangd18bc302022-12-16 20:55:24 +0000362 jpegr_exif_struct new_exif;
363 if (exif == nullptr || exif->data == nullptr) {
364 new_exif.length = PSEUDO_EXIF_PACKAGE_LENGTH;
365 } else {
366 new_exif.length = exif->length + EXIF_J_R_ENTRY_LENGTH;
367 }
368
369 new_exif.data = new uint8_t[new_exif.length];
370 std::unique_ptr<uint8_t[]> new_exif_data;
371 new_exif_data.reset(reinterpret_cast<uint8_t*>(new_exif.data));
372 JPEGR_CHECK(updateExif(exif, &new_exif));
373
374 JPEGR_CHECK(appendRecoveryMap(&jpeg, &compressed_map, &new_exif, &metadata, dest));
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400375
Dichen Zhang6947d532022-10-22 02:16:21 +0000376 return NO_ERROR;
377}
378
Dichen Zhang636f5242022-12-07 20:25:44 +0000379/* Encode API-2 */
Dichen Zhang6947d532022-10-22 02:16:21 +0000380status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,
381 jr_uncompressed_ptr uncompressed_yuv_420_image,
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400382 jr_compressed_ptr compressed_jpeg_image,
Nick Deakin6bd90432022-11-20 16:26:37 -0500383 jpegr_transfer_function hdr_tf,
Dichen Zhang95cbb9f2022-11-07 18:32:05 +0000384 jr_compressed_ptr dest) {
Dichen Zhang6947d532022-10-22 02:16:21 +0000385 if (uncompressed_p010_image == nullptr
386 || uncompressed_yuv_420_image == nullptr
387 || compressed_jpeg_image == nullptr
388 || dest == nullptr) {
Dichen Zhang80b72482022-11-02 01:55:35 +0000389 return ERROR_JPEGR_INVALID_NULL_PTR;
Dichen Zhang6947d532022-10-22 02:16:21 +0000390 }
391
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400392 if (uncompressed_p010_image->width != uncompressed_yuv_420_image->width
393 || uncompressed_p010_image->height != uncompressed_yuv_420_image->height) {
394 return ERROR_JPEGR_RESOLUTION_MISMATCH;
395 }
396
Nick Deakin6bd90432022-11-20 16:26:37 -0500397 jpegr_metadata metadata;
398 metadata.version = kJpegrVersion;
399 metadata.transferFunction = hdr_tf;
400 if (hdr_tf == JPEGR_TF_PQ) {
401 metadata.hdr10Metadata.st2086Metadata = kSt2086Metadata;
402 }
403
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400404 jpegr_uncompressed_struct map;
Dichen Zhanga8766262022-11-07 23:48:24 +0000405 JPEGR_CHECK(generateRecoveryMap(
Nick Deakin6bd90432022-11-20 16:26:37 -0500406 uncompressed_yuv_420_image, uncompressed_p010_image, &metadata, &map));
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400407 std::unique_ptr<uint8_t[]> map_data;
408 map_data.reset(reinterpret_cast<uint8_t*>(map.data));
409
410 jpegr_compressed_struct compressed_map;
Dichen Zhang0b9f7de2022-11-18 06:52:46 +0000411 compressed_map.maxLength = map.width * map.height;
412 unique_ptr<uint8_t[]> compressed_map_data = make_unique<uint8_t[]>(compressed_map.maxLength);
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400413 compressed_map.data = compressed_map_data.get();
414 JPEGR_CHECK(compressRecoveryMap(&map, &compressed_map));
415
Dichen Zhangd18bc302022-12-16 20:55:24 +0000416 // Extract EXIF from JPEG without decoding.
417 JpegDecoder jpeg_decoder;
418 if (!jpeg_decoder.extractEXIF(compressed_jpeg_image->data, compressed_jpeg_image->length)) {
419 return ERROR_JPEGR_DECODE_ERROR;
420 }
421
Dichen Zhang63d92512023-01-04 12:01:16 -0800422 // Update exif.
Dichen Zhangd18bc302022-12-16 20:55:24 +0000423 jpegr_exif_struct exif;
424 exif.data = nullptr;
425 exif.length = 0;
Dichen Zhang63d92512023-01-04 12:01:16 -0800426 jpegr_compressed_struct new_jpeg_image;
427 new_jpeg_image.data = nullptr;
428 new_jpeg_image.length = 0;
Dichen Zhangd18bc302022-12-16 20:55:24 +0000429 if (jpeg_decoder.getEXIFPos() != 0) {
Dichen Zhang63d92512023-01-04 12:01:16 -0800430 copyJpegWithoutExif(&new_jpeg_image,
431 compressed_jpeg_image,
432 jpeg_decoder.getEXIFPos(),
433 jpeg_decoder.getEXIFSize());
Dichen Zhangd18bc302022-12-16 20:55:24 +0000434 exif.data = jpeg_decoder.getEXIFPtr();
435 exif.length = jpeg_decoder.getEXIFSize();
436 }
437
438 jpegr_exif_struct new_exif;
439 if (exif.data == nullptr) {
440 new_exif.length = PSEUDO_EXIF_PACKAGE_LENGTH;
441 } else {
442 new_exif.length = exif.length + EXIF_J_R_ENTRY_LENGTH;
443 }
444
445 new_exif.data = new uint8_t[new_exif.length];
446 std::unique_ptr<uint8_t[]> new_exif_data;
447 new_exif_data.reset(reinterpret_cast<uint8_t*>(new_exif.data));
448 JPEGR_CHECK(updateExif(&exif, &new_exif));
449
450 JPEGR_CHECK(appendRecoveryMap(
Dichen Zhang63d92512023-01-04 12:01:16 -0800451 new_jpeg_image.data == nullptr ? compressed_jpeg_image : &new_jpeg_image,
452 &compressed_map, &new_exif, &metadata, dest));
453
454 if (new_jpeg_image.data != nullptr) {
455 free(new_jpeg_image.data);
456 }
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400457
Dichen Zhang6947d532022-10-22 02:16:21 +0000458 return NO_ERROR;
459}
460
Dichen Zhang636f5242022-12-07 20:25:44 +0000461/* Encode API-3 */
Dichen Zhang6947d532022-10-22 02:16:21 +0000462status_t RecoveryMap::encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400463 jr_compressed_ptr compressed_jpeg_image,
Nick Deakin6bd90432022-11-20 16:26:37 -0500464 jpegr_transfer_function hdr_tf,
Dichen Zhang95cbb9f2022-11-07 18:32:05 +0000465 jr_compressed_ptr dest) {
Dichen Zhang6947d532022-10-22 02:16:21 +0000466 if (uncompressed_p010_image == nullptr
467 || compressed_jpeg_image == nullptr
468 || dest == nullptr) {
Dichen Zhang80b72482022-11-02 01:55:35 +0000469 return ERROR_JPEGR_INVALID_NULL_PTR;
Dichen Zhang6947d532022-10-22 02:16:21 +0000470 }
471
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400472 JpegDecoder jpeg_decoder;
473 if (!jpeg_decoder.decompressImage(compressed_jpeg_image->data, compressed_jpeg_image->length)) {
474 return ERROR_JPEGR_DECODE_ERROR;
475 }
476 jpegr_uncompressed_struct uncompressed_yuv_420_image;
477 uncompressed_yuv_420_image.data = jpeg_decoder.getDecompressedImagePtr();
478 uncompressed_yuv_420_image.width = jpeg_decoder.getDecompressedImageWidth();
479 uncompressed_yuv_420_image.height = jpeg_decoder.getDecompressedImageHeight();
Nick Deakin6bd90432022-11-20 16:26:37 -0500480 uncompressed_yuv_420_image.colorGamut = compressed_jpeg_image->colorGamut;
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400481
Dichen Zhang63d92512023-01-04 12:01:16 -0800482 // Update exif.
Dichen Zhangd18bc302022-12-16 20:55:24 +0000483 jpegr_exif_struct exif;
484 exif.data = nullptr;
485 exif.length = 0;
Dichen Zhang63d92512023-01-04 12:01:16 -0800486 jpegr_compressed_struct new_jpeg_image;
487 new_jpeg_image.data = nullptr;
488 new_jpeg_image.length = 0;
Dichen Zhangd18bc302022-12-16 20:55:24 +0000489 if (jpeg_decoder.getEXIFPos() != 0) {
Dichen Zhang63d92512023-01-04 12:01:16 -0800490 copyJpegWithoutExif(&new_jpeg_image,
491 compressed_jpeg_image,
492 jpeg_decoder.getEXIFPos(),
493 jpeg_decoder.getEXIFSize());
Dichen Zhangd18bc302022-12-16 20:55:24 +0000494 exif.data = jpeg_decoder.getEXIFPtr();
495 exif.length = jpeg_decoder.getEXIFSize();
496 }
497
498 jpegr_exif_struct new_exif;
499 if (exif.data == nullptr) {
500 new_exif.length = PSEUDO_EXIF_PACKAGE_LENGTH;
501 } else {
502 new_exif.length = exif.length + EXIF_J_R_ENTRY_LENGTH;
503 }
504 new_exif.data = new uint8_t[new_exif.length];
505 std::unique_ptr<uint8_t[]> new_exif_data;
506 new_exif_data.reset(reinterpret_cast<uint8_t*>(new_exif.data));
507 JPEGR_CHECK(updateExif(&exif, &new_exif));
508
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400509 if (uncompressed_p010_image->width != uncompressed_yuv_420_image.width
510 || uncompressed_p010_image->height != uncompressed_yuv_420_image.height) {
511 return ERROR_JPEGR_RESOLUTION_MISMATCH;
512 }
513
Nick Deakin6bd90432022-11-20 16:26:37 -0500514 jpegr_metadata metadata;
515 metadata.version = kJpegrVersion;
516 metadata.transferFunction = hdr_tf;
517 if (hdr_tf == JPEGR_TF_PQ) {
518 metadata.hdr10Metadata.st2086Metadata = kSt2086Metadata;
519 }
520
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400521 jpegr_uncompressed_struct map;
Dichen Zhanga8766262022-11-07 23:48:24 +0000522 JPEGR_CHECK(generateRecoveryMap(
Nick Deakin6bd90432022-11-20 16:26:37 -0500523 &uncompressed_yuv_420_image, uncompressed_p010_image, &metadata, &map));
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400524 std::unique_ptr<uint8_t[]> map_data;
525 map_data.reset(reinterpret_cast<uint8_t*>(map.data));
526
527 jpegr_compressed_struct compressed_map;
Dichen Zhang0b9f7de2022-11-18 06:52:46 +0000528 compressed_map.maxLength = map.width * map.height;
529 unique_ptr<uint8_t[]> compressed_map_data = make_unique<uint8_t[]>(compressed_map.maxLength);
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400530 compressed_map.data = compressed_map_data.get();
531 JPEGR_CHECK(compressRecoveryMap(&map, &compressed_map));
532
Dichen Zhangd18bc302022-12-16 20:55:24 +0000533 JPEGR_CHECK(appendRecoveryMap(
Dichen Zhang63d92512023-01-04 12:01:16 -0800534 new_jpeg_image.data == nullptr ? compressed_jpeg_image : &new_jpeg_image,
535 &compressed_map, &new_exif, &metadata, dest));
536
537 if (new_jpeg_image.data != nullptr) {
538 free(new_jpeg_image.data);
539 }
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400540
Dichen Zhang6947d532022-10-22 02:16:21 +0000541 return NO_ERROR;
542}
543
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +0000544status_t RecoveryMap::getJPEGRInfo(jr_compressed_ptr compressed_jpegr_image,
545 jr_info_ptr jpegr_info) {
546 if (compressed_jpegr_image == nullptr || jpegr_info == nullptr) {
547 return ERROR_JPEGR_INVALID_NULL_PTR;
548 }
549
550 jpegr_compressed_struct primary_image, recovery_map;
551 JPEGR_CHECK(extractPrimaryImageAndRecoveryMap(compressed_jpegr_image,
552 &primary_image, &recovery_map));
553
554 JpegDecoder jpeg_decoder;
555 if (!jpeg_decoder.getCompressedImageParameters(primary_image.data, primary_image.length,
556 &jpegr_info->width, &jpegr_info->height,
557 jpegr_info->iccData, jpegr_info->exifData)) {
558 return ERROR_JPEGR_DECODE_ERROR;
559 }
560
561 return NO_ERROR;
562}
563
Dichen Zhang636f5242022-12-07 20:25:44 +0000564/* Decode API */
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400565status_t RecoveryMap::decodeJPEGR(jr_compressed_ptr compressed_jpegr_image,
Dichen Zhangffa34012022-11-03 23:21:13 +0000566 jr_uncompressed_ptr dest,
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +0000567 jr_exif_ptr exif,
568 bool request_sdr) {
Dichen Zhang6947d532022-10-22 02:16:21 +0000569 if (compressed_jpegr_image == nullptr || dest == nullptr) {
Dichen Zhang80b72482022-11-02 01:55:35 +0000570 return ERROR_JPEGR_INVALID_NULL_PTR;
Dichen Zhang6947d532022-10-22 02:16:21 +0000571 }
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +0000572 // TODO: fill EXIF data
573 (void) exif;
574
Fyodor Kyslovea9180f2023-01-06 01:11:43 +0000575 if (request_sdr) {
576 JpegDecoder jpeg_decoder;
577 if (!jpeg_decoder.decompressImage(compressed_jpegr_image->data, compressed_jpegr_image->length,
578 true)) {
579 return ERROR_JPEGR_DECODE_ERROR;
580 }
581 jpegr_uncompressed_struct uncompressed_rgba_image;
582 uncompressed_rgba_image.data = jpeg_decoder.getDecompressedImagePtr();
583 uncompressed_rgba_image.width = jpeg_decoder.getDecompressedImageWidth();
584 uncompressed_rgba_image.height = jpeg_decoder.getDecompressedImageHeight();
585 memcpy(dest->data, uncompressed_rgba_image.data,
586 uncompressed_rgba_image.width * uncompressed_rgba_image.height * 4);
587 dest->width = uncompressed_rgba_image.width;
588 dest->height = uncompressed_rgba_image.height;
589 return NO_ERROR;
590 }
591
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400592 jpegr_compressed_struct compressed_map;
Nick Deakin6bd90432022-11-20 16:26:37 -0500593 jpegr_metadata metadata;
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +0000594 JPEGR_CHECK(extractRecoveryMap(compressed_jpegr_image, &compressed_map));
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400595
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400596 JpegDecoder jpeg_decoder;
597 if (!jpeg_decoder.decompressImage(compressed_jpegr_image->data, compressed_jpegr_image->length)) {
598 return ERROR_JPEGR_DECODE_ERROR;
599 }
600
Fyodor Kyslovbf241572022-12-13 22:38:07 +0000601 JpegDecoder recovery_map_decoder;
Fyodor Kyslovea9180f2023-01-06 01:11:43 +0000602 if (!recovery_map_decoder.decompressImage(compressed_map.data, compressed_map.length)) {
Fyodor Kyslovbf241572022-12-13 22:38:07 +0000603 return ERROR_JPEGR_DECODE_ERROR;
604 }
605
606 jpegr_uncompressed_struct map;
607 map.data = recovery_map_decoder.getDecompressedImagePtr();
608 map.width = recovery_map_decoder.getDecompressedImageWidth();
609 map.height = recovery_map_decoder.getDecompressedImageHeight();
610
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400611 jpegr_uncompressed_struct uncompressed_yuv_420_image;
612 uncompressed_yuv_420_image.data = jpeg_decoder.getDecompressedImagePtr();
613 uncompressed_yuv_420_image.width = jpeg_decoder.getDecompressedImageWidth();
614 uncompressed_yuv_420_image.height = jpeg_decoder.getDecompressedImageHeight();
615
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +0000616 if (!getMetadataFromXMP(static_cast<uint8_t*>(jpeg_decoder.getXMPPtr()),
Fyodor Kyslovea9180f2023-01-06 01:11:43 +0000617 jpeg_decoder.getXMPSize(), &metadata)) {
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +0000618 return ERROR_JPEGR_DECODE_ERROR;
619 }
620
Fyodor Kyslovea9180f2023-01-06 01:11:43 +0000621 JPEGR_CHECK(applyRecoveryMap(&uncompressed_yuv_420_image, &map, &metadata, dest));
Dichen Zhang6947d532022-10-22 02:16:21 +0000622 return NO_ERROR;
623}
624
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400625status_t RecoveryMap::compressRecoveryMap(jr_uncompressed_ptr uncompressed_recovery_map,
626 jr_compressed_ptr dest) {
Dichen Zhang596a7562022-10-12 14:57:05 -0700627 if (uncompressed_recovery_map == nullptr || dest == nullptr) {
Dichen Zhang80b72482022-11-02 01:55:35 +0000628 return ERROR_JPEGR_INVALID_NULL_PTR;
Dichen Zhang596a7562022-10-12 14:57:05 -0700629 }
630
Nick Deakin6bd90432022-11-20 16:26:37 -0500631 // TODO: should we have ICC data for the map?
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400632 JpegEncoder jpeg_encoder;
Dichen Zhang0b9f7de2022-11-18 06:52:46 +0000633 if (!jpeg_encoder.compressImage(uncompressed_recovery_map->data,
634 uncompressed_recovery_map->width,
635 uncompressed_recovery_map->height,
636 kMapCompressQuality,
637 nullptr,
638 0,
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400639 true /* isSingleChannel */)) {
640 return ERROR_JPEGR_ENCODE_ERROR;
641 }
642
Dichen Zhang0b9f7de2022-11-18 06:52:46 +0000643 if (dest->maxLength < jpeg_encoder.getCompressedImageSize()) {
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400644 return ERROR_JPEGR_BUFFER_TOO_SMALL;
645 }
646
647 memcpy(dest->data, jpeg_encoder.getCompressedImagePtr(), jpeg_encoder.getCompressedImageSize());
648 dest->length = jpeg_encoder.getCompressedImageSize();
Nick Deakin6bd90432022-11-20 16:26:37 -0500649 dest->colorGamut = JPEGR_COLORGAMUT_UNSPECIFIED;
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400650
Dichen Zhang6947d532022-10-22 02:16:21 +0000651 return NO_ERROR;
Dichen Zhang596a7562022-10-12 14:57:05 -0700652}
653
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800654const int kJobSzInRows = 16;
655static_assert(kJobSzInRows > 0 && kJobSzInRows % kMapDimensionScaleFactor == 0,
656 "align job size to kMapDimensionScaleFactor");
657
658class JobQueue {
659 public:
660 bool dequeueJob(size_t& rowStart, size_t& rowEnd);
661 void enqueueJob(size_t rowStart, size_t rowEnd);
662 void markQueueForEnd();
663 void reset();
664
665 private:
666 bool mQueuedAllJobs = false;
667 std::deque<std::tuple<size_t, size_t>> mJobs;
668 std::mutex mMutex;
669 std::condition_variable mCv;
670};
671
672bool JobQueue::dequeueJob(size_t& rowStart, size_t& rowEnd) {
673 std::unique_lock<std::mutex> lock{mMutex};
674 while (true) {
675 if (mJobs.empty()) {
676 if (mQueuedAllJobs) {
677 return false;
678 } else {
679 mCv.wait(lock);
680 }
681 } else {
682 auto it = mJobs.begin();
683 rowStart = std::get<0>(*it);
684 rowEnd = std::get<1>(*it);
685 mJobs.erase(it);
686 return true;
687 }
688 }
689 return false;
690}
691
692void JobQueue::enqueueJob(size_t rowStart, size_t rowEnd) {
693 std::unique_lock<std::mutex> lock{mMutex};
694 mJobs.push_back(std::make_tuple(rowStart, rowEnd));
695 lock.unlock();
696 mCv.notify_one();
697}
698
699void JobQueue::markQueueForEnd() {
700 std::unique_lock<std::mutex> lock{mMutex};
701 mQueuedAllJobs = true;
702}
703
704void JobQueue::reset() {
705 std::unique_lock<std::mutex> lock{mMutex};
706 mJobs.clear();
707 mQueuedAllJobs = false;
708}
709
Dichen Zhang6947d532022-10-22 02:16:21 +0000710status_t RecoveryMap::generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image,
711 jr_uncompressed_ptr uncompressed_p010_image,
Nick Deakin6bd90432022-11-20 16:26:37 -0500712 jr_metadata_ptr metadata,
713 jr_uncompressed_ptr dest) {
Dichen Zhang596a7562022-10-12 14:57:05 -0700714 if (uncompressed_yuv_420_image == nullptr
715 || uncompressed_p010_image == nullptr
Nick Deakin6bd90432022-11-20 16:26:37 -0500716 || metadata == nullptr
Dichen Zhang596a7562022-10-12 14:57:05 -0700717 || dest == nullptr) {
Dichen Zhang80b72482022-11-02 01:55:35 +0000718 return ERROR_JPEGR_INVALID_NULL_PTR;
Dichen Zhang596a7562022-10-12 14:57:05 -0700719 }
720
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400721 if (uncompressed_yuv_420_image->width != uncompressed_p010_image->width
722 || uncompressed_yuv_420_image->height != uncompressed_p010_image->height) {
723 return ERROR_JPEGR_RESOLUTION_MISMATCH;
724 }
725
Nick Deakin6bd90432022-11-20 16:26:37 -0500726 if (uncompressed_yuv_420_image->colorGamut == JPEGR_COLORGAMUT_UNSPECIFIED
727 || uncompressed_p010_image->colorGamut == JPEGR_COLORGAMUT_UNSPECIFIED) {
728 return ERROR_JPEGR_INVALID_COLORGAMUT;
729 }
730
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400731 size_t image_width = uncompressed_yuv_420_image->width;
732 size_t image_height = uncompressed_yuv_420_image->height;
733 size_t map_width = image_width / kMapDimensionScaleFactor;
734 size_t map_height = image_height / kMapDimensionScaleFactor;
735
736 dest->width = map_width;
737 dest->height = map_height;
Nick Deakin6bd90432022-11-20 16:26:37 -0500738 dest->colorGamut = JPEGR_COLORGAMUT_UNSPECIFIED;
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400739 dest->data = new uint8_t[map_width * map_height];
740 std::unique_ptr<uint8_t[]> map_data;
741 map_data.reset(reinterpret_cast<uint8_t*>(dest->data));
742
Nick Deakin6bd90432022-11-20 16:26:37 -0500743 ColorTransformFn hdrInvOetf = nullptr;
Nick Deakin65f492a2022-11-29 22:47:40 -0500744 float hdr_white_nits = 0.0f;
Nick Deakin6bd90432022-11-20 16:26:37 -0500745 switch (metadata->transferFunction) {
Dichen Zhangdc8452b2022-11-23 17:17:56 +0000746 case JPEGR_TF_LINEAR:
747 hdrInvOetf = identityConversion;
748 break;
Nick Deakin6bd90432022-11-20 16:26:37 -0500749 case JPEGR_TF_HLG:
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800750#if USE_HLG_INVOETF_LUT
751 hdrInvOetf = hlgInvOetfLUT;
752#else
Nick Deakin6bd90432022-11-20 16:26:37 -0500753 hdrInvOetf = hlgInvOetf;
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800754#endif
Nick Deakin65f492a2022-11-29 22:47:40 -0500755 hdr_white_nits = kHlgMaxNits;
Nick Deakin6bd90432022-11-20 16:26:37 -0500756 break;
757 case JPEGR_TF_PQ:
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800758#if USE_PQ_INVOETF_LUT
759 hdrInvOetf = pqInvOetfLUT;
760#else
Nick Deakin6bd90432022-11-20 16:26:37 -0500761 hdrInvOetf = pqInvOetf;
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800762#endif
Nick Deakin65f492a2022-11-29 22:47:40 -0500763 hdr_white_nits = kPqMaxNits;
Nick Deakin6bd90432022-11-20 16:26:37 -0500764 break;
Dichen Zhangb27d06d2022-12-14 19:57:50 +0000765 case JPEGR_TF_UNSPECIFIED:
766 // Should be impossible to hit after input validation.
767 return ERROR_JPEGR_INVALID_TRANS_FUNC;
Nick Deakin6bd90432022-11-20 16:26:37 -0500768 }
769
770 ColorTransformFn hdrGamutConversionFn = getHdrConversionFn(
771 uncompressed_yuv_420_image->colorGamut, uncompressed_p010_image->colorGamut);
772
773 ColorCalculationFn luminanceFn = nullptr;
774 switch (uncompressed_yuv_420_image->colorGamut) {
775 case JPEGR_COLORGAMUT_BT709:
776 luminanceFn = srgbLuminance;
777 break;
778 case JPEGR_COLORGAMUT_P3:
779 luminanceFn = p3Luminance;
780 break;
781 case JPEGR_COLORGAMUT_BT2100:
782 luminanceFn = bt2100Luminance;
783 break;
784 case JPEGR_COLORGAMUT_UNSPECIFIED:
785 // Should be impossible to hit after input validation.
786 return ERROR_JPEGR_INVALID_COLORGAMUT;
787 }
788
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800789 std::mutex mutex;
Nick Deakin594a4ca2022-11-16 20:57:42 -0500790 float hdr_y_nits_max = 0.0f;
Nick Deakin6bd90432022-11-20 16:26:37 -0500791 double hdr_y_nits_avg = 0.0f;
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800792 const int threads = std::clamp(GetCPUCoreCount(), 1, 4);
793 size_t rowStep = threads == 1 ? image_height : kJobSzInRows;
794 JobQueue jobQueue;
Nick Deakin594a4ca2022-11-16 20:57:42 -0500795
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800796 std::function<void()> computeMetadata = [uncompressed_p010_image, hdrInvOetf,
797 hdrGamutConversionFn, luminanceFn, hdr_white_nits,
798 threads, &mutex, &hdr_y_nits_avg,
799 &hdr_y_nits_max, &jobQueue]() -> void {
800 size_t rowStart, rowEnd;
801 float hdr_y_nits_max_th = 0.0f;
802 double hdr_y_nits_avg_th = 0.0f;
803 while (jobQueue.dequeueJob(rowStart, rowEnd)) {
804 for (size_t y = rowStart; y < rowEnd; ++y) {
805 for (size_t x = 0; x < uncompressed_p010_image->width; ++x) {
806 Color hdr_yuv_gamma = getP010Pixel(uncompressed_p010_image, x, y);
807 Color hdr_rgb_gamma = bt2100YuvToRgb(hdr_yuv_gamma);
808 Color hdr_rgb = hdrInvOetf(hdr_rgb_gamma);
809 hdr_rgb = hdrGamutConversionFn(hdr_rgb);
810 float hdr_y_nits = luminanceFn(hdr_rgb) * hdr_white_nits;
811
812 hdr_y_nits_avg_th += hdr_y_nits;
813 if (hdr_y_nits > hdr_y_nits_max_th) {
814 hdr_y_nits_max_th = hdr_y_nits;
815 }
816 }
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400817 }
818 }
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800819 std::unique_lock<std::mutex> lock{mutex};
820 hdr_y_nits_avg += hdr_y_nits_avg_th;
821 hdr_y_nits_max = std::max(hdr_y_nits_max, hdr_y_nits_max_th);
822 };
823
824 std::function<void()> generateMap = [uncompressed_yuv_420_image, uncompressed_p010_image,
825 metadata, dest, hdrInvOetf, hdrGamutConversionFn,
826 luminanceFn, hdr_white_nits, &jobQueue]() -> void {
827 size_t rowStart, rowEnd;
828 while (jobQueue.dequeueJob(rowStart, rowEnd)) {
829 for (size_t y = rowStart; y < rowEnd; ++y) {
830 for (size_t x = 0; x < dest->width; ++x) {
831 Color sdr_yuv_gamma =
832 sampleYuv420(uncompressed_yuv_420_image, kMapDimensionScaleFactor, x, y);
833 Color sdr_rgb_gamma = srgbYuvToRgb(sdr_yuv_gamma);
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800834#if USE_SRGB_INVOETF_LUT
835 Color sdr_rgb = srgbInvOetfLUT(sdr_rgb_gamma);
836#else
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800837 Color sdr_rgb = srgbInvOetf(sdr_rgb_gamma);
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800838#endif
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800839 float sdr_y_nits = luminanceFn(sdr_rgb) * kSdrWhiteNits;
840
841 Color hdr_yuv_gamma = sampleP010(uncompressed_p010_image, kMapDimensionScaleFactor, x, y);
842 Color hdr_rgb_gamma = bt2100YuvToRgb(hdr_yuv_gamma);
843 Color hdr_rgb = hdrInvOetf(hdr_rgb_gamma);
844 hdr_rgb = hdrGamutConversionFn(hdr_rgb);
845 float hdr_y_nits = luminanceFn(hdr_rgb) * hdr_white_nits;
846
847 size_t pixel_idx = x + y * dest->width;
848 reinterpret_cast<uint8_t*>(dest->data)[pixel_idx] =
849 encodeRecovery(sdr_y_nits, hdr_y_nits, metadata->rangeScalingFactor);
850 }
851 }
852 }
853 };
854
855 std::vector<std::thread> workers;
856 for (int th = 0; th < threads - 1; th++) {
857 workers.push_back(std::thread(computeMetadata));
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400858 }
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800859
860 // compute metadata
861 for (size_t rowStart = 0; rowStart < image_height;) {
862 size_t rowEnd = std::min(rowStart + rowStep, image_height);
863 jobQueue.enqueueJob(rowStart, rowEnd);
864 rowStart = rowEnd;
865 }
866 jobQueue.markQueueForEnd();
867 computeMetadata();
868 std::for_each(workers.begin(), workers.end(), [](std::thread& t) { t.join(); });
869 workers.clear();
Nick Deakin6bd90432022-11-20 16:26:37 -0500870 hdr_y_nits_avg /= image_width * image_height;
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400871
Nick Deakin6bd90432022-11-20 16:26:37 -0500872 metadata->rangeScalingFactor = hdr_y_nits_max / kSdrWhiteNits;
873 if (metadata->transferFunction == JPEGR_TF_PQ) {
874 metadata->hdr10Metadata.maxFALL = hdr_y_nits_avg;
875 metadata->hdr10Metadata.maxCLL = hdr_y_nits_max;
876 }
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400877
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800878 // generate map
879 jobQueue.reset();
880 for (int th = 0; th < threads - 1; th++) {
881 workers.push_back(std::thread(generateMap));
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400882 }
883
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800884 rowStep = (threads == 1 ? image_height : kJobSzInRows) / kMapDimensionScaleFactor;
885 for (size_t rowStart = 0; rowStart < map_height;) {
886 size_t rowEnd = std::min(rowStart + rowStep, map_height);
887 jobQueue.enqueueJob(rowStart, rowEnd);
888 rowStart = rowEnd;
889 }
890 jobQueue.markQueueForEnd();
891 generateMap();
892 std::for_each(workers.begin(), workers.end(), [](std::thread& t) { t.join(); });
893
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400894 map_data.release();
Dichen Zhang6947d532022-10-22 02:16:21 +0000895 return NO_ERROR;
Dichen Zhang596a7562022-10-12 14:57:05 -0700896}
897
Dichen Zhang6947d532022-10-22 02:16:21 +0000898status_t RecoveryMap::applyRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image,
899 jr_uncompressed_ptr uncompressed_recovery_map,
Nick Deakin6bd90432022-11-20 16:26:37 -0500900 jr_metadata_ptr metadata,
Dichen Zhang6947d532022-10-22 02:16:21 +0000901 jr_uncompressed_ptr dest) {
Dichen Zhang596a7562022-10-12 14:57:05 -0700902 if (uncompressed_yuv_420_image == nullptr
903 || uncompressed_recovery_map == nullptr
Nick Deakin6bd90432022-11-20 16:26:37 -0500904 || metadata == nullptr
Dichen Zhang596a7562022-10-12 14:57:05 -0700905 || dest == nullptr) {
Dichen Zhang80b72482022-11-02 01:55:35 +0000906 return ERROR_JPEGR_INVALID_NULL_PTR;
Dichen Zhang596a7562022-10-12 14:57:05 -0700907 }
908
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800909 dest->width = uncompressed_yuv_420_image->width;
910 dest->height = uncompressed_yuv_420_image->height;
Ram Mohanfe723d62022-12-15 00:59:11 +0530911 ShepardsIDW idwTable(kMapDimensionScaleFactor);
912
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800913 JobQueue jobQueue;
914 std::function<void()> applyRecMap = [uncompressed_yuv_420_image, uncompressed_recovery_map,
915 metadata, dest, &jobQueue, &idwTable]() -> void {
916 const float hdr_ratio = metadata->rangeScalingFactor;
917 size_t width = uncompressed_yuv_420_image->width;
918 size_t height = uncompressed_yuv_420_image->height;
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400919
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800920 ColorTransformFn hdrOetf = nullptr;
921 switch (metadata->transferFunction) {
922 case JPEGR_TF_LINEAR:
923 hdrOetf = identityConversion;
924 break;
925 case JPEGR_TF_HLG:
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800926#if USE_HLG_OETF_LUT
927 hdrOetf = hlgOetfLUT;
928#else
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800929 hdrOetf = hlgOetf;
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800930#endif
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800931 break;
932 case JPEGR_TF_PQ:
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800933#if USE_PQ_OETF_LUT
934 hdrOetf = pqOetfLUT;
935#else
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800936 hdrOetf = pqOetf;
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800937#endif
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800938 break;
939 case JPEGR_TF_UNSPECIFIED:
940 // Should be impossible to hit after input validation.
941 hdrOetf = identityConversion;
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400942 }
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800943
944 size_t rowStart, rowEnd;
945 while (jobQueue.dequeueJob(rowStart, rowEnd)) {
946 for (size_t y = rowStart; y < rowEnd; ++y) {
947 for (size_t x = 0; x < width; ++x) {
948 Color yuv_gamma_sdr = getYuv420Pixel(uncompressed_yuv_420_image, x, y);
949 Color rgb_gamma_sdr = srgbYuvToRgb(yuv_gamma_sdr);
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800950#if USE_SRGB_INVOETF_LUT
951 Color rgb_sdr = srgbInvOetfLUT(rgb_gamma_sdr);
952#else
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800953 Color rgb_sdr = srgbInvOetf(rgb_gamma_sdr);
Harish Mahendrakar555a06b2022-12-14 09:37:27 -0800954#endif
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800955 float recovery;
956 // TODO: determine map scaling factor based on actual map dims
957 size_t map_scale_factor = kMapDimensionScaleFactor;
958 // TODO: If map_scale_factor is guaranteed to be an integer, then remove the following.
959 // Currently map_scale_factor is of type size_t, but it could be changed to a float
960 // later.
961 if (map_scale_factor != floorf(map_scale_factor)) {
962 recovery = sampleMap(uncompressed_recovery_map, map_scale_factor, x, y);
963 } else {
964 recovery = sampleMap(uncompressed_recovery_map, map_scale_factor, x, y,
965 idwTable);
966 }
967 Color rgb_hdr = applyRecovery(rgb_sdr, recovery, hdr_ratio);
968
969 Color rgb_gamma_hdr = hdrOetf(rgb_hdr / metadata->rangeScalingFactor);
970 uint32_t rgba1010102 = colorToRgba1010102(rgb_gamma_hdr);
971
972 size_t pixel_idx = x + y * width;
973 reinterpret_cast<uint32_t*>(dest->data)[pixel_idx] = rgba1010102;
974 }
975 }
976 }
977 };
978
979 const int threads = std::clamp(GetCPUCoreCount(), 1, 4);
980 std::vector<std::thread> workers;
981 for (int th = 0; th < threads - 1; th++) {
982 workers.push_back(std::thread(applyRecMap));
Nick Deakinf6bca5a2022-11-04 10:43:43 -0400983 }
Harish Mahendrakar72b6f302022-12-16 10:39:15 -0800984 const int rowStep = threads == 1 ? uncompressed_yuv_420_image->height : kJobSzInRows;
985 for (int rowStart = 0; rowStart < uncompressed_yuv_420_image->height;) {
986 int rowEnd = std::min(rowStart + rowStep, uncompressed_yuv_420_image->height);
987 jobQueue.enqueueJob(rowStart, rowEnd);
988 rowStart = rowEnd;
989 }
990 jobQueue.markQueueForEnd();
991 applyRecMap();
992 std::for_each(workers.begin(), workers.end(), [](std::thread& t) { t.join(); });
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +0000993 return NO_ERROR;
994}
995
996status_t RecoveryMap::extractPrimaryImageAndRecoveryMap(jr_compressed_ptr compressed_jpegr_image,
997 jr_compressed_ptr primary_image,
998 jr_compressed_ptr recovery_map) {
999 if (compressed_jpegr_image == nullptr) {
1000 return ERROR_JPEGR_INVALID_NULL_PTR;
1001 }
1002
1003 MessageHandler msg_handler;
1004 std::shared_ptr<DataSegment> seg =
1005 DataSegment::Create(DataRange(0, compressed_jpegr_image->length),
1006 static_cast<const uint8_t*>(compressed_jpegr_image->data),
1007 DataSegment::BufferDispositionPolicy::kDontDelete);
1008 DataSegmentDataSource data_source(seg);
1009 JpegInfoBuilder jpeg_info_builder;
1010 jpeg_info_builder.SetImageLimit(2);
1011 JpegScanner jpeg_scanner(&msg_handler);
1012 jpeg_scanner.Run(&data_source, &jpeg_info_builder);
1013 data_source.Reset();
1014
1015 if (jpeg_scanner.HasError()) {
1016 return ERROR_JPEGR_INVALID_INPUT_TYPE;
1017 }
1018
1019 const auto& jpeg_info = jpeg_info_builder.GetInfo();
1020 const auto& image_ranges = jpeg_info.GetImageRanges();
1021 if (image_ranges.empty()) {
1022 return ERROR_JPEGR_INVALID_INPUT_TYPE;
1023 }
1024
1025 if (image_ranges.size() != 2) {
1026 // Must be 2 JPEG Images
1027 return ERROR_JPEGR_INVALID_INPUT_TYPE;
1028 }
1029
1030 if (primary_image != nullptr) {
1031 primary_image->data = static_cast<uint8_t*>(compressed_jpegr_image->data) +
1032 image_ranges[0].GetBegin();
1033 primary_image->length = image_ranges[0].GetLength();
1034 }
1035
1036 if (recovery_map != nullptr) {
1037 recovery_map->data = static_cast<uint8_t*>(compressed_jpegr_image->data) +
1038 image_ranges[1].GetBegin();
1039 recovery_map->length = image_ranges[1].GetLength();
1040 }
Nick Deakinf6bca5a2022-11-04 10:43:43 -04001041
Dichen Zhang6947d532022-10-22 02:16:21 +00001042 return NO_ERROR;
Dichen Zhang596a7562022-10-12 14:57:05 -07001043}
1044
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +00001045
Nick Deakinf6bca5a2022-11-04 10:43:43 -04001046status_t RecoveryMap::extractRecoveryMap(jr_compressed_ptr compressed_jpegr_image,
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +00001047 jr_compressed_ptr dest) {
1048 if (compressed_jpegr_image == nullptr || dest == nullptr) {
Dichen Zhang80b72482022-11-02 01:55:35 +00001049 return ERROR_JPEGR_INVALID_NULL_PTR;
Dichen Zhang85b37562022-10-11 11:08:28 -07001050 }
1051
Fyodor Kyslov1dcc4422022-11-16 01:40:53 +00001052 return extractPrimaryImageAndRecoveryMap(compressed_jpegr_image, nullptr, dest);
Dichen Zhang85b37562022-10-11 11:08:28 -07001053}
1054
Dichen Zhangd18bc302022-12-16 20:55:24 +00001055// JPEG/R structure:
1056// SOI (ff d8)
1057// APP1 (ff e1)
1058// 2 bytes of length (2 + length of exif package)
1059// EXIF package (this includes the first two bytes representing the package length)
1060// APP1 (ff e1)
1061// 2 bytes of length (2 + 29 + length of xmp package)
1062// name space ("http://ns.adobe.com/xap/1.0/\0")
1063// xmp
1064// primary image (without the first two bytes (SOI) and without EXIF, may have other packages)
1065// secondary image (the recovery map)
1066//
1067// Metadata versions we are using:
1068// ECMA TR-98 for JFIF marker
1069// Exif 2.2 spec for EXIF marker
1070// Adobe XMP spec part 3 for XMP marker
1071// ICC v4.3 spec for ICC
Nick Deakinf6bca5a2022-11-04 10:43:43 -04001072status_t RecoveryMap::appendRecoveryMap(jr_compressed_ptr compressed_jpeg_image,
1073 jr_compressed_ptr compressed_recovery_map,
Dichen Zhangd18bc302022-12-16 20:55:24 +00001074 jr_exif_ptr exif,
Nick Deakin6bd90432022-11-20 16:26:37 -05001075 jr_metadata_ptr metadata,
Nick Deakinf6bca5a2022-11-04 10:43:43 -04001076 jr_compressed_ptr dest) {
Dichen Zhang6947d532022-10-22 02:16:21 +00001077 if (compressed_jpeg_image == nullptr
1078 || compressed_recovery_map == nullptr
Dichen Zhangd18bc302022-12-16 20:55:24 +00001079 || exif == nullptr
Nick Deakin6bd90432022-11-20 16:26:37 -05001080 || metadata == nullptr
Dichen Zhang6947d532022-10-22 02:16:21 +00001081 || dest == nullptr) {
Dichen Zhang80b72482022-11-02 01:55:35 +00001082 return ERROR_JPEGR_INVALID_NULL_PTR;
Dichen Zhang85b37562022-10-11 11:08:28 -07001083 }
1084
Dichen Zhanga8766262022-11-07 23:48:24 +00001085 int pos = 0;
1086
Dichen Zhangd18bc302022-12-16 20:55:24 +00001087 // Write SOI
Dichen Zhanga8766262022-11-07 23:48:24 +00001088 JPEGR_CHECK(Write(dest, &photos_editing_formats::image_io::JpegMarker::kStart, 1, pos));
1089 JPEGR_CHECK(Write(dest, &photos_editing_formats::image_io::JpegMarker::kSOI, 1, pos));
Dichen Zhangd18bc302022-12-16 20:55:24 +00001090
1091 // Write EXIF
1092 {
1093 const int length = 2 + exif->length;
1094 const uint8_t lengthH = ((length >> 8) & 0xff);
1095 const uint8_t lengthL = (length & 0xff);
1096 JPEGR_CHECK(Write(dest, &photos_editing_formats::image_io::JpegMarker::kStart, 1, pos));
1097 JPEGR_CHECK(Write(dest, &photos_editing_formats::image_io::JpegMarker::kAPP1, 1, pos));
1098 JPEGR_CHECK(Write(dest, &lengthH, 1, pos));
1099 JPEGR_CHECK(Write(dest, &lengthL, 1, pos));
1100 JPEGR_CHECK(Write(dest, exif->data, exif->length, pos));
1101 }
1102
1103 // Prepare and write XMP
1104 {
1105 const string xmp = generateXmp(compressed_recovery_map->length, *metadata);
1106 const string nameSpace = "http://ns.adobe.com/xap/1.0/\0";
1107 const int nameSpaceLength = nameSpace.size() + 1; // need to count the null terminator
1108 // 2 bytes: representing the length of the package
1109 // 29 bytes: length of name space "http://ns.adobe.com/xap/1.0/\0",
1110 // x bytes: length of xmp packet
Dichen Zhang25df9c82023-01-03 17:04:10 -08001111 const int length = 2 + nameSpaceLength + xmp.size();
Dichen Zhangd18bc302022-12-16 20:55:24 +00001112 const uint8_t lengthH = ((length >> 8) & 0xff);
1113 const uint8_t lengthL = (length & 0xff);
1114 JPEGR_CHECK(Write(dest, &photos_editing_formats::image_io::JpegMarker::kStart, 1, pos));
1115 JPEGR_CHECK(Write(dest, &photos_editing_formats::image_io::JpegMarker::kAPP1, 1, pos));
1116 JPEGR_CHECK(Write(dest, &lengthH, 1, pos));
1117 JPEGR_CHECK(Write(dest, &lengthL, 1, pos));
1118 JPEGR_CHECK(Write(dest, (void*)nameSpace.c_str(), nameSpaceLength, pos));
1119 JPEGR_CHECK(Write(dest, (void*)xmp.c_str(), xmp.size(), pos));
1120 }
1121
1122 // Write primary image
Dichen Zhanga8766262022-11-07 23:48:24 +00001123 JPEGR_CHECK(Write(dest,
1124 (uint8_t*)compressed_jpeg_image->data + 2, compressed_jpeg_image->length - 2, pos));
Dichen Zhangd18bc302022-12-16 20:55:24 +00001125
1126 // Write secondary image
Dichen Zhanga8766262022-11-07 23:48:24 +00001127 JPEGR_CHECK(Write(dest, compressed_recovery_map->data, compressed_recovery_map->length, pos));
Dichen Zhangd18bc302022-12-16 20:55:24 +00001128
1129 // Set back length
Dichen Zhanga8766262022-11-07 23:48:24 +00001130 dest->length = pos;
1131
Dichen Zhangd18bc302022-12-16 20:55:24 +00001132 // Done!
Dichen Zhang6947d532022-10-22 02:16:21 +00001133 return NO_ERROR;
Dichen Zhang85b37562022-10-11 11:08:28 -07001134}
1135
Dichen Zhangc3437ca2023-01-04 14:00:08 -08001136status_t RecoveryMap::toneMap(jr_uncompressed_ptr src,
Dichen Zhang636f5242022-12-07 20:25:44 +00001137 jr_uncompressed_ptr dest) {
Dichen Zhangc3437ca2023-01-04 14:00:08 -08001138 if (src == nullptr || dest == nullptr) {
Dichen Zhang636f5242022-12-07 20:25:44 +00001139 return ERROR_JPEGR_INVALID_NULL_PTR;
1140 }
1141
Dichen Zhangc3437ca2023-01-04 14:00:08 -08001142 dest->width = src->width;
1143 dest->height = src->height;
Dichen Zhang636f5242022-12-07 20:25:44 +00001144
Dichen Zhangc3437ca2023-01-04 14:00:08 -08001145 size_t pixel_count = src->width * src->height;
1146 for (size_t y = 0; y < src->height; ++y) {
1147 for (size_t x = 0; x < src->width; ++x) {
1148 size_t pixel_y_idx = x + y * src->width;
1149 size_t pixel_uv_idx = x / 2 + (y / 2) * (src->width / 2);
1150
1151 uint16_t y_uint = reinterpret_cast<uint16_t*>(src->data)[pixel_y_idx]
1152 >> 6;
1153 uint16_t u_uint = reinterpret_cast<uint16_t*>(src->data)[pixel_count + pixel_uv_idx * 2]
1154 >> 6;
1155 uint16_t v_uint = reinterpret_cast<uint16_t*>(src->data)[pixel_count + pixel_uv_idx * 2 + 1]
1156 >> 6;
1157
1158 uint8_t* y = &reinterpret_cast<uint8_t*>(dest->data)[pixel_y_idx];
1159 uint8_t* u = &reinterpret_cast<uint8_t*>(dest->data)[pixel_count + pixel_uv_idx];
1160 uint8_t* v = &reinterpret_cast<uint8_t*>(dest->data)[pixel_count * 5 / 4 + pixel_uv_idx];
1161
1162 *y = static_cast<uint8_t>((y_uint >> 2) & 0xff);
1163 *u = static_cast<uint8_t>((u_uint >> 2) & 0xff);
1164 *v = static_cast<uint8_t>((v_uint >> 2) & 0xff);
1165 }
1166 }
1167
1168 dest->colorGamut = src->colorGamut;
Dichen Zhang636f5242022-12-07 20:25:44 +00001169
1170 return NO_ERROR;
1171}
1172
Dichen Zhang85b37562022-10-11 11:08:28 -07001173} // namespace android::recoverymap