Camera: Extend HEIC output support
- Enable HEIC composite streams on devices with SW
HEVC codecs and no dedicated HEIC stream combination
support.
- Switch the default HEVC YUV420_888 input format to P010.
- Tonemap the P010 camera output to an SDR base image.
- Generate an HDR gainmap.
- Write the the HDR gainmap metadata in the final HEIF
container.
- Add a separate gainmap HEVC encoding pipeline
- Stitch final HEIC including the gainmap and metadata
according to ISO/IEC 23008-12:2024
- Introduce public APIs
- Wire up public APIs with the Heic composite functionality
TODO:
- Enable support for native/Hal HEIF UltraHDR
- Code cleanup and polish
Flag: com.android.internal.camera.flags.camera_heif_gainmap
Bug: 362608343
Test: test -c -d cts/tests/camera/src/android/hardware/camera2/cts/ImageReaderTest.java#testHeicUltraHdr
Change-Id: I09f0d55a5a11699871801583be0bce5c36cacc7a
diff --git a/services/camera/libcameraservice/common/CameraDeviceBase.h b/services/camera/libcameraservice/common/CameraDeviceBase.h
index f5e960b..e17d700 100644
--- a/services/camera/libcameraservice/common/CameraDeviceBase.h
+++ b/services/camera/libcameraservice/common/CameraDeviceBase.h
@@ -44,6 +44,10 @@
namespace camera3 {
+// TODO: Remove this once the GFX native dataspace
+// dependencies are available
+enum { HEIC_ULTRAHDR, ADATASPACE_HEIF_ULTRAHDR = 0x1006 };
+
typedef enum camera_stream_configuration_mode {
CAMERA_STREAM_CONFIGURATION_NORMAL_MODE = 0,
CAMERA_STREAM_CONFIGURATION_CONSTRAINED_HIGH_SPEED_MODE = 1,
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp
index 6394ec1..2d58652 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.cpp
+++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp
@@ -14,6 +14,7 @@
* limitations under the License.
*/
+#include "system/camera_metadata.h"
#include "system/graphics-base-v1.0.h"
#include "system/graphics-base-v1.1.h"
#define LOG_TAG "CameraProviderManager"
@@ -76,6 +77,10 @@
const float CameraProviderManager::kDepthARTolerance = .1f;
const bool CameraProviderManager::kFrameworkJpegRDisabled =
property_get_bool("ro.camera.disableJpegR", false);
+const bool CameraProviderManager::kFrameworkHeicUltraHDRDisabled =
+ property_get_bool("ro.camera.disableHeicUltraHDR", false);
+const bool CameraProviderManager::kFrameworkHeicAllowSWCodecs =
+ property_get_bool("ro.camera.enableSWHEVC", false);
CameraProviderManager::HidlServiceInteractionProxyImpl
CameraProviderManager::sHidlServiceInteractionProxy{};
@@ -1246,6 +1251,165 @@
return false;
}
+status_t CameraProviderManager::ProviderInfo::DeviceInfo3::deriveHeicUltraHDRTags(
+ bool maxResolution) {
+ if (!flags::camera_heif_gainmap() || kFrameworkHeicUltraHDRDisabled ||
+ mCompositeHeicUltraHDRDisabled ||
+ !camera3::HeicCompositeStream::isInMemoryTempFileSupported()) {
+ return OK;
+ }
+
+ const int32_t scalerSizesTag =
+ SessionConfigurationUtils::getAppropriateModeTag(
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, maxResolution);
+ const int32_t scalerMinFrameDurationsTag = SessionConfigurationUtils::getAppropriateModeTag(
+ ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS, maxResolution);
+ const int32_t scalerStallDurationsTag =
+ SessionConfigurationUtils::getAppropriateModeTag(
+ ANDROID_SCALER_AVAILABLE_STALL_DURATIONS, maxResolution);
+
+ const int32_t heicUltraHDRSizesTag =
+ SessionConfigurationUtils::getAppropriateModeTag(
+ ANDROID_HEIC_AVAILABLE_HEIC_ULTRA_HDR_STREAM_CONFIGURATIONS, maxResolution);
+ const int32_t heicUltraHDRStallDurationsTag =
+ SessionConfigurationUtils::getAppropriateModeTag(
+ ANDROID_HEIC_AVAILABLE_HEIC_ULTRA_HDR_STALL_DURATIONS, maxResolution);
+ const int32_t heicUltraHDRFrameDurationsTag =
+ SessionConfigurationUtils::getAppropriateModeTag(
+ ANDROID_HEIC_AVAILABLE_HEIC_ULTRA_HDR_MIN_FRAME_DURATIONS, maxResolution);
+
+ auto& c = mCameraCharacteristics;
+ std::vector<std::tuple<size_t, size_t>> supportedP010Sizes, filteredSizes;
+ auto capabilities = c.find(ANDROID_REQUEST_AVAILABLE_CAPABILITIES);
+ if (capabilities.count == 0) {
+ ALOGE("%s: Supported camera capabilities is empty!", __FUNCTION__);
+ return BAD_VALUE;
+ }
+
+ auto end = capabilities.data.u8 + capabilities.count;
+ bool isTenBitOutputSupported = std::find(capabilities.data.u8, end,
+ ANDROID_REQUEST_AVAILABLE_CAPABILITIES_DYNAMIC_RANGE_TEN_BIT) != end;
+ if (!isTenBitOutputSupported) {
+ // No 10-bit support, nothing more to do.
+ return OK;
+ }
+
+ getSupportedSizes(c, scalerSizesTag,
+ static_cast<android_pixel_format_t>(HAL_PIXEL_FORMAT_YCBCR_P010), &supportedP010Sizes);
+ auto it = supportedP010Sizes.begin();
+ if (supportedP010Sizes.empty()) {
+ // Nothing to do in this case.
+ return OK;
+ }
+
+ std::vector<int32_t> heicUltraHDREntries;
+ int64_t stall = 0;
+ bool useHeic = false;
+ bool useGrid = false;
+ for (const auto& it : supportedP010Sizes) {
+ int32_t width = std::get<0>(it);
+ int32_t height = std::get<1>(it);
+ int32_t gainmapWidth = std::get<0>(it) / HeicCompositeStream::kGainmapScale;
+ int32_t gainmapHeight = std::get<1>(it) / HeicCompositeStream::kGainmapScale;
+ if (camera3::HeicCompositeStream::isSizeSupportedByHeifEncoder(width, height,
+ &useHeic, &useGrid, &stall, nullptr /*hevcName*/, kFrameworkHeicAllowSWCodecs) &&
+ camera3::HeicCompositeStream::isSizeSupportedByHeifEncoder(gainmapWidth,
+ gainmapHeight, &useHeic, &useGrid, &stall, nullptr /*hevcName*/,
+ kFrameworkHeicAllowSWCodecs) ) {
+ int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(it)),
+ static_cast<int32_t> (std::get<1>(it)),
+ ANDROID_HEIC_AVAILABLE_HEIC_ULTRA_HDR_STREAM_CONFIGURATIONS_OUTPUT };
+ heicUltraHDREntries.insert(heicUltraHDREntries.end(), entry, entry + 4);
+ filteredSizes.push_back(it);
+ }
+ }
+
+ std::vector<int64_t> heicUltraHDRMinDurations, heicUltraHDRStallDurations;
+ auto ret = deriveBlobDurationEntries(c, maxResolution, filteredSizes,
+ &heicUltraHDRStallDurations, &heicUltraHDRMinDurations);
+ if (ret != OK) {
+ return ret;
+ }
+
+ return insertStreamConfigTags(heicUltraHDRSizesTag, heicUltraHDRFrameDurationsTag,
+ heicUltraHDRStallDurationsTag, heicUltraHDREntries,
+ heicUltraHDRMinDurations, heicUltraHDRStallDurations, &c);
+}
+
+status_t CameraProviderManager::ProviderInfo::DeviceInfo3::insertStreamConfigTags(
+ int32_t sizeTag, int32_t minFrameDurationTag, int32_t stallDurationTag,
+ const std::vector<int32_t>& sizeEntries,
+ const std::vector<int64_t>& minFrameDurationEntries,
+ const std::vector<int64_t>& stallDurationEntries, CameraMetadata* c /*out*/) {
+ std::vector<int32_t> supportedChTags;
+ auto chTags = c->find(ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS);
+ if (chTags.count == 0) {
+ ALOGE("%s: No supported camera characteristics keys!", __FUNCTION__);
+ return BAD_VALUE;
+ }
+ supportedChTags.reserve(chTags.count + 3);
+ supportedChTags.insert(supportedChTags.end(), chTags.data.i32, chTags.data.i32 + chTags.count);
+ supportedChTags.push_back(sizeTag);
+ supportedChTags.push_back(minFrameDurationTag);
+ supportedChTags.push_back(stallDurationTag);
+ c->update(sizeTag, sizeEntries.data(), sizeEntries.size());
+ c->update(minFrameDurationTag, minFrameDurationEntries.data(), minFrameDurationEntries.size());
+ c->update(stallDurationTag, stallDurationEntries.data(), stallDurationEntries.size());
+ c->update(ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS, supportedChTags.data(),
+ supportedChTags.size());
+
+ return OK;
+}
+
+status_t CameraProviderManager::ProviderInfo::DeviceInfo3::deriveBlobDurationEntries(
+ const CameraMetadata& c, bool maxResolution,
+ const std::vector<std::tuple<size_t, size_t>>& filteredSizes,
+ std::vector<int64_t>* filteredStallDurations /*out*/,
+ std::vector<int64_t>* filteredMinDurations /*out*/) {
+ std::vector<int64_t> blobMinDurations, blobStallDurations;
+ const int32_t scalerMinFrameDurationsTag = SessionConfigurationUtils::getAppropriateModeTag(
+ ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS, maxResolution);
+ const int32_t scalerStallDurationsTag = SessionConfigurationUtils::getAppropriateModeTag(
+ ANDROID_SCALER_AVAILABLE_STALL_DURATIONS, maxResolution);
+ // We use the jpeg stall and min frame durations to approximate the respective Heic UltraHDR
+ // durations.
+ getSupportedDurations(c, scalerMinFrameDurationsTag, HAL_PIXEL_FORMAT_BLOB, filteredSizes,
+ &blobMinDurations);
+ getSupportedDurations(c, scalerStallDurationsTag, HAL_PIXEL_FORMAT_BLOB, filteredSizes,
+ &blobStallDurations);
+ if (blobStallDurations.empty() || blobMinDurations.empty() ||
+ filteredSizes.size() != blobMinDurations.size() ||
+ blobMinDurations.size() != blobStallDurations.size()) {
+ ALOGE("%s: Unexpected number of available blob durations! %zu vs. %zu with "
+ "filteredSizes size: %zu",
+ __FUNCTION__, blobMinDurations.size(), blobStallDurations.size(),
+ filteredSizes.size());
+ return BAD_VALUE;
+ }
+
+ auto itDuration = blobMinDurations.begin();
+ auto itSize = filteredSizes.begin();
+ while (itDuration != blobMinDurations.end()) {
+ int64_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t>(std::get<0>(*itSize)),
+ static_cast<int32_t>(std::get<1>(*itSize)), *itDuration};
+ filteredMinDurations->insert(filteredMinDurations->end(), entry, entry + 4);
+ itDuration++;
+ itSize++;
+ }
+
+ itDuration = blobStallDurations.begin();
+ itSize = filteredSizes.begin();
+ while (itDuration != blobStallDurations.end()) {
+ int64_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t>(std::get<0>(*itSize)),
+ static_cast<int32_t>(std::get<1>(*itSize)), *itDuration};
+ filteredStallDurations->insert(filteredStallDurations->end(), entry, entry + 4);
+ itDuration++;
+ itSize++;
+ }
+
+ return OK;
+}
+
status_t CameraProviderManager::ProviderInfo::DeviceInfo3::deriveJpegRTags(bool maxResolution) {
if (kFrameworkJpegRDisabled || mCompositeJpegRDisabled) {
return OK;
@@ -1271,13 +1435,6 @@
ANDROID_JPEGR_AVAILABLE_JPEG_R_MIN_FRAME_DURATIONS, maxResolution);
auto& c = mCameraCharacteristics;
- std::vector<int32_t> supportedChTags;
- auto chTags = c.find(ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS);
- if (chTags.count == 0) {
- ALOGE("%s: No supported camera characteristics keys!", __FUNCTION__);
- return BAD_VALUE;
- }
-
std::vector<std::tuple<size_t, size_t>> supportedP010Sizes, supportedBlobSizes;
auto capabilities = c.find(ANDROID_REQUEST_AVAILABLE_CAPABILITIES);
if (capabilities.count == 0) {
@@ -1331,54 +1488,19 @@
jpegREntries.insert(jpegREntries.end(), entry, entry + 4);
}
- std::vector<int64_t> blobMinDurations, blobStallDurations;
std::vector<int64_t> jpegRMinDurations, jpegRStallDurations;
-
- // We use the jpeg stall and min frame durations to approximate the respective jpeg/r
- // durations.
- getSupportedDurations(c, scalerMinFrameDurationsTag, HAL_PIXEL_FORMAT_BLOB,
- supportedP010Sizes, &blobMinDurations);
- getSupportedDurations(c, scalerStallDurationsTag, HAL_PIXEL_FORMAT_BLOB,
- supportedP010Sizes, &blobStallDurations);
- if (blobStallDurations.empty() || blobMinDurations.empty() ||
- supportedP010Sizes.size() != blobMinDurations.size() ||
- blobMinDurations.size() != blobStallDurations.size()) {
- ALOGE("%s: Unexpected number of available blob durations! %zu vs. %zu with "
- "supportedP010Sizes size: %zu", __FUNCTION__, blobMinDurations.size(),
- blobStallDurations.size(), supportedP010Sizes.size());
- return BAD_VALUE;
+ auto ret = deriveBlobDurationEntries(c, maxResolution, supportedP010Sizes, &jpegRStallDurations,
+ &jpegRMinDurations);
+ if (ret != OK) {
+ return ret;
}
- auto itDuration = blobMinDurations.begin();
- auto itSize = supportedP010Sizes.begin();
- while (itDuration != blobMinDurations.end()) {
- int64_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(*itSize)),
- static_cast<int32_t> (std::get<1>(*itSize)), *itDuration};
- jpegRMinDurations.insert(jpegRMinDurations.end(), entry, entry + 4);
- itDuration++; itSize++;
+ ret = insertStreamConfigTags(jpegRSizesTag, jpegRMinFrameDurationsTag, jpegRStallDurationsTag,
+ jpegREntries, jpegRMinDurations, jpegRStallDurations, &c);
+ if (ret != OK) {
+ return ret;
}
- itDuration = blobStallDurations.begin();
- itSize = supportedP010Sizes.begin();
- while (itDuration != blobStallDurations.end()) {
- int64_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(*itSize)),
- static_cast<int32_t> (std::get<1>(*itSize)), *itDuration};
- jpegRStallDurations.insert(jpegRStallDurations.end(), entry, entry + 4);
- itDuration++; itSize++;
- }
-
- supportedChTags.reserve(chTags.count + 3);
- supportedChTags.insert(supportedChTags.end(), chTags.data.i32,
- chTags.data.i32 + chTags.count);
- supportedChTags.push_back(jpegRSizesTag);
- supportedChTags.push_back(jpegRMinFrameDurationsTag);
- supportedChTags.push_back(jpegRStallDurationsTag);
- c.update(jpegRSizesTag, jpegREntries.data(), jpegREntries.size());
- c.update(jpegRMinFrameDurationsTag, jpegRMinDurations.data(), jpegRMinDurations.size());
- c.update(jpegRStallDurationsTag, jpegRStallDurations.data(), jpegRStallDurations.size());
- c.update(ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS, supportedChTags.data(),
- supportedChTags.size());
-
auto colorSpaces = c.find(ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP);
if (colorSpaces.count > 0 && !maxResolution) {
bool displayP3Support = false;
@@ -1976,7 +2098,7 @@
bool useGrid = false;
if (camera3::HeicCompositeStream::isSizeSupportedByHeifEncoder(
halStreamConfigs.data.i32[i+1], halStreamConfigs.data.i32[i+2],
- &useHeic, &useGrid, &stall)) {
+ &useHeic, &useGrid, &stall, nullptr /*hevcName*/, kFrameworkHeicAllowSWCodecs)) {
if (useGrid != (format == HAL_PIXEL_FORMAT_YCBCR_420_888)) {
continue;
}
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.h b/services/camera/libcameraservice/common/CameraProviderManager.h
index f0db8bc..e629218 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.h
+++ b/services/camera/libcameraservice/common/CameraProviderManager.h
@@ -470,6 +470,9 @@
static const float kDepthARTolerance;
static const bool kFrameworkJpegRDisabled;
+ static const bool kFrameworkHeicUltraHDRDisabled;
+ static const bool kFrameworkHeicAllowSWCodecs;
+
private:
// All private members, unless otherwise noted, expect mInterfaceMutex to be locked before use
mutable std::mutex mInterfaceMutex;
@@ -629,6 +632,7 @@
bool hasFlashUnit() const { return mHasFlashUnit; }
bool supportNativeZoomRatio() const { return mSupportNativeZoomRatio; }
bool isCompositeJpegRDisabled() const { return mCompositeJpegRDisabled; }
+ bool isCompositeHeicUltraHDRDisabled() const { return mCompositeHeicUltraHDRDisabled; }
virtual status_t setTorchMode(bool enabled) = 0;
virtual status_t turnOnTorchWithStrengthLevel(int32_t torchStrength) = 0;
virtual status_t getTorchStrengthLevel(int32_t *torchStrength) = 0;
@@ -685,14 +689,15 @@
mParentProvider(parentProvider), mTorchStrengthLevel(0),
mTorchMaximumStrengthLevel(0), mTorchDefaultStrengthLevel(0),
mHasFlashUnit(false), mSupportNativeZoomRatio(false),
- mPublicCameraIds(publicCameraIds), mCompositeJpegRDisabled(false) {}
+ mPublicCameraIds(publicCameraIds), mCompositeJpegRDisabled(false),
+ mCompositeHeicUltraHDRDisabled(false) {}
virtual ~DeviceInfo() {}
protected:
bool mHasFlashUnit; // const after constructor
bool mSupportNativeZoomRatio; // const after constructor
const std::vector<std::string>& mPublicCameraIds;
- bool mCompositeJpegRDisabled;
+ bool mCompositeJpegRDisabled, mCompositeHeicUltraHDRDisabled;
};
std::vector<std::unique_ptr<DeviceInfo>> mDevices;
std::unordered_set<std::string> mUniqueCameraIds;
@@ -757,6 +762,18 @@
status_t addDynamicDepthTags(bool maxResolution = false);
status_t deriveHeicTags(bool maxResolution = false);
status_t deriveJpegRTags(bool maxResolution = false);
+ status_t deriveHeicUltraHDRTags(bool maxResolution = false);
+ status_t deriveBlobDurationEntries(
+ const CameraMetadata& c, bool maxResolution,
+ const std::vector<std::tuple<size_t, size_t>>& filteredSizes,
+ std::vector<int64_t>* filteredStallDurations /*out*/,
+ std::vector<int64_t>* filteredMinDurations /*out*/);
+ status_t insertStreamConfigTags(int32_t sizeTag, int32_t minFrameDurationTag,
+ int32_t stallDurationTag,
+ const std::vector<int32_t>& sizeEntries,
+ const std::vector<int64_t>& minFrameDurationEntries,
+ const std::vector<int64_t>& stallDurationEntries,
+ CameraMetadata* c /*out*/);
status_t addRotateCropTags();
status_t addAutoframingTags();
status_t addPreCorrectionActiveArraySize();
diff --git a/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp b/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp
index e1efd90..3d6a23f 100644
--- a/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp
+++ b/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp
@@ -517,6 +517,8 @@
mCompositeJpegRDisabled = mCameraCharacteristics.exists(
ANDROID_JPEGR_AVAILABLE_JPEG_R_STREAM_CONFIGURATIONS);
+ mCompositeHeicUltraHDRDisabled = mCameraCharacteristics.exists(
+ ANDROID_HEIC_AVAILABLE_HEIC_ULTRA_HDR_STREAM_CONFIGURATIONS);
mSystemCameraKind = getSystemCameraKind();
@@ -548,6 +550,12 @@
ALOGE("%s: Unable to derive Jpeg/R tags based on camera and media capabilities: %s (%d)",
__FUNCTION__, strerror(-res), res);
}
+ res = deriveHeicUltraHDRTags();
+ if (OK != res) {
+ ALOGE("%s: Unable to derive Heic UltraHDR tags based on camera and "
+ "media capabilities: %s (%d)",
+ __FUNCTION__, strerror(-res), res);
+ }
using camera3::SessionConfigurationUtils::supportsUltraHighResolutionCapture;
if (supportsUltraHighResolutionCapture(mCameraCharacteristics)) {
status_t status = addDynamicDepthTags(/*maxResolution*/true);
@@ -567,6 +575,12 @@
ALOGE("%s: Unable to derive Jpeg/R tags based on camera and media capabilities for"
"maximum resolution mode: %s (%d)", __FUNCTION__, strerror(-status), status);
}
+ status = deriveHeicUltraHDRTags(/*maxResolution*/true);
+ if (OK != status) {
+ ALOGE("%s: Unable to derive Heic UltraHDR tags based on camera and "
+ "media capabilities: %s (%d)",
+ __FUNCTION__, strerror(-status), status);
+ }
}
res = addRotateCropTags();