Add FMQ support to camera2 SDK for metadata transfer
This CL adds support for CaptureResult metadata to be transferred over
FMQ instead of binder copies.
Bug: 362791857
Flag: com.android.internal.camera.flags.fmq_metadata
Test: GCA
Test: Perfetto profiling shows decreased cameraserver to client
onResultReceived latency
Change-Id: Ia8df1a4cef5008c06dc2ca4fdd319704d5e049ab
Signed-off-by: Jayant Chowdhary <jchowdhary@google.com>
diff --git a/camera/Android.bp b/camera/Android.bp
index 25b5e2c..71c1673 100644
--- a/camera/Android.bp
+++ b/camera/Android.bp
@@ -82,6 +82,8 @@
include_dirs: [
"frameworks/native/aidl/gui",
"frameworks/native/libs/permission/aidl",
+ "hardware/interfaces/common/fmq/aidl",
+ "hardware/interfaces/common/aidl",
],
},
@@ -112,6 +114,8 @@
],
shared_libs: [
+ "android.hardware.common.fmq-V1-cpp",
+ "android.hardware.common-V2-cpp",
"camera_platform_flags_c_lib",
"framework-permission-aidl-cpp",
"lib-platform-compat-native-api",
@@ -136,6 +140,8 @@
],
export_shared_lib_headers: [
"framework-permission-aidl-cpp",
+ "android.hardware.common.fmq-V1-cpp",
+ "android.hardware.common-V2-cpp",
"libcamera_metadata",
"libgui",
"libnativewindow",
@@ -187,6 +193,7 @@
"aidl/android/hardware/camera2/ICameraInjectionCallback.aidl",
"aidl/android/hardware/camera2/ICameraInjectionSession.aidl",
"aidl/android/hardware/camera2/ICameraOfflineSession.aidl",
+ "aidl/android/hardware/camera2/CameraMetadataInfo.aidl",
],
path: "aidl",
}
diff --git a/camera/CaptureResult.cpp b/camera/CaptureResult.cpp
index 9ff2578..254984f 100644
--- a/camera/CaptureResult.cpp
+++ b/camera/CaptureResult.cpp
@@ -98,7 +98,6 @@
status_t res;
mPhysicalCameraId = "";
- mPhysicalCameraMetadata.clear();
String16 physicalCameraId;
if ((res = parcel->readString16(&physicalCameraId)) != OK) {
@@ -107,10 +106,11 @@
}
mPhysicalCameraId = toStdString(physicalCameraId);
- if ((res = mPhysicalCameraMetadata.readFromParcel(parcel)) != OK) {
+ if ((res = mCameraMetadataInfo.readFromParcel(parcel)) != OK) {
ALOGE("%s: Failed to read metadata from parcel: %d", __FUNCTION__, res);
return res;
}
+
return OK;
}
@@ -121,11 +121,13 @@
__FUNCTION__, res);
return res;
}
- if ((res = mPhysicalCameraMetadata.writeToParcel(parcel)) != OK) {
+
+ if ((res = mCameraMetadataInfo.writeToParcel(parcel)) != OK) {
ALOGE("%s: Failed to write physical camera metadata to parcel: %d",
__FUNCTION__, res);
return res;
}
+
return OK;
}
@@ -178,20 +180,12 @@
}
for (int32_t i = 0; i < physicalMetadataCount; i++) {
- String16 cameraId;
- if ((res = parcel->readString16(&cameraId)) != OK) {
- ALOGE("%s: Failed to read camera id: %d", __FUNCTION__, res);
+ PhysicalCaptureResultInfo result;
+ if ((res = result.readFromParcel(parcel)) != OK) {
+ ALOGE("%s: Failed to read physical result from parcel: %d", __FUNCTION__, res);
return res;
}
-
- CameraMetadata physicalMetadata;
- if ((res = physicalMetadata.readFromParcel(parcel)) != OK) {
- ALOGE("%s: Failed to read metadata from parcel: %d", __FUNCTION__, res);
- return res;
- }
-
- mPhysicalMetadatas.emplace(mPhysicalMetadatas.end(), toStdString(cameraId),
- physicalMetadata);
+ mPhysicalMetadatas.emplace(mPhysicalMetadatas.end(), result);
}
ALOGV("%s: Read physical metadata from parcel", __FUNCTION__);
@@ -232,13 +226,8 @@
return BAD_VALUE;
}
for (const auto& physicalMetadata : mPhysicalMetadatas) {
- if ((res = parcel->writeString16(toString16(physicalMetadata.mPhysicalCameraId))) != OK) {
- ALOGE("%s: Failed to write physical camera ID to parcel: %d",
- __FUNCTION__, res);
- return res;
- }
- if ((res = physicalMetadata.mPhysicalCameraMetadata.writeToParcel(parcel)) != OK) {
- ALOGE("%s: Failed to write physical camera metadata to parcel: %d",
+ if ((res = physicalMetadata.writeToParcel(parcel)) != OK) {
+ ALOGE("%s: Failed to write physicalMetadata to parcel: %d",
__FUNCTION__, res);
return res;
}
diff --git a/camera/aidl/android/hardware/camera2/CameraMetadataInfo.aidl b/camera/aidl/android/hardware/camera2/CameraMetadataInfo.aidl
new file mode 100644
index 0000000..74c207e
--- /dev/null
+++ b/camera/aidl/android/hardware/camera2/CameraMetadataInfo.aidl
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.hardware.camera2;
+
+import android.hardware.camera2.impl.CameraMetadataNative;
+
+/** @hide */
+union CameraMetadataInfo {
+ long fmqSize;
+ CameraMetadataNative metadata;
+}
\ No newline at end of file
diff --git a/camera/aidl/android/hardware/camera2/ICameraDeviceCallbacks.aidl b/camera/aidl/android/hardware/camera2/ICameraDeviceCallbacks.aidl
index 49e9920..68e6354 100644
--- a/camera/aidl/android/hardware/camera2/ICameraDeviceCallbacks.aidl
+++ b/camera/aidl/android/hardware/camera2/ICameraDeviceCallbacks.aidl
@@ -16,6 +16,7 @@
package android.hardware.camera2;
+import android.hardware.camera2.CameraMetadataInfo;
import android.hardware.camera2.impl.CameraMetadataNative;
import android.hardware.camera2.impl.CaptureResultExtras;
import android.hardware.camera2.impl.PhysicalCaptureResultInfo;
@@ -36,7 +37,7 @@
oneway void onDeviceError(int errorCode, in CaptureResultExtras resultExtras);
oneway void onDeviceIdle();
oneway void onCaptureStarted(in CaptureResultExtras resultExtras, long timestamp);
- oneway void onResultReceived(in CameraMetadataNative result,
+ oneway void onResultReceived(in CameraMetadataInfo resultInfo,
in CaptureResultExtras resultExtras,
in PhysicalCaptureResultInfo[] physicalCaptureResultInfos);
oneway void onPrepared(int streamId);
diff --git a/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl b/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl
index c1da126..a9191eb 100644
--- a/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl
+++ b/camera/aidl/android/hardware/camera2/ICameraDeviceUser.aidl
@@ -23,6 +23,8 @@
import android.hardware.camera2.params.OutputConfiguration;
import android.hardware.camera2.params.SessionConfiguration;
import android.hardware.camera2.utils.SubmitInfo;
+import android.hardware.common.fmq.MQDescriptor;
+import android.hardware.common.fmq.SynchronizedReadWrite;
import android.view.Surface;
/** @hide */
@@ -173,6 +175,7 @@
void finalizeOutputConfigurations(int streamId, in OutputConfiguration outputConfiguration);
+ MQDescriptor<byte, SynchronizedReadWrite> getCaptureResultMetadataQueue();
// Keep in sync with public API in
// frameworks/base/core/java/android/hardware/camera2/CameraDevice.java
diff --git a/camera/camera_platform.aconfig b/camera/camera_platform.aconfig
index 3b199b3..5682ad2 100644
--- a/camera/camera_platform.aconfig
+++ b/camera/camera_platform.aconfig
@@ -263,3 +263,13 @@
purpose: PURPOSE_BUGFIX
}
}
+
+flag {
+ namespace: "camera_platform"
+ name: "fmq_metadata"
+ description: "Allow CameraMetadata transfer for ndk / sdk clients."
+ bug: "362791857"
+ metadata {
+ purpose: PURPOSE_BUGFIX
+ }
+}
\ No newline at end of file
diff --git a/camera/include/camera/CameraMetadata.h b/camera/include/camera/CameraMetadata.h
index 2903dfb..10ecc4f 100644
--- a/camera/include/camera/CameraMetadata.h
+++ b/camera/include/camera/CameraMetadata.h
@@ -247,6 +247,38 @@
*/
metadata_vendor_id_t getVendorId() const;
+ // Needed for auto-generated code if CameraMetadata is used in
+ // parcelables in .aidl files.
+ inline bool operator == (const CameraMetadata& rhs) const {
+ return mBuffer == rhs.mBuffer;
+ }
+
+ inline bool operator < (const CameraMetadata& rhs) const {
+ return mBuffer < rhs.mBuffer;
+ }
+
+ inline bool operator != (const CameraMetadata& rhs) const {
+ return !(*this == rhs);
+ }
+
+ inline bool operator > (const CameraMetadata& rhs) const {
+ return rhs < *this;
+ }
+
+ inline bool operator >= (const CameraMetadata& rhs) const {
+ return !(*this < rhs);
+ }
+
+ inline bool operator <= (const CameraMetadata& rhs) const {
+ return !(rhs < *this);
+ }
+
+ inline std::string toString() const {
+ std::string descStr = "CameraMetadata";
+ return descStr;
+ }
+
+
private:
camera_metadata_t *mBuffer;
mutable bool mLocked;
@@ -265,7 +297,6 @@
* Resize metadata buffer if needed by reallocating it and copying it over.
*/
status_t resizeIfNeeded(size_t extraEntries, size_t extraData);
-
};
namespace hardware {
diff --git a/camera/include/camera/CaptureResult.h b/camera/include/camera/CaptureResult.h
index e08c9ca..cc6b529 100644
--- a/camera/include/camera/CaptureResult.h
+++ b/camera/include/camera/CaptureResult.h
@@ -20,7 +20,7 @@
#include <utils/RefBase.h>
#include <binder/Parcelable.h>
#include <camera/CameraMetadata.h>
-
+#include <android/hardware/camera2/CameraMetadataInfo.h>
namespace android {
@@ -145,19 +145,26 @@
};
struct PhysicalCaptureResultInfo : public android::Parcelable {
-
+ using CameraMetadataInfo = android::hardware::camera2::CameraMetadataInfo;
PhysicalCaptureResultInfo()
: mPhysicalCameraId(),
- mPhysicalCameraMetadata() {
+ mCameraMetadataInfo() {
}
PhysicalCaptureResultInfo(const std::string& cameraId,
const CameraMetadata& cameraMetadata)
- : mPhysicalCameraId(cameraId),
- mPhysicalCameraMetadata(cameraMetadata) {
+ : mPhysicalCameraId(cameraId) {
+ mCameraMetadataInfo.set<CameraMetadataInfo::metadata>(cameraMetadata);
+ }
+
+ PhysicalCaptureResultInfo(const std::string& cameraId,
+ uint64_t fmqSize)
+ : mPhysicalCameraId(cameraId) {
+ mCameraMetadataInfo.set<CameraMetadataInfo::fmqSize>(fmqSize);
}
std::string mPhysicalCameraId;
- CameraMetadata mPhysicalCameraMetadata;
+
+ CameraMetadataInfo mCameraMetadataInfo;
virtual status_t readFromParcel(const android::Parcel* parcel) override;
virtual status_t writeToParcel(android::Parcel* parcel) const override;
diff --git a/camera/ndk/Android.bp b/camera/ndk/Android.bp
index 508808f..fc1e547 100644
--- a/camera/ndk/Android.bp
+++ b/camera/ndk/Android.bp
@@ -79,6 +79,8 @@
shared_libs: [
"android.companion.virtual.virtualdevice_aidl-cpp",
"android.companion.virtualdevice.flags-aconfig-cc",
+ "android.hardware.common-V2-cpp",
+ "android.hardware.common.fmq-V1-cpp",
"camera_platform_flags_c_lib",
"framework-permission-aidl-cpp",
"libandroid_runtime",
@@ -86,6 +88,7 @@
"libcamera_client",
"libcamera_metadata",
"libcutils",
+ "libfmq",
"libgui",
"liblog",
"libmediandk",
diff --git a/camera/ndk/impl/ACameraDevice.cpp b/camera/ndk/impl/ACameraDevice.cpp
index aed740f..7840fa0 100644
--- a/camera/ndk/impl/ACameraDevice.cpp
+++ b/camera/ndk/impl/ACameraDevice.cpp
@@ -37,6 +37,8 @@
namespace android {
namespace acam {
+using android::hardware::common::fmq::MQDescriptor;
+
// Static member definitions
const char* CameraDevice::kContextKey = "Context";
const char* CameraDevice::kDeviceKey = "Device";
@@ -788,6 +790,27 @@
mRemote = remote;
}
+bool CameraDevice::setDeviceMetadataQueues() {
+ if (mRemote == nullptr) {
+ ALOGE("mRemote must not be null while trying to fetch metadata queues");
+ return false;
+ }
+ MQDescriptor<int8_t, SynchronizedReadWrite> resMqDescriptor;
+ binder::Status ret = mRemote->getCaptureResultMetadataQueue(&resMqDescriptor);
+ if (!ret.isOk()) {
+ ALOGE("Transaction error trying to get capture result metadata queue");
+ return false;
+ }
+ mCaptureResultMetadataQueue = std::make_unique<ResultMetadataQueue>(resMqDescriptor);
+ if (!mCaptureResultMetadataQueue->isValid()) {
+ ALOGE("Empty fmq from cameraserver");
+ mCaptureResultMetadataQueue = nullptr;
+ return false;
+ }
+
+ return true;
+}
+
camera_status_t
CameraDevice::checkCameraClosedOrErrorLocked() const {
if (mRemote == nullptr) {
@@ -1247,7 +1270,9 @@
String8 physicalId8 = toString8(physicalResultInfo[i].mPhysicalCameraId);
physicalCameraIds.push_back(physicalId8.c_str());
- CameraMetadata clone = physicalResultInfo[i].mPhysicalCameraMetadata;
+ CameraMetadata clone =
+ physicalResultInfo[i].
+ mCameraMetadataInfo.get<CameraMetadataInfo::metadata>();
clone.update(ANDROID_SYNC_FRAME_NUMBER,
&physicalResult->mFrameNumber, /*data_count*/1);
sp<ACameraMetadata> metadata =
@@ -1777,7 +1802,7 @@
binder::Status
CameraDevice::ServiceCallback::onResultReceived(
- const CameraMetadata& metadata,
+ const CameraMetadataInfo &resultMetadata,
const CaptureResultExtras& resultExtras,
const std::vector<PhysicalCaptureResultInfo>& physicalResultInfos) {
binder::Status ret = binder::Status::ok();
@@ -1786,11 +1811,11 @@
if (dev == nullptr) {
return ret; // device has been closed
}
+
int sequenceId = resultExtras.requestId;
int64_t frameNumber = resultExtras.frameNumber;
int32_t burstId = resultExtras.burstId;
bool isPartialResult = (resultExtras.partialResultCount < dev->mPartialResultCount);
-
if (!isPartialResult) {
ALOGV("SeqId %d frame %" PRId64 " result arrive.", sequenceId, frameNumber);
}
@@ -1808,7 +1833,13 @@
return ret;
}
- CameraMetadata metadataCopy = metadata;
+ CameraMetadata metadataCopy;
+ camera_status_t status = readOneResultMetadata(resultMetadata,
+ dev->mCaptureResultMetadataQueue.get(), &metadataCopy);
+ if (status != ACAMERA_OK) {
+ ALOGE("%s: result metadata couldn't be converted", __FUNCTION__);
+ return ret;
+ }
metadataCopy.update(ANDROID_LENS_INFO_SHADING_MAP_SIZE, dev->mShadingMapSize, /*data_count*/2);
metadataCopy.update(ANDROID_SYNC_FRAME_NUMBER, &frameNumber, /*data_count*/1);
@@ -1824,8 +1855,24 @@
sp<CaptureRequest> request = cbh.mRequests[burstId];
sp<ACameraMetadata> result(new ACameraMetadata(
metadataCopy.release(), ACameraMetadata::ACM_RESULT));
+
+ std::vector<PhysicalCaptureResultInfo> localPhysicalResult;
+ localPhysicalResult.resize(physicalResultInfos.size());
+ for (size_t i = 0; i < physicalResultInfos.size(); i++) {
+ CameraMetadata physicalMetadata;
+ localPhysicalResult[i].mPhysicalCameraId = physicalResultInfos[i].mPhysicalCameraId;
+ status = readOneResultMetadata(physicalResultInfos[i].mCameraMetadataInfo,
+ dev->mCaptureResultMetadataQueue.get(),
+ &physicalMetadata);
+ if (status != ACAMERA_OK) {
+ ALOGE("%s: physical camera result metadata couldn't be converted", __FUNCTION__);
+ return ret;
+ }
+ localPhysicalResult[i].mCameraMetadataInfo.set<CameraMetadataInfo::metadata>(
+ std::move(physicalMetadata));
+ }
sp<ACameraPhysicalCaptureResultInfo> physicalResult(
- new ACameraPhysicalCaptureResultInfo(physicalResultInfos, frameNumber));
+ new ACameraPhysicalCaptureResultInfo(localPhysicalResult, frameNumber));
sp<AMessage> msg = new AMessage(
cbh.mIsLogicalCameraCallback ? kWhatLogicalCaptureResult : kWhatCaptureResult,
@@ -1946,5 +1993,28 @@
}
}
+camera_status_t CameraDevice::ServiceCallback::readOneResultMetadata(
+ const CameraMetadataInfo& resultInfo, ResultMetadataQueue* metadataQueue,
+ CameraMetadata* metadata) {
+ if (metadataQueue == nullptr || metadata == nullptr) {
+ return ACAMERA_ERROR_INVALID_PARAMETER;
+ }
+ if (resultInfo.getTag() == CameraMetadataInfo::fmqSize) {
+ int64_t metadataSize = resultInfo.get<CameraMetadataInfo::fmqSize>();
+ auto metadataVec = std::make_unique<int8_t []>(metadataSize);
+ bool read = metadataQueue->read(reinterpret_cast<int8_t*>(metadataVec.get()), metadataSize);
+ if (!read) {
+ ALOGE("%s capture request settings could't be read from fmq", __FUNCTION__);
+ return ACAMERA_ERROR_UNKNOWN;
+ }
+ *metadata = CameraMetadata(reinterpret_cast<camera_metadata_t *>(metadataVec.release()));
+ } else {
+ *metadata =
+ resultInfo.get<CameraMetadataInfo::metadata>();
+ }
+
+ return ACAMERA_OK;
+}
+
} // namespace acam
} // namespace android
diff --git a/camera/ndk/impl/ACameraDevice.h b/camera/ndk/impl/ACameraDevice.h
index d3aed4b..ff68bb9 100644
--- a/camera/ndk/impl/ACameraDevice.h
+++ b/camera/ndk/impl/ACameraDevice.h
@@ -37,6 +37,7 @@
#include <camera/camera2/OutputConfiguration.h>
#include <camera/camera2/SessionConfiguration.h>
#include <camera/camera2/CaptureRequest.h>
+#include <fmq/AidlMessageQueueCpp.h>
#include <camera/NdkCameraManager.h>
#include <camera/NdkCameraCaptureSession.h>
@@ -46,6 +47,9 @@
namespace android {
namespace acam {
+using android::hardware::common::fmq::SynchronizedReadWrite;
+using ResultMetadataQueue = AidlMessageQueueCpp<int8_t, SynchronizedReadWrite>;
+
// Wrap ACameraCaptureFailure so it can be ref-counted
struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure {};
@@ -61,6 +65,8 @@
class CameraDevice final : public RefBase {
public:
+
+ using CameraMetadataInfo = android::hardware::camera2::CameraMetadataInfo;
CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb,
sp<ACameraMetadata> chars,
ACameraDevice* wrapper, bool sharedMode);
@@ -91,7 +97,7 @@
binder::Status onDeviceIdle() override;
binder::Status onCaptureStarted(const CaptureResultExtras& resultExtras,
int64_t timestamp) override;
- binder::Status onResultReceived(const CameraMetadata& metadata,
+ binder::Status onResultReceived(const CameraMetadataInfo &resultInfo,
const CaptureResultExtras& resultExtras,
const std::vector<PhysicalCaptureResultInfo>& physicalResultInfos) override;
binder::Status onPrepared(int streamId) override;
@@ -100,6 +106,9 @@
int32_t stoppedSequenceId) override;
binder::Status onClientSharedAccessPriorityChanged(bool isPrimaryClient) override;
private:
+ camera_status_t readOneResultMetadata(
+ const CameraMetadataInfo& resultInfo, ResultMetadataQueue* metadataQueue,
+ CameraMetadata* metadata);
const wp<CameraDevice> mDevice;
};
inline sp<hardware::camera2::ICameraDeviceCallbacks> getServiceCallback() {
@@ -108,6 +117,7 @@
// Camera device is only functional after remote being set
void setRemoteDevice(sp<hardware::camera2::ICameraDeviceUser> remote);
+ bool setDeviceMetadataQueues();
inline ACameraDevice* getWrapper() const { return mWrapper; };
@@ -399,6 +409,9 @@
int32_t mPartialResultCount; // const after constructor
std::vector<std::string> mPhysicalIds; // const after constructor
+ // Metadata queue to write the result metadata to.
+ std::unique_ptr<ResultMetadataQueue> mCaptureResultMetadataQueue;
+
};
} // namespace acam;
@@ -452,6 +465,10 @@
mDevice->setRemoteDevice(remote);
}
+ inline bool setDeviceMetadataQueues() {
+ return mDevice->setDeviceMetadataQueues();
+ }
+
inline void setPrimaryClient(bool isPrimary) {
mDevice->setPrimaryClient(isPrimary);
}
diff --git a/camera/ndk/impl/ACameraManager.cpp b/camera/ndk/impl/ACameraManager.cpp
index f9c1a8a..acd7917 100644
--- a/camera/ndk/impl/ACameraManager.cpp
+++ b/camera/ndk/impl/ACameraManager.cpp
@@ -971,6 +971,7 @@
return ACAMERA_ERROR_CAMERA_DISCONNECTED;
}
device->setRemoteDevice(deviceRemote);
+ device->setDeviceMetadataQueues();
if (flags::camera_multi_client() && sharedMode) {
binder::Status remoteRet = deviceRemote->isPrimaryClient(primaryClient);
if (!remoteRet.isOk()) {
diff --git a/camera/tests/CameraBinderTests.cpp b/camera/tests/CameraBinderTests.cpp
index 4384df9..5f7f2f6 100644
--- a/camera/tests/CameraBinderTests.cpp
+++ b/camera/tests/CameraBinderTests.cpp
@@ -63,6 +63,7 @@
using namespace android;
using ::android::hardware::ICameraService;
using ::android::hardware::camera2::ICameraDeviceUser;
+using ::android::hardware::camera2::CameraMetadataInfo;
#define ASSERT_NOT_NULL(x) \
ASSERT_TRUE((x) != nullptr)
@@ -249,10 +250,10 @@
return binder::Status::ok();
}
- virtual binder::Status onResultReceived(const CameraMetadata& metadata,
+ virtual binder::Status onResultReceived(const CameraMetadataInfo& resultInfo,
const CaptureResultExtras& resultExtras,
const std::vector<PhysicalCaptureResultInfo>& physicalResultInfos) {
- (void) metadata;
+ (void) resultInfo;
(void) resultExtras;
(void) physicalResultInfos;
Mutex::Autolock l(mLock);