Merge changes from topic "active_physical_camera_crop" into main
* changes:
Camera: Add the physical camera source crop metadata tag
Camera: Add concert mode feature flag
diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg
index 62cf827..1cf63b0 100644
--- a/PREUPLOAD.cfg
+++ b/PREUPLOAD.cfg
@@ -9,7 +9,8 @@
[Builtin Hooks Options]
# Only turn on clang-format check for the following subfolders.
clang_format = --commit ${PREUPLOAD_COMMIT} --style file --extensions c,h,cc,cpp
- media/libmediatranscoding/
- services/mediatranscoding/
media/libaudioclient/tests/
media/libaudiohal/tests/
+ media/libmediatranscoding/
+ services/camera/virtualcamera/
+ services/mediatranscoding/
diff --git a/camera/camera_platform.aconfig b/camera/camera_platform.aconfig
index 3236be4..90d73e4 100644
--- a/camera/camera_platform.aconfig
+++ b/camera/camera_platform.aconfig
@@ -43,7 +43,7 @@
flag {
namespace: "camera_platform"
- name: "virtual_camera_service_discovery"
- description: "Enable discovery of the Virtual Camera HAL without a VINTF entry"
- bug: "305170199"
-}
+ name: "session_hal_buf_manager"
+ description: "Enable or disable HAL buffer manager as requested by the camera HAL"
+ bug: "311263114"
+}
\ No newline at end of file
diff --git a/camera/cameraserver/manifest_android.frameworks.cameraservice.service.xml b/camera/cameraserver/manifest_android.frameworks.cameraservice.service.xml
index f7e455f..5d85909 100644
--- a/camera/cameraserver/manifest_android.frameworks.cameraservice.service.xml
+++ b/camera/cameraserver/manifest_android.frameworks.cameraservice.service.xml
@@ -11,7 +11,7 @@
<hal format="aidl">
<name>android.frameworks.cameraservice.service</name>
- <version>1</version>
+ <version>2</version>
<interface>
<name>ICameraService</name>
<instance>default</instance>
diff --git a/media/audio/aconfig/Android.bp b/media/audio/aconfig/Android.bp
index ee1a2c8..1c1ac0e 100644
--- a/media/audio/aconfig/Android.bp
+++ b/media/audio/aconfig/Android.bp
@@ -17,6 +17,12 @@
srcs: ["audio.aconfig"],
}
+aconfig_declarations {
+ name: "com.android.media.aaudio-aconfig",
+ package: "com.android.media.aaudio",
+ srcs: ["aaudio.aconfig"],
+}
+
cc_aconfig_library {
name: "com.android.media.audioserver-aconfig-cc",
aconfig_declarations: "com.android.media.audioserver-aconfig",
@@ -39,6 +45,12 @@
defaults: ["audio-aconfig-cc-defaults"],
}
+cc_aconfig_library {
+ name: "com.android.media.aaudio-aconfig-cc",
+ aconfig_declarations: "com.android.media.aaudio-aconfig",
+ defaults: ["audio-aconfig-cc-defaults"],
+}
+
java_aconfig_library {
name: "com.android.media.audio-aconfig-java",
aconfig_declarations: "com.android.media.audio-aconfig",
diff --git a/media/audio/aconfig/aaudio.aconfig b/media/audio/aconfig/aaudio.aconfig
new file mode 100644
index 0000000..7196525
--- /dev/null
+++ b/media/audio/aconfig/aaudio.aconfig
@@ -0,0 +1,12 @@
+# Flags for aaudio
+#
+# Please add flags in alphabetical order.
+
+package: "com.android.media.aaudio"
+
+flag {
+ name: "sample_rate_conversion"
+ namespace: "media_audio"
+ description: "Enable the AAudio sample rate converter."
+ bug: "219533889"
+}
diff --git a/media/audio/aconfig/audio.aconfig b/media/audio/aconfig/audio.aconfig
index 5559f09..73cb8ca 100644
--- a/media/audio/aconfig/audio.aconfig
+++ b/media/audio/aconfig/audio.aconfig
@@ -5,6 +5,13 @@
package: "com.android.media.audio"
flag {
+ name: "alarm_min_volume_zero"
+ namespace: "media_audio"
+ description: "Support configuring alarm min vol to zero"
+ bug: "296884402"
+}
+
+flag {
name: "bluetooth_mac_address_anonymization"
namespace: "media_audio"
description:
@@ -28,3 +35,10 @@
"Enable dynamic spatial audio over Bluetooth LE Audio."
bug: "307588546"
}
+
+flag {
+ name: "spatializer_offload"
+ namespace: "media_audio"
+ description: "Enable spatializer offload"
+ bug: "307842941"
+}
diff --git a/media/audio/aconfig/audio_framework.aconfig b/media/audio/aconfig/audio_framework.aconfig
index c6d0036..294e67d 100644
--- a/media/audio/aconfig/audio_framework.aconfig
+++ b/media/audio/aconfig/audio_framework.aconfig
@@ -12,6 +12,15 @@
bug: "302751899"
}
+flag {
+ name: "automatic_bt_device_type"
+ namespace: "media_audio"
+ description:
+ "Enable the automatic Bluetooth audio device type "
+ "categorization based on BluetoothDevice class metadata."
+ bug: "302323921"
+}
+
# TODO remove
flag {
name: "focus_freeze_test_api"
diff --git a/media/audio/aconfig/audioserver.aconfig b/media/audio/aconfig/audioserver.aconfig
index a25dd5f..21ea1a2 100644
--- a/media/audio/aconfig/audioserver.aconfig
+++ b/media/audio/aconfig/audioserver.aconfig
@@ -5,6 +5,22 @@
package: "com.android.media.audioserver"
flag {
+ name: "direct_track_reprioritization"
+ namespace: "media_audio"
+ description:
+ "Modify opening a direct output on a mixport to disrupt existing clients instead "
+ "of failing to open when resource limit is reached"
+ bug: "294525897"
+}
+
+flag {
+ name: "fdtostring_timeout_fix"
+ namespace: "media_audio"
+ description: "Improve fdtostring implementation to properly handle timing out."
+ bug: "306283018"
+}
+
+flag {
name: "mutex_priority_inheritance"
namespace: "media_audio"
description:
@@ -13,4 +29,3 @@
"This feature helps reduce audio glitching caused by low priority blocking threads."
bug: "209491695"
}
-
diff --git a/media/codec2/hal/aidl/BufferTypes.cpp b/media/codec2/hal/aidl/BufferTypes.cpp
index b1af579..bc4948b 100644
--- a/media/codec2/hal/aidl/BufferTypes.cpp
+++ b/media/codec2/hal/aidl/BufferTypes.cpp
@@ -201,7 +201,7 @@
template<>
void SetHandle(BaseBlock *block, const C2Handle *handle) {
- block->set<BaseBlock::nativeBlock>(makeToAidl(handle));
+ block->set<BaseBlock::nativeBlock>(dupToAidl(handle));
}
template<>
diff --git a/media/codec2/hal/aidl/Component.cpp b/media/codec2/hal/aidl/Component.cpp
index 4f5b899..4605af3 100644
--- a/media/codec2/hal/aidl/Component.cpp
+++ b/media/codec2/hal/aidl/Component.cpp
@@ -294,46 +294,49 @@
static constexpr IComponent::BlockPoolAllocator::Tag IGBA =
IComponent::BlockPoolAllocator::allocator;
c2_status_t status = C2_OK;
+ ::android::C2PlatformAllocatorDesc allocatorParam;
switch (allocator.getTag()) {
- case ALLOCATOR_ID:
-#ifdef __ANDROID_APEX__
- status = ::android::CreateCodec2BlockPool(
- static_cast<::android::C2PlatformAllocatorStore::id_t>(
- allocator.get<ALLOCATOR_ID>()),
- mComponent,
- &c2BlockPool);
-#else
- status = ComponentStore::GetFilterWrapper()->createBlockPool(
- static_cast<::android::C2PlatformAllocatorStore::id_t>(
- allocator.get<ALLOCATOR_ID>()),
- mComponent,
- &c2BlockPool);
-#endif
- if (status != C2_OK) {
- blockPool = nullptr;
- }
- break;
- case IGBA:
- // FIXME
- break;
+ case ALLOCATOR_ID: {
+ allocatorParam.allocatorId =
+ allocator.get<IComponent::BlockPoolAllocator::allocatorId>();
+ }
+ break;
+ case IGBA: {
+ allocatorParam.allocatorId = ::android::C2PlatformAllocatorStore::IGBA;
+ allocatorParam.igba =
+ allocator.get<IComponent::BlockPoolAllocator::allocator>().igba;
+ allocatorParam.waitableFd.reset(
+ allocator.get<IComponent::BlockPoolAllocator::allocator>()
+ .waitableFd.dup().release());
+ }
+ break;
default:
- break;
+ return ScopedAStatus::fromServiceSpecificError(C2_CORRUPTED);
}
- if (blockPool) {
+#ifdef __ANDROID_APEX__
+ status = ::android::CreateCodec2BlockPool(
+ allocatorParam,
+ mComponent,
+ &c2BlockPool);
+#else
+ status = ComponentStore::GetFilterWrapper()->createBlockPool(
+ allocatorParam,
+ mComponent,
+ &c2BlockPool);
+#endif
+ if (status != C2_OK) {
+ return ScopedAStatus::fromServiceSpecificError(status);
+ }
+ {
mBlockPoolsMutex.lock();
mBlockPools.emplace(c2BlockPool->getLocalId(), c2BlockPool);
mBlockPoolsMutex.unlock();
- } else if (status == C2_OK) {
- status = C2_CORRUPTED;
}
blockPool->blockPoolId = c2BlockPool ? c2BlockPool->getLocalId() : 0;
blockPool->configurable = SharedRefBase::make<CachedConfigurable>(
std::make_unique<BlockPoolIntf>(c2BlockPool));
- if (status == C2_OK) {
- return ScopedAStatus::ok();
- }
- return ScopedAStatus::fromServiceSpecificError(status);
+ return ScopedAStatus::ok();
}
ScopedAStatus Component::destroyBlockPool(int64_t blockPoolId) {
diff --git a/media/codec2/hal/client/client.cpp b/media/codec2/hal/client/client.cpp
index ce10109..2d19ecc 100644
--- a/media/codec2/hal/client/client.cpp
+++ b/media/codec2/hal/client/client.cpp
@@ -19,6 +19,8 @@
#define ATRACE_TAG ATRACE_TAG_VIDEO
#include <android-base/logging.h>
#include <utils/Trace.h>
+
+#include <codec2/aidl/GraphicBufferAllocator.h>
#include <codec2/hidl/client.h>
#include <C2Debug.h>
#include <C2BufferPriv.h>
@@ -74,6 +76,7 @@
#include <limits>
#include <map>
#include <mutex>
+#include <optional>
#include <sstream>
#include <thread>
#include <type_traits>
@@ -96,6 +99,9 @@
V2_0::utils::H2BGraphicBufferProducer;
using ::android::hardware::media::c2::V1_2::SurfaceSyncObj;
+using AidlGraphicBufferAllocator = ::aidl::android::hardware::media::c2::
+ implementation::GraphicBufferAllocator;
+
namespace bufferpool2_aidl = ::aidl::android::hardware::media::bufferpool2;
namespace bufferpool_hidl = ::android::hardware::media::bufferpool::V2_0;
namespace c2_aidl = ::aidl::android::hardware::media::c2;
@@ -1041,6 +1047,85 @@
}
};
+// The class holds GraphicBufferAllocator and the associated id of
+// HAL side BlockPool.
+// This is tightly coupled with BlockPool creation and destruction.
+// The life cycle inside class will be as follows.
+//
+// On createBlockPool client request.
+// 1. this::create() creates a GraphicBufferAllocator and set it as
+// the current.
+// 2. C2AIDL_HAL::createBlockPool() creates a C2BlockPool using
+// the GraphicBufferAllocator created in #1.
+// 3. this::setCurrentId() associates the id returned in #2 to the current
+//
+// On destroyBlockPool cliet request
+// 1. C2AIDL_HAL::destroyBlockPool() destroys the block pool
+// from HAL process.
+// 2. this::remove() destroys GraphicBufferAllocator which is associatted
+// with the C2BlockPool in #1.
+//
+struct Codec2Client::Component::GraphicBufferAllocators {
+private:
+ std::optional<C2BlockPool::local_id_t> mCurrentId;
+ std::shared_ptr<AidlGraphicBufferAllocator> mCurrent;
+
+ // A new BlockPool is created before the old BlockPool is destroyed.
+ // This holds the reference of the old BlockPool when a new BlockPool is
+ // created until the old BlockPool is explicitly requested for destruction.
+ std::map<C2BlockPool::local_id_t, std::shared_ptr<AidlGraphicBufferAllocator>> mOlds;
+ std::mutex mMutex;
+
+public:
+ // Creates a GraphicBufferAllocator which will be passed to HAL
+ // for creating C2BlockPool. And the created GraphicBufferAllocator
+ // will be used afterwards by current().
+ std::shared_ptr<AidlGraphicBufferAllocator> create() {
+ std::unique_lock<std::mutex> l(mMutex);
+ if (mCurrent) {
+ // If this is not stopped.
+ mCurrent->reset();
+ if (mCurrentId.has_value()) {
+ mOlds.emplace(mCurrentId.value(), mCurrent);
+ }
+ mCurrentId.reset();
+ mCurrent.reset();
+ }
+ // TODO: integrate initial value with CCodec/CCodecBufferChannel
+ mCurrent =
+ AidlGraphicBufferAllocator::CreateGraphicBufferAllocator(3 /* maxDequeueCount */);
+ ALOGD("GraphicBufferAllocator created");
+ return mCurrent;
+ }
+
+ // Associates the blockpool Id returned from HAL to the
+ // current GraphicBufferAllocator.
+ void setCurrentId(C2BlockPool::local_id_t id) {
+ std::unique_lock<std::mutex> l(mMutex);
+ CHECK(!mCurrentId.has_value());
+ mCurrentId = id;
+ }
+
+ // Returns the current GraphicBufferAllocator.
+ std::shared_ptr<AidlGraphicBufferAllocator> current() {
+ std::unique_lock<std::mutex> l(mMutex);
+ return mCurrent;
+ }
+
+ // Removes the GraphicBufferAllocator associated with given \p id.
+ void remove(C2BlockPool::local_id_t id) {
+ std::unique_lock<std::mutex> l(mMutex);
+ mOlds.erase(id);
+ if (mCurrentId == id) {
+ if (mCurrent) {
+ mCurrent->reset();
+ mCurrent.reset();
+ }
+ mCurrentId.reset();
+ }
+ }
+};
+
// Codec2Client
Codec2Client::Codec2Client(sp<HidlBase> const& base,
sp<c2_hidl::IConfigurable> const& configurable,
@@ -1125,6 +1210,7 @@
<< status << ".";
}
(*component)->mAidlBufferPoolSender->setReceiver(mAidlHostPoolManager);
+ aidlListener->component = *component;
return status;
}
@@ -1951,7 +2037,7 @@
},
mAidlBase{base},
mAidlBufferPoolSender{std::make_unique<AidlBufferPoolSender>()},
- mOutputBufferQueue{std::make_unique<OutputBufferQueue>()} {
+ mGraphicBufferAllocators{std::make_unique<GraphicBufferAllocators>()} {
}
Codec2Client::Component::~Component() {
@@ -1966,11 +2052,42 @@
std::shared_ptr<Codec2Client::Configurable>* configurable) {
if (mAidlBase) {
c2_aidl::IComponent::BlockPool aidlBlockPool;
- ::ndk::ScopedAStatus transStatus = mAidlBase->createBlockPool(static_cast<int32_t>(id),
- &aidlBlockPool);
- c2_status_t status = GetC2Status(transStatus, "createBlockPool");
- if (status != C2_OK) {
- return status;
+ c2_status_t status = C2_OK;
+
+ // TODO: Temporary mapping for the current CCodecBufferChannel.
+ // Handle this properly and remove this temporary allocator mapping.
+ id = id == C2PlatformAllocatorStore::BUFFERQUEUE ?
+ C2PlatformAllocatorStore::IGBA : id;
+
+ if (id == C2PlatformAllocatorStore::IGBA) {
+ std::shared_ptr<AidlGraphicBufferAllocator> gba =
+ mGraphicBufferAllocators->create();
+ ::ndk::ScopedFileDescriptor waitableFd;
+ ::ndk::ScopedAStatus ret = gba->getWaitableFd(&waitableFd);
+ status = GetC2Status(ret, "Gba::getWaitableFd");
+ if (status != C2_OK) {
+ return status;
+ }
+ c2_aidl::IComponent::BlockPoolAllocator allocator;
+ allocator.set<c2_aidl::IComponent::BlockPoolAllocator::allocator>();
+ allocator.get<c2_aidl::IComponent::BlockPoolAllocator::allocator>().igba =
+ c2_aidl::IGraphicBufferAllocator::fromBinder(gba->asBinder());
+ allocator.get<c2_aidl::IComponent::BlockPoolAllocator::allocator>().waitableFd =
+ std::move(waitableFd);
+ ::ndk::ScopedAStatus transStatus = mAidlBase->createBlockPool(
+ allocator, &aidlBlockPool);
+ status = GetC2Status(transStatus, "createBlockPool");
+ if (status != C2_OK) {
+ return status;
+ }
+ mGraphicBufferAllocators->setCurrentId(aidlBlockPool.blockPoolId);
+ } else {
+ ::ndk::ScopedAStatus transStatus = mAidlBase->createBlockPool(
+ static_cast<int32_t>(id), &aidlBlockPool);
+ status = GetC2Status(transStatus, "createBlockPool");
+ if (status != C2_OK) {
+ return status;
+ }
}
*blockPoolId = aidlBlockPool.blockPoolId;
*configurable = std::make_shared<Configurable>(aidlBlockPool.configurable);
@@ -2003,6 +2120,7 @@
c2_status_t Codec2Client::Component::destroyBlockPool(
C2BlockPool::local_id_t localId) {
if (mAidlBase) {
+ mGraphicBufferAllocators->remove(localId);
::ndk::ScopedAStatus transStatus = mAidlBase->destroyBlockPool(localId);
return GetC2Status(transStatus, "destroyBlockPool");
}
@@ -2017,8 +2135,12 @@
void Codec2Client::Component::handleOnWorkDone(
const std::list<std::unique_ptr<C2Work>> &workItems) {
- // Output bufferqueue-based blocks' lifetime management
- mOutputBufferQueue->holdBufferQueueBlocks(workItems);
+ if (mAidlBase) {
+ holdIgbaBlocks(workItems);
+ } else {
+ // Output bufferqueue-based blocks' lifetime management
+ mOutputBufferQueue->holdBufferQueueBlocks(workItems);
+ }
}
c2_status_t Codec2Client::Component::queue(
@@ -2102,8 +2224,12 @@
}
}
- // Output bufferqueue-based blocks' lifetime management
- mOutputBufferQueue->holdBufferQueueBlocks(*flushedWork);
+ if (mAidlBase) {
+ holdIgbaBlocks(*flushedWork);
+ } else {
+ // Output bufferqueue-based blocks' lifetime management
+ mOutputBufferQueue->holdBufferQueueBlocks(*flushedWork);
+ }
return status;
}
@@ -2242,6 +2368,17 @@
const sp<IGraphicBufferProducer>& surface,
uint32_t generation,
int maxDequeueCount) {
+ if (mAidlBase) {
+ std::shared_ptr<AidlGraphicBufferAllocator> gba =
+ mGraphicBufferAllocators->current();
+ if (!gba) {
+ LOG(ERROR) << "setOutputSurface for AIDL -- "
+ "GraphicBufferAllocator was not created.";
+ return C2_CORRUPTED;
+ }
+ bool ret = gba->configure(surface, generation, maxDequeueCount);
+ return ret ? C2_OK : C2_CORRUPTED;
+ }
uint64_t bqId = 0;
sp<IGraphicBufferProducer> nullIgbp;
sp<HGraphicBufferProducer2> nullHgbp;
@@ -2303,10 +2440,6 @@
ALOGD("setOutputSurface -- generation=%u consumer usage=%#llx%s",
generation, (long long)consumerUsage, syncObj ? " sync" : "");
- if (mAidlBase) {
- // FIXME
- return C2_OMITTED;
- }
Return<c2_hidl::Status> transStatus = syncObj ?
mHidlBase1_2->setOutputSurfaceWithSyncObj(
static_cast<uint64_t>(blockPoolId),
@@ -2335,26 +2468,51 @@
const QueueBufferInput& input,
QueueBufferOutput* output) {
ScopedTrace trace(ATRACE_TAG,"Codec2Client::Component::queueToOutputSurface");
+ if (mAidlBase) {
+ std::shared_ptr<AidlGraphicBufferAllocator> gba =
+ mGraphicBufferAllocators->current();
+ if (gba) {
+ return gba->displayBuffer(block, input, output);
+ } else {
+ return C2_NOT_FOUND;
+ }
+ }
return mOutputBufferQueue->outputBuffer(block, input, output);
}
void Codec2Client::Component::pollForRenderedFrames(FrameEventHistoryDelta* delta) {
+ if (mAidlBase) {
+ // TODO b/311348680
+ return;
+ }
mOutputBufferQueue->pollForRenderedFrames(delta);
}
void Codec2Client::Component::setOutputSurfaceMaxDequeueCount(
int maxDequeueCount) {
+ if (mAidlBase) {
+ std::shared_ptr<AidlGraphicBufferAllocator> gba =
+ mGraphicBufferAllocators->current();
+ if (gba) {
+ gba->updateMaxDequeueBufferCount(maxDequeueCount);
+ }
+ return;
+ }
mOutputBufferQueue->updateMaxDequeueBufferCount(maxDequeueCount);
}
void Codec2Client::Component::stopUsingOutputSurface(
C2BlockPool::local_id_t blockPoolId) {
- std::scoped_lock lock(mOutputMutex);
- mOutputBufferQueue->stop();
if (mAidlBase) {
- // FIXME
+ std::shared_ptr<AidlGraphicBufferAllocator> gba =
+ mGraphicBufferAllocators->current();
+ if (gba) {
+ gba->reset();
+ }
return;
}
+ std::scoped_lock lock(mOutputMutex);
+ mOutputBufferQueue->stop();
Return<c2_hidl::Status> transStatus = mHidlBase1_0->setOutputSurface(
static_cast<uint64_t>(blockPoolId), nullptr);
if (!transStatus.isOk()) {
@@ -2370,6 +2528,52 @@
mOutputBufferQueue->expireOldWaiters();
}
+void Codec2Client::Component::onBufferReleasedFromOutputSurface(
+ uint32_t generation) {
+ if (mAidlBase) {
+ std::shared_ptr<AidlGraphicBufferAllocator> gba =
+ mGraphicBufferAllocators->current();
+ if (gba) {
+ gba->onBufferReleased(generation);
+ }
+ return;
+ }
+ mOutputBufferQueue->onBufferReleased(generation);
+}
+
+void Codec2Client::Component::holdIgbaBlocks(
+ const std::list<std::unique_ptr<C2Work>>& workList) {
+ if (!mAidlBase) {
+ return;
+ }
+ std::shared_ptr<AidlGraphicBufferAllocator> gba =
+ mGraphicBufferAllocators->current();
+ if (!gba) {
+ return;
+ }
+ std::shared_ptr<c2_aidl::IGraphicBufferAllocator> igba =
+ c2_aidl::IGraphicBufferAllocator::fromBinder(gba->asBinder());
+ for (const std::unique_ptr<C2Work>& work : workList) {
+ if (!work) [[unlikely]] {
+ continue;
+ }
+ for (const std::unique_ptr<C2Worklet>& worklet : work->worklets) {
+ if (!worklet) {
+ continue;
+ }
+ for (const std::shared_ptr<C2Buffer>& buffer : worklet->output.buffers) {
+ if (buffer) {
+ for (const C2ConstGraphicBlock& block : buffer->data().graphicBlocks()) {
+ std::shared_ptr<_C2BlockPoolData> poolData =
+ _C2BlockFactory::GetGraphicBlockPoolData(block);
+ _C2BlockFactory::RegisterIgba(poolData, igba);
+ }
+ }
+ }
+ }
+ }
+}
+
c2_status_t Codec2Client::Component::connectToInputSurface(
const std::shared_ptr<InputSurface>& inputSurface,
std::shared_ptr<InputSurfaceConnection>* connection) {
diff --git a/media/codec2/hal/client/include/codec2/hidl/client.h b/media/codec2/hal/client/include/codec2/hidl/client.h
index 0c7dd77..3b7f7a6 100644
--- a/media/codec2/hal/client/include/codec2/hidl/client.h
+++ b/media/codec2/hal/client/include/codec2/hidl/client.h
@@ -474,6 +474,18 @@
void stopUsingOutputSurface(
C2BlockPool::local_id_t blockPoolId);
+ // Notify a buffer is released from output surface.
+ void onBufferReleasedFromOutputSurface(
+ uint32_t generation);
+
+ // When the client received \p workList and the blocks inside
+ // \p workList are IGBA based graphic blocks, specify the owner
+ // as the current IGBA for the future operations.
+ // Future operations could be rendering the blocks to the surface
+ // or deallocating blocks to the surface.
+ void holdIgbaBlocks(
+ const std::list<std::unique_ptr<C2Work>>& workList);
+
// Connect to a given InputSurface.
c2_status_t connectToInputSurface(
const std::shared_ptr<InputSurface>& inputSurface,
@@ -513,6 +525,9 @@
// In order to prevent the race condition mutex is added.
std::mutex mOutputMutex;
+ struct GraphicBufferAllocators;
+ std::unique_ptr<GraphicBufferAllocators> mGraphicBufferAllocators;
+
class AidlDeathManager;
static AidlDeathManager *GetAidlDeathManager();
std::optional<size_t> mAidlDeathSeq;
diff --git a/media/codec2/hal/client/include/codec2/hidl/output.h b/media/codec2/hal/client/include/codec2/hidl/output.h
index 2e89c3b..fda34a8 100644
--- a/media/codec2/hal/client/include/codec2/hidl/output.h
+++ b/media/codec2/hal/client/include/codec2/hidl/output.h
@@ -65,6 +65,10 @@
const BnGraphicBufferProducer::QueueBufferInput& input,
BnGraphicBufferProducer::QueueBufferOutput* output);
+ // Nofify a buffer is released from the output surface. If HAL ver is 1.2
+ // update the number of dequeueable/allocatable buffers.
+ void onBufferReleased(uint32_t generation);
+
// Retrieve frame event history from the output surface.
void pollForRenderedFrames(FrameEventHistoryDelta* delta);
diff --git a/media/codec2/hal/client/output.cpp b/media/codec2/hal/client/output.cpp
index 7f4f86b..36322f5 100644
--- a/media/codec2/hal/client/output.cpp
+++ b/media/codec2/hal/client/output.cpp
@@ -441,9 +441,11 @@
status = outputIgbp->queueBuffer(static_cast<int>(bqSlot),
input, output);
if (status == OK) {
- syncVar->lock();
- syncVar->notifyQueuedLocked();
- syncVar->unlock();
+ if (output->bufferReplaced) {
+ syncVar->lock();
+ syncVar->notifyQueuedLocked();
+ syncVar->unlock();
+ }
}
} else {
status = outputIgbp->queueBuffer(static_cast<int>(bqSlot),
@@ -496,9 +498,11 @@
status = outputIgbp->queueBuffer(static_cast<int>(bqSlot),
input, output);
if (status == OK) {
- syncVar->lock();
- syncVar->notifyQueuedLocked();
- syncVar->unlock();
+ if (output->bufferReplaced) {
+ syncVar->lock();
+ syncVar->notifyQueuedLocked();
+ syncVar->unlock();
+ }
}
} else {
status = outputIgbp->queueBuffer(static_cast<int>(bqSlot),
@@ -514,6 +518,30 @@
return OK;
}
+void OutputBufferQueue::onBufferReleased(uint32_t generation) {
+ std::shared_ptr<C2SurfaceSyncMemory> syncMem;
+ sp<IGraphicBufferProducer> outputIgbp;
+ uint32_t outputGeneration = 0;
+ {
+ std::unique_lock<std::mutex> l(mMutex);
+ if (mStopped) {
+ return;
+ }
+ outputIgbp = mIgbp;
+ outputGeneration = mGeneration;
+ syncMem = mSyncMem;
+ }
+
+ if (outputIgbp && generation == outputGeneration) {
+ auto syncVar = syncMem ? syncMem->mem() : nullptr;
+ if (syncVar) {
+ syncVar->lock();
+ syncVar->notifyQueuedLocked();
+ syncVar->unlock();
+ }
+ }
+}
+
void OutputBufferQueue::pollForRenderedFrames(FrameEventHistoryDelta* delta) {
if (mIgbp) {
mIgbp->getFrameTimestamps(delta);
diff --git a/media/codec2/hal/common/include/codec2/common/BufferTypes.h b/media/codec2/hal/common/include/codec2/common/BufferTypes.h
index afd2db0..af71122 100644
--- a/media/codec2/hal/common/include/codec2/common/BufferTypes.h
+++ b/media/codec2/hal/common/include/codec2/common/BufferTypes.h
@@ -183,7 +183,8 @@
baseBlocks, baseBlockIndices);
}
switch (blockPoolData->getType()) {
- case _C2BlockPoolData::TYPE_BUFFERPOOL: {
+ case _C2BlockPoolData::TYPE_BUFFERPOOL:
+ case _C2BlockPoolData::TYPE_BUFFERPOOL2: {
// BufferPoolData
std::shared_ptr<typename BufferPoolTypes::BufferPoolData> bpData;
if (!GetBufferPoolData<BufferPoolTypes>(blockPoolData, &bpData) || !bpData) {
@@ -194,28 +195,30 @@
index, bpData,
bufferPoolSender, baseBlocks, baseBlockIndices);
}
- case _C2BlockPoolData::TYPE_BUFFERQUEUE:
- uint32_t gen;
- uint64_t bqId;
- int32_t bqSlot;
- // Update handle if migration happened.
- if (_C2BlockFactory::GetBufferQueueData(
- blockPoolData, &gen, &bqId, &bqSlot)) {
- android::MigrateNativeCodec2GrallocHandle(
- const_cast<native_handle_t*>(handle), gen, bqId, bqSlot);
+ case _C2BlockPoolData::TYPE_BUFFERQUEUE: {
+ uint32_t gen;
+ uint64_t bqId;
+ int32_t bqSlot;
+ // Update handle if migration happened.
+ if (_C2BlockFactory::GetBufferQueueData(
+ blockPoolData, &gen, &bqId, &bqSlot)) {
+ android::MigrateNativeCodec2GrallocHandle(
+ const_cast<native_handle_t*>(handle), gen, bqId, bqSlot);
+ }
+ return _addBaseBlock(
+ index, handle,
+ baseBlocks, baseBlockIndices);
}
- return _addBaseBlock(
- index, handle,
- baseBlocks, baseBlockIndices);
- case _C2BlockPoolData::TYPE_AHWBUFFER:
- AHardwareBuffer *pBuf;
- if (!_C2BlockFactory::GetAHardwareBuffer(blockPoolData, &pBuf)) {
- LOG(ERROR) << "AHardwareBuffer unavailable in a block.";
- return false;
+ case _C2BlockPoolData::TYPE_AHWBUFFER: {
+ AHardwareBuffer *pBuf;
+ if (!_C2BlockFactory::GetAHardwareBuffer(blockPoolData, &pBuf)) {
+ LOG(ERROR) << "AHardwareBuffer unavailable in a block.";
+ return false;
+ }
+ return _addBaseBlock(
+ index, pBuf,
+ baseBlocks, baseBlockIndices);
}
- return _addBaseBlock(
- index, pBuf,
- baseBlocks, baseBlockIndices);
default:
LOG(ERROR) << "Unknown C2BlockPoolData type.";
return false;
diff --git a/media/codec2/hal/hidl/1.0/vts/functional/audio/VtsHalMediaC2V1_0TargetAudioDecTest.cpp b/media/codec2/hal/hidl/1.0/vts/functional/audio/VtsHalMediaC2V1_0TargetAudioDecTest.cpp
index 222c3d2..ce9fc39 100644
--- a/media/codec2/hal/hidl/1.0/vts/functional/audio/VtsHalMediaC2V1_0TargetAudioDecTest.cpp
+++ b/media/codec2/hal/hidl/1.0/vts/functional/audio/VtsHalMediaC2V1_0TargetAudioDecTest.cpp
@@ -14,10 +14,11 @@
* limitations under the License.
*/
-// #define LOG_NDEBUG 0
+//#define LOG_NDEBUG 0
#define LOG_TAG "codec2_hidl_hal_audio_dec_test"
#include <android-base/logging.h>
+#include <android/binder_process.h>
#include <gtest/gtest.h>
#include <hidl/GtestPrinter.h>
#include <stdio.h>
@@ -27,6 +28,7 @@
#include <C2BufferPriv.h>
#include <C2Config.h>
#include <C2Debug.h>
+#include <codec2/aidl/ParamTypes.h>
#include <codec2/hidl/client.h>
#include "media_c2_hidl_test_common.h"
@@ -88,7 +90,8 @@
std::shared_ptr<C2AllocatorStore> store = android::GetCodec2PlatformAllocatorStore();
CHECK_EQ(store->fetchAllocator(C2AllocatorStore::DEFAULT_LINEAR, &mLinearAllocator), C2_OK);
- mLinearPool = std::make_shared<C2PooledBlockPool>(mLinearAllocator, mBlockPoolId++);
+ mLinearPool = std::make_shared<C2PooledBlockPool>(
+ mLinearAllocator, mBlockPoolId++, getBufferPoolVer());
ASSERT_NE(mLinearPool, nullptr);
std::vector<std::unique_ptr<C2Param>> queried;
@@ -864,5 +867,6 @@
}
::testing::InitGoogleTest(&argc, argv);
+ ABinderProcess_startThreadPool();
return RUN_ALL_TESTS();
}
diff --git a/media/codec2/hal/hidl/1.0/vts/functional/audio/VtsHalMediaC2V1_0TargetAudioEncTest.cpp b/media/codec2/hal/hidl/1.0/vts/functional/audio/VtsHalMediaC2V1_0TargetAudioEncTest.cpp
index 327717b..f8c2903 100644
--- a/media/codec2/hal/hidl/1.0/vts/functional/audio/VtsHalMediaC2V1_0TargetAudioEncTest.cpp
+++ b/media/codec2/hal/hidl/1.0/vts/functional/audio/VtsHalMediaC2V1_0TargetAudioEncTest.cpp
@@ -14,10 +14,11 @@
* limitations under the License.
*/
-// #define LOG_NDEBUG 0
+//#define LOG_NDEBUG 0
#define LOG_TAG "codec2_hidl_hal_audio_enc_test"
#include <android-base/logging.h>
+#include <android/binder_process.h>
#include <gtest/gtest.h>
#include <hidl/GtestPrinter.h>
#include <stdio.h>
@@ -69,7 +70,8 @@
std::shared_ptr<C2AllocatorStore> store = android::GetCodec2PlatformAllocatorStore();
CHECK_EQ(store->fetchAllocator(C2AllocatorStore::DEFAULT_LINEAR, &mLinearAllocator), C2_OK);
- mLinearPool = std::make_shared<C2PooledBlockPool>(mLinearAllocator, mBlockPoolId++);
+ mLinearPool = std::make_shared<C2PooledBlockPool>(
+ mLinearAllocator, mBlockPoolId++, getBufferPoolVer());
ASSERT_NE(mLinearPool, nullptr);
std::vector<std::unique_ptr<C2Param>> queried;
@@ -775,6 +777,7 @@
std::make_tuple(std::get<0>(params), std::get<1>(params), true, 2));
}
+ ABinderProcess_startThreadPool();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
diff --git a/media/codec2/hal/hidl/1.0/vts/functional/common/Android.bp b/media/codec2/hal/hidl/1.0/vts/functional/common/Android.bp
index be4bafa..0f07077 100644
--- a/media/codec2/hal/hidl/1.0/vts/functional/common/Android.bp
+++ b/media/codec2/hal/hidl/1.0/vts/functional/common/Android.bp
@@ -11,6 +11,7 @@
name: "VtsHalMediaC2V1_0CommonUtil",
defaults: [
"VtsHalTargetTestDefaults",
+ "libcodec2-aidl-client-defaults",
"libcodec2-hidl-client-defaults",
],
@@ -29,6 +30,7 @@
name: "VtsHalMediaC2V1_0Defaults",
defaults: [
"VtsHalTargetTestDefaults",
+ "libcodec2-aidl-client-defaults",
"libcodec2-hidl-client-defaults",
],
@@ -38,6 +40,7 @@
],
shared_libs: [
+ "libbinder_ndk",
"libcodec2_client",
],
test_suites: [
diff --git a/media/codec2/hal/hidl/1.0/vts/functional/common/media_c2_hidl_test_common.cpp b/media/codec2/hal/hidl/1.0/vts/functional/common/media_c2_hidl_test_common.cpp
index 1f1681d..f36bc41 100644
--- a/media/codec2/hal/hidl/1.0/vts/functional/common/media_c2_hidl_test_common.cpp
+++ b/media/codec2/hal/hidl/1.0/vts/functional/common/media_c2_hidl_test_common.cpp
@@ -14,13 +14,14 @@
* limitations under the License.
*/
-// #define LOG_NDEBUG 0
+//#define LOG_NDEBUG 0
#define LOG_TAG "media_c2_hidl_test_common"
#include <stdio.h>
#include "media_c2_hidl_test_common.h"
#include <android/hardware/media/c2/1.0/IComponentStore.h>
+#include <codec2/aidl/ParamTypes.h>
std::string sResourceDir = "";
@@ -44,6 +45,14 @@
std::cerr << "\t -h, --help: Print usage \n";
}
+C2PooledBlockPool::BufferPoolVer getBufferPoolVer() {
+ if (::aidl::android::hardware::media::c2::utils::IsSelected()) {
+ return C2PooledBlockPool::VER_AIDL2;
+ } else {
+ return C2PooledBlockPool::VER_HIDL;
+ }
+}
+
void parseArgs(int argc, char** argv) {
int arg;
int option_index;
diff --git a/media/codec2/hal/hidl/1.0/vts/functional/common/media_c2_hidl_test_common.h b/media/codec2/hal/hidl/1.0/vts/functional/common/media_c2_hidl_test_common.h
index ecab0cb..48e80a4 100644
--- a/media/codec2/hal/hidl/1.0/vts/functional/common/media_c2_hidl_test_common.h
+++ b/media/codec2/hal/hidl/1.0/vts/functional/common/media_c2_hidl_test_common.h
@@ -17,8 +17,10 @@
#ifndef MEDIA_C2_HIDL_TEST_COMMON_H
#define MEDIA_C2_HIDL_TEST_COMMON_H
+#include <C2BufferPriv.h>
#include <C2Component.h>
#include <C2Config.h>
+#include <C2PlatformSupport.h>
#include <codec2/hidl/client.h>
#include <getopt.h>
@@ -126,6 +128,8 @@
std::function<void(std::list<std::unique_ptr<C2Work>>& workItems)> callBack;
};
+C2PooledBlockPool::BufferPoolVer getBufferPoolVer();
+
void parseArgs(int argc, char** argv);
// Return all test parameters, a list of tuple of <instance, component>.
diff --git a/media/codec2/hal/hidl/1.0/vts/functional/video/VtsHalMediaC2V1_0TargetVideoDecTest.cpp b/media/codec2/hal/hidl/1.0/vts/functional/video/VtsHalMediaC2V1_0TargetVideoDecTest.cpp
index d561adc..2cf0d6e 100644
--- a/media/codec2/hal/hidl/1.0/vts/functional/video/VtsHalMediaC2V1_0TargetVideoDecTest.cpp
+++ b/media/codec2/hal/hidl/1.0/vts/functional/video/VtsHalMediaC2V1_0TargetVideoDecTest.cpp
@@ -18,6 +18,7 @@
#define LOG_TAG "codec2_hidl_hal_video_dec_test"
#include <android-base/logging.h>
+#include <android/binder_process.h>
#include <gtest/gtest.h>
#include <hidl/GtestPrinter.h>
#include <stdio.h>
@@ -119,7 +120,8 @@
std::shared_ptr<C2AllocatorStore> store = android::GetCodec2PlatformAllocatorStore();
CHECK_EQ(store->fetchAllocator(C2AllocatorStore::DEFAULT_LINEAR, &mLinearAllocator), C2_OK);
- mLinearPool = std::make_shared<C2PooledBlockPool>(mLinearAllocator, mBlockPoolId++);
+ mLinearPool = std::make_shared<C2PooledBlockPool>(
+ mLinearAllocator, mBlockPoolId++, getBufferPoolVer());
ASSERT_NE(mLinearPool, nullptr);
std::vector<std::unique_ptr<C2Param>> queried;
@@ -1132,5 +1134,6 @@
}
::testing::InitGoogleTest(&argc, argv);
+ ABinderProcess_startThreadPool();
return RUN_ALL_TESTS();
}
diff --git a/media/codec2/hal/hidl/1.0/vts/functional/video/VtsHalMediaC2V1_0TargetVideoEncTest.cpp b/media/codec2/hal/hidl/1.0/vts/functional/video/VtsHalMediaC2V1_0TargetVideoEncTest.cpp
index db68b96..fbb4f18 100644
--- a/media/codec2/hal/hidl/1.0/vts/functional/video/VtsHalMediaC2V1_0TargetVideoEncTest.cpp
+++ b/media/codec2/hal/hidl/1.0/vts/functional/video/VtsHalMediaC2V1_0TargetVideoEncTest.cpp
@@ -18,6 +18,7 @@
#define LOG_TAG "codec2_hidl_hal_video_enc_test"
#include <android-base/logging.h>
+#include <android/binder_process.h>
#include <gtest/gtest.h>
#include <hidl/GtestPrinter.h>
#include <stdio.h>
@@ -930,5 +931,6 @@
}
::testing::InitGoogleTest(&argc, argv);
+ ABinderProcess_startThreadPool();
return RUN_ALL_TESTS();
}
diff --git a/media/codec2/hal/plugin/FilterWrapper.cpp b/media/codec2/hal/plugin/FilterWrapper.cpp
index d5124fd..197d6e7 100644
--- a/media/codec2/hal/plugin/FilterWrapper.cpp
+++ b/media/codec2/hal/plugin/FilterWrapper.cpp
@@ -969,6 +969,15 @@
C2PlatformAllocatorStore::id_t allocatorId,
std::shared_ptr<const C2Component> component,
std::shared_ptr<C2BlockPool> *pool) {
+ C2PlatformAllocatorDesc allocatorParam;
+ allocatorParam.allocatorId = allocatorId;
+ return createBlockPool(allocatorParam, component, pool);
+}
+
+c2_status_t FilterWrapper::createBlockPool(
+ C2PlatformAllocatorDesc &allocatorParam,
+ std::shared_ptr<const C2Component> component,
+ std::shared_ptr<C2BlockPool> *pool) {
std::unique_lock lock(mWrappedComponentsMutex);
for (auto it = mWrappedComponents.begin(); it != mWrappedComponents.end(); ) {
std::shared_ptr<const C2Component> comp = it->front().lock();
@@ -983,13 +992,13 @@
[](const std::weak_ptr<const C2Component> &el) {
return el.lock();
});
- if (C2_OK == CreateCodec2BlockPool(allocatorId, components, pool)) {
+ if (C2_OK == CreateCodec2BlockPool(allocatorParam, components, pool)) {
return C2_OK;
}
}
++it;
}
- return CreateCodec2BlockPool(allocatorId, component, pool);
+ return CreateCodec2BlockPool(allocatorParam, component, pool);
}
c2_status_t FilterWrapper::queryParamsForPreviousComponent(
diff --git a/media/codec2/hal/plugin/FilterWrapperStub.cpp b/media/codec2/hal/plugin/FilterWrapperStub.cpp
index 01ca596..3fd5409 100644
--- a/media/codec2/hal/plugin/FilterWrapperStub.cpp
+++ b/media/codec2/hal/plugin/FilterWrapperStub.cpp
@@ -45,7 +45,16 @@
C2PlatformAllocatorStore::id_t allocatorId,
std::shared_ptr<const C2Component> component,
std::shared_ptr<C2BlockPool> *pool) {
- return CreateCodec2BlockPool(allocatorId, component, pool);
+ C2PlatformAllocatorDesc allocatorParam;
+ allocatorParam.allocatorId = allocatorId;
+ return createBlockPool(allocatorParam, component, pool);
+}
+
+c2_status_t FilterWrapper::createBlockPool(
+ C2PlatformAllocatorDesc &allocatorParam,
+ std::shared_ptr<const C2Component> component,
+ std::shared_ptr<C2BlockPool> *pool) {
+ return CreateCodec2BlockPool(allocatorParam, component, pool);
}
} // namespace android
diff --git a/media/codec2/hal/plugin/internal/FilterWrapper.h b/media/codec2/hal/plugin/internal/FilterWrapper.h
index cf2cc30..dcffb5c 100644
--- a/media/codec2/hal/plugin/internal/FilterWrapper.h
+++ b/media/codec2/hal/plugin/internal/FilterWrapper.h
@@ -90,6 +90,14 @@
std::shared_ptr<C2BlockPool> *pool);
/**
+ * Create a C2BlockPool object with |allocatorParam| for |component|.
+ */
+ c2_status_t createBlockPool(
+ C2PlatformAllocatorDesc &allocatorParam,
+ std::shared_ptr<const C2Component> component,
+ std::shared_ptr<C2BlockPool> *pool);
+
+ /**
* Query parameters that |intf| wants from the previous component.
*/
c2_status_t queryParamsForPreviousComponent(
diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp
index ea13989..9c264af 100644
--- a/media/codec2/sfplugin/CCodec.cpp
+++ b/media/codec2/sfplugin/CCodec.cpp
@@ -867,6 +867,8 @@
sp<Surface> surface;
if (msg->findObject("native-window", &obj)) {
surface = static_cast<Surface *>(obj.get());
+ int32_t generation;
+ (void)msg->findInt32("native-window-generation", &generation);
// setup tunneled playback
if (surface != nullptr) {
Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
@@ -901,7 +903,7 @@
}
}
}
- setSurface(surface);
+ setSurface(surface, (uint32_t)generation);
}
Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
@@ -2041,7 +2043,7 @@
}
}
-status_t CCodec::setSurface(const sp<Surface> &surface) {
+status_t CCodec::setSurface(const sp<Surface> &surface, uint32_t generation) {
bool pushBlankBuffer = false;
{
Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
@@ -2070,7 +2072,7 @@
}
pushBlankBuffer = config->mPushBlankBuffersOnStop;
}
- return mChannel->setSurface(surface, pushBlankBuffer);
+ return mChannel->setSurface(surface, generation, pushBlankBuffer);
}
void CCodec::signalFlush() {
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp
index 718bc8f..6b45e0e 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.cpp
+++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp
@@ -1132,6 +1132,17 @@
processRenderedFrames(delta);
}
+void CCodecBufferChannel::onBufferReleasedFromOutputSurface(uint32_t generation) {
+ // Note: Since this is called asynchronously from IProducerListener not
+ // knowing the internal state of CCodec/CCodecBufferChannel,
+ // prevent mComponent from being destroyed by holding the shared reference
+ // during this interface being executed.
+ std::shared_ptr<Codec2Client::Component> comp = mComponent;
+ if (comp) {
+ comp->onBufferReleasedFromOutputSurface(generation);
+ }
+}
+
status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
bool released = false;
@@ -2285,12 +2296,8 @@
}
}
-status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface, bool pushBlankBuffer) {
- static std::atomic_uint32_t surfaceGeneration{0};
- uint32_t generation = (getpid() << 10) |
- ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
- & ((1 << 10) - 1));
-
+status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface,
+ uint32_t generation, bool pushBlankBuffer) {
sp<IGraphicBufferProducer> producer;
int maxDequeueCount;
sp<Surface> oldSurface;
@@ -2304,7 +2311,6 @@
newSurface->setDequeueTimeout(kDequeueTimeoutNs);
newSurface->setMaxDequeuedBufferCount(maxDequeueCount);
producer = newSurface->getIGraphicBufferProducer();
- producer->setGenerationNumber(generation);
} else {
ALOGE("[%s] setting output surface to null", mName);
return INVALID_OPERATION;
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.h b/media/codec2/sfplugin/CCodecBufferChannel.h
index 775bbbf..8dc9fb6 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.h
+++ b/media/codec2/sfplugin/CCodecBufferChannel.h
@@ -91,6 +91,7 @@
status_t renderOutputBuffer(
const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) override;
void pollForRenderedBuffers() override;
+ void onBufferReleasedFromOutputSurface(uint32_t generation) override;
status_t discardBuffer(const sp<MediaCodecBuffer> &buffer) override;
void getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) override;
void getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) override;
@@ -105,7 +106,7 @@
/**
* Set output graphic surface for rendering.
*/
- status_t setSurface(const sp<Surface> &surface, bool pushBlankBuffer);
+ status_t setSurface(const sp<Surface> &surface, uint32_t generation, bool pushBlankBuffer);
/**
* Set GraphicBufferSource object from which the component extracts input
diff --git a/media/codec2/sfplugin/include/media/stagefright/CCodec.h b/media/codec2/sfplugin/include/media/stagefright/CCodec.h
index 3b23470..2b1cf60 100644
--- a/media/codec2/sfplugin/include/media/stagefright/CCodec.h
+++ b/media/codec2/sfplugin/include/media/stagefright/CCodec.h
@@ -56,7 +56,7 @@
virtual void initiateStart() override;
virtual void initiateShutdown(bool keepComponentAllocated = false) override;
- virtual status_t setSurface(const sp<Surface> &surface) override;
+ virtual status_t setSurface(const sp<Surface> &surface, uint32_t generation) override;
virtual void signalFlush() override;
virtual void signalResume() override;
diff --git a/media/codec2/vndk/Android.bp b/media/codec2/vndk/Android.bp
index 265d87d..af2683b 100644
--- a/media/codec2/vndk/Android.bp
+++ b/media/codec2/vndk/Android.bp
@@ -52,6 +52,9 @@
"com.android.media.swcodec",
],
+ defaults: [
+ "libcodec2_hal_selection",
+ ],
srcs: [
"C2AllocatorBlob.cpp",
@@ -129,6 +132,10 @@
cc_defaults {
name: "libcodec2-static-defaults",
+ defaults: [
+ "libcodec2_hal_selection",
+ ],
+
static_libs: [
"liblog",
"libion",
@@ -173,6 +180,10 @@
name: "libcodec2-impl-defaults",
cpp_std: "gnu++17",
+ defaults: [
+ "libcodec2_hal_selection",
+ ],
+
shared_libs: [
"libbase", // for C2_LOG
"liblog", // for ALOG
diff --git a/media/codec2/vndk/C2Fence.cpp b/media/codec2/vndk/C2Fence.cpp
index 4c385f1..52ebe25 100644
--- a/media/codec2/vndk/C2Fence.cpp
+++ b/media/codec2/vndk/C2Fence.cpp
@@ -26,6 +26,8 @@
#include <C2FenceFactory.h>
#include <C2SurfaceSyncObj.h>
+#include <utility>
+
#define MAX_FENCE_FDS 1
class C2Fence::Impl {
@@ -485,17 +487,26 @@
mValid = (mPipeFd.get() >= 0);
}
+ PipeFenceImpl(::android::base::unique_fd &&ufd) : mPipeFd{std::move(ufd)} {
+ mValid = (mPipeFd.get() >= 0);
+ }
+
private:
friend struct _C2FenceFactory;
static constexpr int kPipeFenceWaitLimitSecs = 5;
mutable std::atomic<bool> mValid;
- ::android::base::unique_fd mPipeFd;
+ const ::android::base::unique_fd mPipeFd;
};
C2Fence _C2FenceFactory::CreatePipeFence(int fd) {
+ ::android::base::unique_fd ufd{fd};
+ return CreatePipeFence(std::move(ufd));
+}
+
+C2Fence _C2FenceFactory::CreatePipeFence(::android::base::unique_fd &&ufd) {
std::shared_ptr<_C2FenceFactory::PipeFenceImpl> impl =
- std::make_shared<_C2FenceFactory::PipeFenceImpl>(fd);
+ std::make_shared<_C2FenceFactory::PipeFenceImpl>(std::move(ufd));
std::shared_ptr<C2Fence::Impl> p = std::static_pointer_cast<C2Fence::Impl>(impl);
if (!p) {
ALOGE("PipeFence creation failure");
diff --git a/media/codec2/vndk/C2Store.cpp b/media/codec2/vndk/C2Store.cpp
index 61aafa7..e7fd14f 100644
--- a/media/codec2/vndk/C2Store.cpp
+++ b/media/codec2/vndk/C2Store.cpp
@@ -26,11 +26,15 @@
#include <C2BqBufferPriv.h>
#include <C2Component.h>
#include <C2Config.h>
+#include <C2IgbaBufferPriv.h>
#include <C2PlatformStorePluginLoader.h>
#include <C2PlatformSupport.h>
+#include <codec2/common/HalSelection.h>
#include <cutils/properties.h>
#include <util/C2InterfaceHelper.h>
+#include <aidl/android/hardware/media/c2/IGraphicBufferAllocator.h>
+
#include <dlfcn.h>
#include <unistd.h> // getpagesize
@@ -91,6 +95,9 @@
/// returns a shared-singleton bufferqueue supporting gralloc allocator
std::shared_ptr<C2Allocator> fetchBufferQueueAllocator();
+ /// returns a shared-singleton IGBA supporting AHardwareBuffer/gralloc allocator
+ std::shared_ptr<C2Allocator> fetchIgbaAllocator();
+
/// component store to use
std::mutex _mComponentStoreSetLock; // protects the entire updating _mComponentStore and its
// dependencies
@@ -157,6 +164,10 @@
*allocator = fetchBlobAllocator();
break;
+ case C2PlatformAllocatorStore::IGBA:
+ *allocator = fetchIgbaAllocator();
+ break;
+
default:
// Try to create allocator from platform store plugins.
c2_status_t res =
@@ -388,6 +399,18 @@
return allocator;
}
+std::shared_ptr<C2Allocator> C2PlatformAllocatorStoreImpl::fetchIgbaAllocator() {
+ static std::mutex mutex;
+ static std::weak_ptr<C2Allocator> ahwbAllocator;
+ std::lock_guard<std::mutex> lock(mutex);
+ std::shared_ptr<C2Allocator> allocator = ahwbAllocator.lock();
+ if (allocator == nullptr) {
+ allocator = std::make_shared<C2AllocatorAhwb>(C2PlatformAllocatorStore::IGBA);
+ ahwbAllocator = allocator;
+ }
+ return allocator;
+}
+
namespace {
std::mutex gPreferredComponentStoreMutex;
std::shared_ptr<C2ComponentStore> gPreferredComponentStore;
@@ -447,18 +470,25 @@
namespace {
+static C2PooledBlockPool::BufferPoolVer GetBufferPoolVer() {
+ static C2PooledBlockPool::BufferPoolVer sVer =
+ IsCodec2AidlHalSelected() ? C2PooledBlockPool::VER_AIDL2 : C2PooledBlockPool::VER_HIDL;
+ return sVer;
+}
+
class _C2BlockPoolCache {
public:
_C2BlockPoolCache() : mBlockPoolSeqId(C2BlockPool::PLATFORM_START + 1) {}
private:
c2_status_t _createBlockPool(
- C2PlatformAllocatorStore::id_t allocatorId,
+ C2PlatformAllocatorDesc &allocatorParam,
std::vector<std::shared_ptr<const C2Component>> components,
C2BlockPool::local_id_t poolId,
std::shared_ptr<C2BlockPool> *pool) {
std::shared_ptr<C2AllocatorStore> allocatorStore =
GetCodec2PlatformAllocatorStore();
+ C2PlatformAllocatorStore::id_t allocatorId = allocatorParam.allocatorId;
std::shared_ptr<C2Allocator> allocator;
c2_status_t res = C2_NOT_FOUND;
@@ -477,7 +507,7 @@
C2PlatformAllocatorStore::ION, &allocator);
if (res == C2_OK) {
std::shared_ptr<C2BlockPool> ptr(
- new C2PooledBlockPool(allocator, poolId), deleter);
+ new C2PooledBlockPool(allocator, poolId, GetBufferPoolVer()), deleter);
*pool = ptr;
mBlockPools[poolId] = ptr;
mComponents[poolId].insert(
@@ -490,7 +520,7 @@
C2PlatformAllocatorStore::BLOB, &allocator);
if (res == C2_OK) {
std::shared_ptr<C2BlockPool> ptr(
- new C2PooledBlockPool(allocator, poolId), deleter);
+ new C2PooledBlockPool(allocator, poolId, GetBufferPoolVer()), deleter);
*pool = ptr;
mBlockPools[poolId] = ptr;
mComponents[poolId].insert(
@@ -504,7 +534,7 @@
C2AllocatorStore::DEFAULT_GRAPHIC, &allocator);
if (res == C2_OK) {
std::shared_ptr<C2BlockPool> ptr(
- new C2PooledBlockPool(allocator, poolId), deleter);
+ new C2PooledBlockPool(allocator, poolId, GetBufferPoolVer()), deleter);
*pool = ptr;
mBlockPools[poolId] = ptr;
mComponents[poolId].insert(
@@ -525,6 +555,22 @@
components.begin(), components.end());
}
break;
+ case C2PlatformAllocatorStore::IGBA:
+ res = allocatorStore->fetchAllocator(
+ C2PlatformAllocatorStore::IGBA, &allocator);
+ if (res == C2_OK) {
+ std::shared_ptr<C2BlockPool> ptr(
+ new C2IgbaBlockPool(allocator,
+ allocatorParam.igba,
+ std::move(allocatorParam.waitableFd),
+ poolId), deleter);
+ *pool = ptr;
+ mBlockPools[poolId] = ptr;
+ mComponents[poolId].insert(
+ mComponents[poolId].end(),
+ components.begin(), components.end());
+ }
+ break;
default:
// Try to create block pool from platform store plugins.
std::shared_ptr<C2BlockPool> ptr;
@@ -547,10 +593,20 @@
C2PlatformAllocatorStore::id_t allocatorId,
std::vector<std::shared_ptr<const C2Component>> components,
std::shared_ptr<C2BlockPool> *pool) {
- std::unique_lock lock(mMutex);
- return _createBlockPool(allocatorId, components, mBlockPoolSeqId++, pool);
+ C2PlatformAllocatorDesc allocator;
+ allocator.allocatorId = allocatorId;
+ return createBlockPool(allocator, components, pool);
}
+ c2_status_t createBlockPool(
+ C2PlatformAllocatorDesc &allocator,
+ std::vector<std::shared_ptr<const C2Component>> components,
+ std::shared_ptr<C2BlockPool> *pool) {
+ std::unique_lock lock(mMutex);
+ return _createBlockPool(allocator, components, mBlockPoolSeqId++, pool);
+ }
+
+
c2_status_t getBlockPool(
C2BlockPool::local_id_t blockPoolId,
std::shared_ptr<const C2Component> component,
@@ -579,8 +635,10 @@
}
// TODO: remove this. this is temporary
if (blockPoolId == C2BlockPool::PLATFORM_START) {
+ C2PlatformAllocatorDesc allocator;
+ allocator.allocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
return _createBlockPool(
- C2PlatformAllocatorStore::BUFFERQUEUE, {component}, blockPoolId, pool);
+ allocator, {component}, blockPoolId, pool);
}
return C2_NOT_FOUND;
}
@@ -637,7 +695,9 @@
std::shared_ptr<C2BlockPool> *pool) {
pool->reset();
- return sBlockPoolCache->createBlockPool(allocatorId, components, pool);
+ C2PlatformAllocatorDesc allocator;
+ allocator.allocatorId = allocatorId;
+ return sBlockPoolCache->createBlockPool(allocator, components, pool);
}
c2_status_t CreateCodec2BlockPool(
@@ -646,7 +706,27 @@
std::shared_ptr<C2BlockPool> *pool) {
pool->reset();
- return sBlockPoolCache->createBlockPool(allocatorId, {component}, pool);
+ C2PlatformAllocatorDesc allocator;
+ allocator.allocatorId = allocatorId;
+ return sBlockPoolCache->createBlockPool(allocator, {component}, pool);
+}
+
+c2_status_t CreateCodec2BlockPool(
+ C2PlatformAllocatorDesc &allocator,
+ const std::vector<std::shared_ptr<const C2Component>> &components,
+ std::shared_ptr<C2BlockPool> *pool) {
+ pool->reset();
+
+ return sBlockPoolCache->createBlockPool(allocator, components, pool);
+}
+
+c2_status_t CreateCodec2BlockPool(
+ C2PlatformAllocatorDesc &allocator,
+ std::shared_ptr<const C2Component> component,
+ std::shared_ptr<C2BlockPool> *pool) {
+ pool->reset();
+
+ return sBlockPoolCache->createBlockPool(allocator, {component}, pool);
}
class C2PlatformComponentStore : public C2ComponentStore {
diff --git a/media/codec2/vndk/include/C2FenceFactory.h b/media/codec2/vndk/include/C2FenceFactory.h
index 9b09980..4f974ca 100644
--- a/media/codec2/vndk/include/C2FenceFactory.h
+++ b/media/codec2/vndk/include/C2FenceFactory.h
@@ -20,6 +20,8 @@
#include <C2Buffer.h>
+#include <android-base/unique_fd.h>
+
/*
* Create a list of fds from fence
*
@@ -69,6 +71,7 @@
/*
* Create C2Fence from an fd created by pipe()/pipe2() syscall.
+ * The ownership of \p fd is transterred to the returned C2Fence.
*
* \param fd An fd representing the write end from a pair of
* file descriptors which are created by
@@ -76,6 +79,15 @@
*/
static C2Fence CreatePipeFence(int fd);
+ /*
+ * Create C2Fence from a unique_fd created by pipe()/pipe2() syscall.
+ *
+ * \param ufd A unique_fd representing the write end from a pair
+ * of file descriptors which are created by
+ * pipe()/pipe2() syscall.
+ */
+ static C2Fence CreatePipeFence(::android::base::unique_fd &&ufd);
+
/**
* Create a native handle from fence for marshalling
*
diff --git a/media/codec2/vndk/include/C2IgbaBufferPriv.h b/media/codec2/vndk/include/C2IgbaBufferPriv.h
index a5676b7..5879263 100644
--- a/media/codec2/vndk/include/C2IgbaBufferPriv.h
+++ b/media/codec2/vndk/include/C2IgbaBufferPriv.h
@@ -17,6 +17,8 @@
#include <C2Buffer.h>
+#include <android-base/unique_fd.h>
+
#include <memory>
namespace aidl::android::hardware::media::c2 {
@@ -32,8 +34,9 @@
public:
explicit C2IgbaBlockPool(
const std::shared_ptr<C2Allocator> &allocator,
- const std::shared_ptr<
- ::aidl::android::hardware::media::c2::IGraphicBufferAllocator> &igba,
+ const std::shared_ptr<::aidl::android::hardware::media::c2::IGraphicBufferAllocator>
+ &igba,
+ ::android::base::unique_fd &&ufd,
const local_id_t localId);
virtual ~C2IgbaBlockPool() = default;
@@ -89,8 +92,7 @@
C2IgbaBlockPoolData(
const AHardwareBuffer *buffer,
- const std::shared_ptr<::aidl::android::hardware::media::c2::IGraphicBufferAllocator>
- &igba);
+ std::shared_ptr<::aidl::android::hardware::media::c2::IGraphicBufferAllocator> &igba);
virtual ~C2IgbaBlockPoolData() override;
@@ -103,7 +105,10 @@
void disown();
+ void registerIgba(std::shared_ptr<
+ ::aidl::android::hardware::media::c2::IGraphicBufferAllocator> &igba);
+
bool mOwned;
const AHardwareBuffer *mBuffer;
- const std::weak_ptr<::aidl::android::hardware::media::c2::IGraphicBufferAllocator> mIgba;
+ std::weak_ptr<::aidl::android::hardware::media::c2::IGraphicBufferAllocator> mIgba;
};
diff --git a/media/codec2/vndk/include/C2PlatformSupport.h b/media/codec2/vndk/include/C2PlatformSupport.h
index 221a799..6fa155a 100644
--- a/media/codec2/vndk/include/C2PlatformSupport.h
+++ b/media/codec2/vndk/include/C2PlatformSupport.h
@@ -22,6 +22,12 @@
#include <memory>
+#include <android-base/unique_fd.h>
+
+namespace aidl::android::hardware::media::c2 {
+class IGraphicBufferAllocator;
+}
+
namespace android {
/**
@@ -164,6 +170,53 @@
std::shared_ptr<C2BlockPool> *pool);
/**
+ * BlockPool creation parameters regarding allocator.
+ *
+ * igba, waitableFd are required only when allocatorId is
+ * C2PlatformAllocatorStore::IGBA.
+ */
+struct C2PlatformAllocatorDesc {
+ C2PlatformAllocatorStore::id_t allocatorId;
+ std::shared_ptr<::aidl::android::hardware::media::c2::IGraphicBufferAllocator> igba;
+ ::android::base::unique_fd waitableFd; // This will be passed and moved to C2Fence
+ // implementation.
+};
+
+/**
+ * Creates a block pool.
+ * \param allocator allocator ID and parameters which are used to allocate blocks
+ * \param component the component using the block pool (must be non-null)
+ * \param pool pointer to where the created block pool shall be store on success.
+ * nullptr will be stored here on failure
+ *
+ * \retval C2_OK the operation was successful
+ * \retval C2_BAD_VALUE the component is null
+ * \retval C2_NOT_FOUND if the allocator does not exist
+ * \retval C2_NO_MEMORY not enough memory to create a block pool
+ */
+c2_status_t CreateCodec2BlockPool(
+ C2PlatformAllocatorDesc &allocator,
+ std::shared_ptr<const C2Component> component,
+ std::shared_ptr<C2BlockPool> *pool);
+
+/**
+ * Creates a block pool.
+ * \param allocator allocator ID and parameters which are used to allocate blocks
+ * \param components the components using the block pool
+ * \param pool pointer to where the created block pool shall be store on success.
+ * nullptr will be stored here on failure
+ *
+ * \retval C2_OK the operation was successful
+ * \retval C2_BAD_VALUE the component is null
+ * \retval C2_NOT_FOUND if the allocator does not exist
+ * \retval C2_NO_MEMORY not enough memory to create a block pool
+ */
+c2_status_t CreateCodec2BlockPool(
+ C2PlatformAllocatorDesc &allocator,
+ const std::vector<std::shared_ptr<const C2Component>> &components,
+ std::shared_ptr<C2BlockPool> *pool);
+
+/**
* Returns the platform component store.
* \retval nullptr if the platform component store could not be obtained
*/
diff --git a/media/codec2/vndk/internal/C2BlockInternal.h b/media/codec2/vndk/internal/C2BlockInternal.h
index 8198ee1..4baf2db 100644
--- a/media/codec2/vndk/internal/C2BlockInternal.h
+++ b/media/codec2/vndk/internal/C2BlockInternal.h
@@ -39,6 +39,12 @@
}
+namespace aidl::android::hardware::media::c2 {
+
+// IGraphicBufferAllocator for media.c2 aidl
+class IGraphicBufferAllocator;
+}
+
typedef struct AHardwareBuffer AHardwareBuffer;
using bufferpool_BufferPoolData = android::hardware::media::bufferpool::BufferPoolData;
@@ -472,6 +478,16 @@
*/
static void DisownIgbaBlock(
const std::shared_ptr<_C2BlockPoolData>& poolData);
+
+ /**
+ * When the client receives a block from HAL, the client needs to store
+ * IGraphicBufferAllocator from which the block was originally allocated.
+ * The stored \p igba will be used in the dtor to deallocate the buffer.
+ * (calling IGraphicBufferAllocator::deallocate to reclaim.)
+ */
+ static void RegisterIgba(
+ const std::shared_ptr<_C2BlockPoolData>& poolData,
+ std::shared_ptr<::aidl::android::hardware::media::c2::IGraphicBufferAllocator> &igba);
};
#endif // ANDROID_STAGEFRIGHT_C2BLOCK_INTERNAL_H_
diff --git a/media/codec2/vndk/platform/C2BqBuffer.cpp b/media/codec2/vndk/platform/C2BqBuffer.cpp
index 960fa79..62b0ab5 100644
--- a/media/codec2/vndk/platform/C2BqBuffer.cpp
+++ b/media/codec2/vndk/platform/C2BqBuffer.cpp
@@ -35,6 +35,7 @@
#include <C2FenceFactory.h>
#include <C2SurfaceSyncObj.h>
+#include <atomic>
#include <list>
#include <map>
#include <mutex>
@@ -753,8 +754,8 @@
}
void invalidate() {
- std::scoped_lock<std::mutex> lock(mMutex);
mInvalidated = true;
+ mIgbpValidityToken.reset();
}
private:
@@ -794,7 +795,7 @@
// if the token has been expired, the buffers will not call IGBP::cancelBuffer()
// when they are no longer used.
std::shared_ptr<int> mIgbpValidityToken;
- bool mInvalidated{false};
+ std::atomic<bool> mInvalidated{false};
};
C2BufferQueueBlockPoolData::C2BufferQueueBlockPoolData(
diff --git a/media/codec2/vndk/platform/C2IgbaBuffer.cpp b/media/codec2/vndk/platform/C2IgbaBuffer.cpp
index 853d5a3..2051e8f 100644
--- a/media/codec2/vndk/platform/C2IgbaBuffer.cpp
+++ b/media/codec2/vndk/platform/C2IgbaBuffer.cpp
@@ -67,7 +67,8 @@
return err;
}
std::shared_ptr<C2IgbaBlockPoolData> poolData =
- std::make_shared<C2IgbaBlockPoolData>(ahwb, igba);
+ std::make_shared<C2IgbaBlockPoolData>(
+ ahwb, const_cast<std::shared_ptr<C2IGBA>&>(igba));
*block = _C2BlockFactory::CreateGraphicBlock(alloc, poolData);
return C2_OK;
} else {
@@ -79,7 +80,7 @@
C2IgbaBlockPoolData::C2IgbaBlockPoolData(
const AHardwareBuffer *buffer,
- const std::shared_ptr<C2IGBA> &igba) : mOwned(true), mBuffer(buffer), mIgba(igba) {
+ std::shared_ptr<C2IGBA> &igba) : mOwned(true), mBuffer(buffer), mIgba(igba) {
CHECK(mBuffer);
AHardwareBuffer_acquire(const_cast<AHardwareBuffer *>(mBuffer));
}
@@ -115,6 +116,10 @@
mOwned = false;
}
+void C2IgbaBlockPoolData::registerIgba(std::shared_ptr<C2IGBA> &igba) {
+ mIgba = igba;
+}
+
std::shared_ptr<C2GraphicBlock> _C2BlockFactory::CreateGraphicBlock(AHardwareBuffer *ahwb) {
// TODO: get proper allocator? and synchronization? or allocator-less?
static std::shared_ptr<C2AllocatorAhwb> sAllocator = std::make_shared<C2AllocatorAhwb>(0);
@@ -148,22 +153,30 @@
}
}
+void _C2BlockFactory::RegisterIgba(
+ const std::shared_ptr<_C2BlockPoolData>& data,
+ std::shared_ptr<C2IGBA> &igba) {
+ if (data && data->getType() == _C2BlockPoolData::TYPE_AHWBUFFER) {
+ const std::shared_ptr<C2IgbaBlockPoolData> poolData =
+ std::static_pointer_cast<C2IgbaBlockPoolData>(data);
+ poolData->registerIgba(igba);
+ }
+}
+
C2IgbaBlockPool::C2IgbaBlockPool(
const std::shared_ptr<C2Allocator> &allocator,
const std::shared_ptr<C2IGBA> &igba,
+ ::android::base::unique_fd &&ufd,
const local_id_t localId) : mAllocator(allocator), mIgba(igba), mLocalId(localId) {
if (!mIgba) {
mValid = false;
return;
}
- // TODO: Remove IPC (This is a nested IPC call during c2aidl creatBlockPool().
- ::ndk::ScopedFileDescriptor fd;
- ::ndk::ScopedAStatus status = mIgba->getWaitableFd(&fd);
- if (!status.isOk()) {
+ if (ufd.get() < 0) {
mValid = false;
return;
}
- mWaitFence = _C2FenceFactory::CreatePipeFence(fd.release());
+ mWaitFence = _C2FenceFactory::CreatePipeFence(std::move(ufd));
if (!mWaitFence.valid()) {
mValid = false;
return;
diff --git a/media/libaaudio/fuzzer/Android.bp b/media/libaaudio/fuzzer/Android.bp
index ee53717..46c4148 100644
--- a/media/libaaudio/fuzzer/Android.bp
+++ b/media/libaaudio/fuzzer/Android.bp
@@ -41,6 +41,7 @@
"libaudioclient_aidl_conversion",
"libaudio_aidl_conversion_common_cpp",
"libutils",
+ "com.android.media.aaudio-aconfig-cc",
],
static_libs: [
"liblog",
diff --git a/media/libaaudio/src/Android.bp b/media/libaaudio/src/Android.bp
index 7882951..886603d 100644
--- a/media/libaaudio/src/Android.bp
+++ b/media/libaaudio/src/Android.bp
@@ -173,6 +173,7 @@
"aaudio-aidl-cpp",
"audioclient-types-aidl-cpp",
"libaudioclient_aidl_conversion",
+ "com.android.media.aaudio-aconfig-cc",
],
cflags: [
diff --git a/media/libaaudio/src/client/AudioStreamInternal.cpp b/media/libaaudio/src/client/AudioStreamInternal.cpp
index 8d9bf20..ca0db0d 100644
--- a/media/libaaudio/src/client/AudioStreamInternal.cpp
+++ b/media/libaaudio/src/client/AudioStreamInternal.cpp
@@ -42,6 +42,7 @@
#include "fifo/FifoBuffer.h"
#include "utility/AudioClock.h"
#include <media/AidlConversion.h>
+#include <com_android_media_aaudio.h>
#include "AudioStreamInternal.h"
@@ -63,8 +64,6 @@
#define LOG_TIMESTAMPS 0
-#define ENABLE_SAMPLE_RATE_CONVERTER 1
-
AudioStreamInternal::AudioStreamInternal(AAudioServiceInterface &serviceInterface, bool inService)
: AudioStream()
, mClockModel()
@@ -193,11 +192,13 @@
setSampleRate(configurationOutput.getSampleRate());
}
-#if !ENABLE_SAMPLE_RATE_CONVERTER
- if (getSampleRate() != getDeviceSampleRate()) {
- goto error;
+ if (!com::android::media::aaudio::sample_rate_conversion()) {
+ if (getSampleRate() != getDeviceSampleRate()) {
+ ALOGD("%s - skipping sample rate converter. SR = %d, Device SR = %d", __func__,
+ getSampleRate(), getDeviceSampleRate());
+ goto error;
+ }
}
-#endif
// Save device format so we can do format conversion and volume scaling together.
setDeviceFormat(configurationOutput.getFormat());
diff --git a/media/libaudioclient/AudioEffect.cpp b/media/libaudioclient/AudioEffect.cpp
index 2870c4c..a7adfbd 100644
--- a/media/libaudioclient/AudioEffect.cpp
+++ b/media/libaudioclient/AudioEffect.cpp
@@ -366,14 +366,15 @@
return mStatus;
}
+ std::unique_lock ul(mLock, std::defer_lock);
if (cmdCode == EFFECT_CMD_ENABLE || cmdCode == EFFECT_CMD_DISABLE) {
+ ul.lock();
if (mEnabled == (cmdCode == EFFECT_CMD_ENABLE)) {
return NO_ERROR;
}
if (replySize == nullptr || *replySize != sizeof(status_t) || replyData == nullptr) {
return BAD_VALUE;
}
- mLock.lock();
}
std::vector<uint8_t> data;
@@ -398,7 +399,6 @@
if (status == NO_ERROR) {
mEnabled = (cmdCode == EFFECT_CMD_ENABLE);
}
- mLock.unlock();
}
return status;
diff --git a/media/libaudioclient/tests/Android.bp b/media/libaudioclient/tests/Android.bp
index 5b90158..f72ac89 100644
--- a/media/libaudioclient/tests/Android.bp
+++ b/media/libaudioclient/tests/Android.bp
@@ -104,6 +104,7 @@
shared_libs: [
"capture_state_listener-aidl-cpp",
"framework-permission-aidl-cpp",
+ "libaudioutils",
"libbase",
"libcgrouprc",
"libdl",
@@ -130,7 +131,6 @@
"libaudiofoundation",
"libaudiomanager",
"libaudiopolicy",
- "libaudioutils",
],
data: ["bbb*.raw"],
test_config_template: "audio_test_template.xml",
diff --git a/media/libaudioclient/tests/audio_aidl_legacy_conversion_tests.cpp b/media/libaudioclient/tests/audio_aidl_legacy_conversion_tests.cpp
index dc37785..0be1d7e 100644
--- a/media/libaudioclient/tests/audio_aidl_legacy_conversion_tests.cpp
+++ b/media/libaudioclient/tests/audio_aidl_legacy_conversion_tests.cpp
@@ -43,7 +43,9 @@
using media::audio::common::AudioGain;
using media::audio::common::AudioGainConfig;
using media::audio::common::AudioGainMode;
+using media::audio::common::AudioInputFlags;
using media::audio::common::AudioIoFlags;
+using media::audio::common::AudioOutputFlags;
using media::audio::common::AudioPortDeviceExt;
using media::audio::common::AudioProfile;
using media::audio::common::AudioStandard;
@@ -851,3 +853,27 @@
EXPECT_EQ(MicrophoneDynamicInfo::ChannelMapping::UNUSED, aidl.dynamic.channelMapping[2]);
EXPECT_EQ(MicrophoneDynamicInfo::ChannelMapping::PROCESSED, aidl.dynamic.channelMapping[3]);
}
+
+TEST(AudioInputFlags, Aidl2Legacy2Aidl) {
+ for (auto flag : enum_range<AudioInputFlags>()) {
+ int32_t aidlMask = 1 << static_cast<int32_t>(flag);
+ auto convMask = aidl2legacy_int32_t_audio_input_flags_t_mask(aidlMask);
+ ASSERT_TRUE(convMask.ok());
+ ASSERT_EQ(1, __builtin_popcount(convMask.value()));
+ auto convFlag = legacy2aidl_audio_input_flags_t_AudioInputFlags(convMask.value());
+ ASSERT_TRUE(convFlag.ok());
+ EXPECT_EQ(flag, convFlag.value());
+ }
+}
+
+TEST(AudioOutputFlags, Aidl2Legacy2Aidl) {
+ for (auto flag : enum_range<AudioOutputFlags>()) {
+ int32_t aidlMask = 1 << static_cast<int32_t>(flag);
+ auto convMask = aidl2legacy_int32_t_audio_output_flags_t_mask(aidlMask);
+ ASSERT_TRUE(convMask.ok());
+ ASSERT_EQ(1, __builtin_popcount(convMask.value()));
+ auto convFlag = legacy2aidl_audio_output_flags_t_AudioOutputFlags(convMask.value());
+ ASSERT_TRUE(convFlag.ok());
+ EXPECT_EQ(flag, convFlag.value());
+ }
+}
diff --git a/media/libaudiohal/impl/DeviceHalAidl.cpp b/media/libaudiohal/impl/DeviceHalAidl.cpp
index a58ce1b..6a6557c 100644
--- a/media/libaudiohal/impl/DeviceHalAidl.cpp
+++ b/media/libaudiohal/impl/DeviceHalAidl.cpp
@@ -267,6 +267,9 @@
if (status_t status = filterAndRetrieveBtA2dpParameters(parameterKeys, &result); status != OK) {
ALOGW("%s: filtering or retrieving BT A2DP parameters failed: %d", __func__, status);
}
+ if (status_t status = filterAndRetrieveBtLeParameters(parameterKeys, &result); status != OK) {
+ ALOGW("%s: filtering or retrieving BT LE parameters failed: %d", __func__, status);
+ }
*values = result.toString();
return parseAndGetVendorParameters(mVendorExt, mModule, parameterKeys, values);
}
@@ -988,6 +991,23 @@
return OK;
}
+status_t DeviceHalAidl::filterAndRetrieveBtLeParameters(
+ AudioParameter &keys, AudioParameter *result) {
+ if (String8 key = String8(AudioParameter::keyReconfigLeSupported); keys.containsKey(key)) {
+ keys.remove(key);
+ if (mBluetoothLe != nullptr) {
+ bool supports;
+ RETURN_STATUS_IF_ERROR(statusTFromBinderStatus(
+ mBluetoothLe->supportsOffloadReconfiguration(&supports)));
+ result->addInt(key, supports ? 1 : 0);
+ } else {
+ ALOGI("%s: no mBluetoothLe on %s", __func__, mInstance.c_str());
+ result->addInt(key, 0);
+ }
+ }
+ return OK;
+}
+
status_t DeviceHalAidl::filterAndUpdateBtA2dpParameters(AudioParameter ¶meters) {
std::optional<bool> a2dpEnabled;
std::optional<std::vector<VendorParameter>> reconfigureOffload;
@@ -1069,6 +1089,7 @@
status_t DeviceHalAidl::filterAndUpdateBtLeParameters(AudioParameter ¶meters) {
std::optional<bool> leEnabled;
+ std::optional<std::vector<VendorParameter>> reconfigureOffload;
(void)VALUE_OR_RETURN_STATUS(filterOutAndProcessParameter<String8>(
parameters, String8(AudioParameter::keyBtLeSuspended),
[&leEnabled](const String8& trueOrFalse) {
@@ -1083,9 +1104,27 @@
AudioParameter::keyBtLeSuspended, trueOrFalse.c_str());
return BAD_VALUE;
}));
+ (void)VALUE_OR_RETURN_STATUS(filterOutAndProcessParameter<String8>(
+ parameters, String8(AudioParameter::keyReconfigLe),
+ [&](const String8& value) -> status_t {
+ if (mVendorExt != nullptr) {
+ std::vector<VendorParameter> result;
+ RETURN_STATUS_IF_ERROR(statusTFromBinderStatus(
+ mVendorExt->parseBluetoothLeReconfigureOffload(
+ std::string(value.c_str()), &result)));
+ reconfigureOffload = std::move(result);
+ } else {
+ reconfigureOffload = std::vector<VendorParameter>();
+ }
+ return OK;
+ }));
if (mBluetoothLe != nullptr && leEnabled.has_value()) {
return statusTFromBinderStatus(mBluetoothLe->setEnabled(leEnabled.value()));
}
+ if (mBluetoothLe != nullptr && reconfigureOffload.has_value()) {
+ return statusTFromBinderStatus(mBluetoothLe->reconfigureOffload(
+ reconfigureOffload.value()));
+ }
return OK;
}
diff --git a/media/libaudiohal/impl/DeviceHalAidl.h b/media/libaudiohal/impl/DeviceHalAidl.h
index 9493e47..f705db7 100644
--- a/media/libaudiohal/impl/DeviceHalAidl.h
+++ b/media/libaudiohal/impl/DeviceHalAidl.h
@@ -207,6 +207,7 @@
~DeviceHalAidl() override = default;
status_t filterAndRetrieveBtA2dpParameters(AudioParameter &keys, AudioParameter *result);
+ status_t filterAndRetrieveBtLeParameters(AudioParameter &keys, AudioParameter *result);
status_t filterAndUpdateBtA2dpParameters(AudioParameter ¶meters);
status_t filterAndUpdateBtHfpParameters(AudioParameter ¶meters);
status_t filterAndUpdateBtLeParameters(AudioParameter ¶meters);
diff --git a/media/libaudiohal/impl/Hal2AidlMapper.cpp b/media/libaudiohal/impl/Hal2AidlMapper.cpp
index f187057..47fcd27 100644
--- a/media/libaudiohal/impl/Hal2AidlMapper.cpp
+++ b/media/libaudiohal/impl/Hal2AidlMapper.cpp
@@ -82,6 +82,16 @@
}
}
+bool containHapticChannel(AudioChannelLayout channel) {
+ return channel.getTag() == AudioChannelLayout::Tag::layoutMask &&
+ ((channel.get<AudioChannelLayout::Tag::layoutMask>()
+ & AudioChannelLayout::CHANNEL_HAPTIC_A)
+ == AudioChannelLayout::CHANNEL_HAPTIC_A ||
+ (channel.get<AudioChannelLayout::Tag::layoutMask>()
+ & AudioChannelLayout::CHANNEL_HAPTIC_B)
+ == AudioChannelLayout::CHANNEL_HAPTIC_B);
+}
+
} // namespace
Hal2AidlMapper::Hal2AidlMapper(const std::string& instance, const std::shared_ptr<IModule>& module)
@@ -438,11 +448,20 @@
Hal2AidlMapper::Ports::iterator Hal2AidlMapper::findPort(
const AudioConfig& config, const AudioIoFlags& flags,
const std::set<int32_t>& destinationPortIds) {
- auto belongsToProfile = [&config](const AudioProfile& prof) {
+ auto channelMaskMatches = [](const std::vector<AudioChannelLayout>& channelMasks,
+ const AudioChannelLayout& channelMask) {
+ // Return true when 1) the channel mask is none and none of the channel mask from the
+ // collection contains haptic channel mask, or 2) the channel mask collection contains
+ // the queried channel mask.
+ return (channelMask.getTag() == AudioChannelLayout::none &&
+ std::none_of(channelMasks.begin(), channelMasks.end(),
+ containHapticChannel)) ||
+ std::find(channelMasks.begin(), channelMasks.end(), channelMask)
+ != channelMasks.end();
+ };
+ auto belongsToProfile = [&config, &channelMaskMatches](const AudioProfile& prof) {
return (isDefaultAudioFormat(config.base.format) || prof.format == config.base.format) &&
- (config.base.channelMask.getTag() == AudioChannelLayout::none ||
- std::find(prof.channelMasks.begin(), prof.channelMasks.end(),
- config.base.channelMask) != prof.channelMasks.end()) &&
+ channelMaskMatches(prof.channelMasks, config.base.channelMask) &&
(config.base.sampleRate == 0 ||
std::find(prof.sampleRates.begin(), prof.sampleRates.end(),
config.base.sampleRate) != prof.sampleRates.end());
diff --git a/media/libaudiohal/impl/effectsAidlConversion/AidlConversionVisualizer.cpp b/media/libaudiohal/impl/effectsAidlConversion/AidlConversionVisualizer.cpp
index 18d0d95..e4ec2ba 100644
--- a/media/libaudiohal/impl/effectsAidlConversion/AidlConversionVisualizer.cpp
+++ b/media/libaudiohal/impl/effectsAidlConversion/AidlConversionVisualizer.cpp
@@ -169,8 +169,8 @@
const auto& measure = VALUE_OR_RETURN_STATUS(GET_PARAMETER_SPECIFIC_FIELD(
aidlParam, Visualizer, visualizer, Visualizer::measurement, Visualizer::Measurement));
int32_t* reply = (int32_t *) pReplyData;
- *reply++ = measure.rms;
- *reply = measure.peak;
+ *reply++ = measure.peak;
+ *reply = measure.rms;
return OK;
}
diff --git a/media/libeffects/downmix/aidl/DownmixContext.cpp b/media/libeffects/downmix/aidl/DownmixContext.cpp
index ac893d8..0e76d1d 100644
--- a/media/libeffects/downmix/aidl/DownmixContext.cpp
+++ b/media/libeffects/downmix/aidl/DownmixContext.cpp
@@ -66,8 +66,8 @@
LOG(DEBUG) << __func__ << " in " << in << " out " << out << " sample " << samples;
IEffect::Status status = {EX_ILLEGAL_ARGUMENT, 0, 0};
- if (in == nullptr || out == nullptr || getInputFrameSize() != getOutputFrameSize() ||
- getInputFrameSize() == 0) {
+ if (in == nullptr || out == nullptr ||
+ getCommon().input.frameCount != getCommon().output.frameCount || getInputFrameSize() == 0) {
return status;
}
@@ -112,7 +112,7 @@
void DownmixContext::init_params(const Parameter::Common& common) {
// when configuring the effect, do not allow a blank or unsupported channel mask
AudioChannelLayout channelMask = common.input.base.channelMask;
- if (isChannelMaskValid(channelMask)) {
+ if (!isChannelMaskValid(channelMask)) {
LOG(ERROR) << "Downmix_Configure error: input channel mask " << channelMask.toString()
<< " not supported";
} else {
@@ -123,7 +123,7 @@
}
bool DownmixContext::isChannelMaskValid(AudioChannelLayout channelMask) {
- if (channelMask.getTag() == AudioChannelLayout::layoutMask) return false;
+ if (channelMask.getTag() != AudioChannelLayout::layoutMask) return false;
int chMask = channelMask.get<AudioChannelLayout::layoutMask>();
// check against unsupported channels (up to FCC_26)
constexpr uint32_t MAXIMUM_CHANNEL_MASK = AudioChannelLayout::LAYOUT_22POINT2 |
diff --git a/media/libeffects/dynamicsproc/aidl/DynamicsProcessing.cpp b/media/libeffects/dynamicsproc/aidl/DynamicsProcessing.cpp
index 63cb48d..85ea53a 100644
--- a/media/libeffects/dynamicsproc/aidl/DynamicsProcessing.cpp
+++ b/media/libeffects/dynamicsproc/aidl/DynamicsProcessing.cpp
@@ -105,14 +105,14 @@
DynamicsProcessing::EqBandConfig({.channel = 0,
.band = 0,
.enable = false,
- .cutoffFrequencyHz = 20,
+ .cutoffFrequencyHz = 0,
.gainDb = -200});
static const DynamicsProcessing::EqBandConfig kEqBandConfigMax =
DynamicsProcessing::EqBandConfig({.channel = std::numeric_limits<int>::max(),
.band = std::numeric_limits<int>::max(),
.enable = true,
- .cutoffFrequencyHz = 20000.1,
+ .cutoffFrequencyHz = 192000,
.gainDb = 200});
static const Range::DynamicsProcessingRange kPreEqBandConfigRange = {
@@ -129,7 +129,7 @@
{.channel = 0,
.band = 0,
.enable = false,
- .cutoffFrequencyHz = 20,
+ .cutoffFrequencyHz = 0,
.attackTimeMs = 0,
.releaseTimeMs = 0,
.ratio = 1,
@@ -144,7 +144,7 @@
{.channel = std::numeric_limits<int>::max(),
.band = std::numeric_limits<int>::max(),
.enable = true,
- .cutoffFrequencyHz = 20000.1,
+ .cutoffFrequencyHz = 192000,
.attackTimeMs = 60000,
.releaseTimeMs = 60000,
.ratio = 50,
@@ -443,7 +443,7 @@
IEffect::Status DynamicsProcessingImpl::effectProcessImpl(float* in, float* out, int samples) {
IEffect::Status status = {EX_NULL_POINTER, 0, 0};
RETURN_VALUE_IF(!mContext, status, "nullContext");
- return mContext->lvmProcess(in, out, samples);
+ return mContext->dpeProcess(in, out, samples);
}
} // namespace aidl::android::hardware::audio::effect
diff --git a/media/libeffects/dynamicsproc/aidl/DynamicsProcessingContext.cpp b/media/libeffects/dynamicsproc/aidl/DynamicsProcessingContext.cpp
index 57c873b..e5e5368 100644
--- a/media/libeffects/dynamicsproc/aidl/DynamicsProcessingContext.cpp
+++ b/media/libeffects/dynamicsproc/aidl/DynamicsProcessingContext.cpp
@@ -19,6 +19,7 @@
#include "DynamicsProcessingContext.h"
#include "DynamicsProcessing.h"
+#include <audio_utils/power.h>
#include <sys/param.h>
#include <functional>
#include <unordered_set>
@@ -68,6 +69,23 @@
return RetCode::SUCCESS;
}
+RetCode DynamicsProcessingContext::setVolumeStereo(const Parameter::VolumeStereo& volumeStereo) {
+ std::lock_guard lg(mMutex);
+ dp_fx::DPChannel* leftChannel = mDpFreq->getChannel(0);
+ dp_fx::DPChannel* rightChannel = mDpFreq->getChannel(1);
+ if (leftChannel != nullptr) {
+ leftChannel->setOutputGain(audio_utils_power_from_amplitude(volumeStereo.left));
+ }
+ if (rightChannel != nullptr) {
+ rightChannel->setOutputGain(audio_utils_power_from_amplitude(volumeStereo.right));
+ }
+ return RetCode::SUCCESS;
+}
+
+Parameter::VolumeStereo DynamicsProcessingContext::getVolumeStereo() {
+ return {1.0f, 1.0f};
+}
+
void DynamicsProcessingContext::dpSetFreqDomainVariant_l(
const DynamicsProcessing::EngineArchitecture& engine) {
mDpFreq.reset(new dp_fx::DPFrequency());
@@ -273,7 +291,7 @@
return ret;
}
-IEffect::Status DynamicsProcessingContext::lvmProcess(float* in, float* out, int samples) {
+IEffect::Status DynamicsProcessingContext::dpeProcess(float* in, float* out, int samples) {
LOG(DEBUG) << __func__ << " in " << in << " out " << out << " sample " << samples;
IEffect::Status status = {EX_NULL_POINTER, 0, 0};
@@ -460,6 +478,11 @@
RetCode ret = RetCode::SUCCESS;
std::unordered_set<int> channelSet;
+ if (!stageInUse) {
+ LOG(WARNING) << __func__ << " not in use " << ::android::internal::ToString(channels);
+ return RetCode::SUCCESS;
+ }
+
RETURN_VALUE_IF(!stageInUse, RetCode::ERROR_ILLEGAL_PARAMETER, "stageNotInUse");
for (auto& it : channels) {
if (0 != channelSet.count(it.channel)) {
diff --git a/media/libeffects/dynamicsproc/aidl/DynamicsProcessingContext.h b/media/libeffects/dynamicsproc/aidl/DynamicsProcessingContext.h
index b8539f6..ced7f19 100644
--- a/media/libeffects/dynamicsproc/aidl/DynamicsProcessingContext.h
+++ b/media/libeffects/dynamicsproc/aidl/DynamicsProcessingContext.h
@@ -45,6 +45,8 @@
// override EffectContext::setCommon to update mChannelCount
RetCode setCommon(const Parameter::Common& common) override;
+ RetCode setVolumeStereo(const Parameter::VolumeStereo& volumeStereo) override;
+ Parameter::VolumeStereo getVolumeStereo() override;
RetCode setEngineArchitecture(const DynamicsProcessing::EngineArchitecture& engineArchitecture);
RetCode setPreEq(const std::vector<DynamicsProcessing::ChannelConfig>& eqChannels);
@@ -66,7 +68,7 @@
std::vector<DynamicsProcessing::LimiterConfig> getLimiter();
std::vector<DynamicsProcessing::InputGain> getInputGain();
- IEffect::Status lvmProcess(float* in, float* out, int samples);
+ IEffect::Status dpeProcess(float* in, float* out, int samples);
private:
static constexpr float kPreferredProcessingDurationMs = 10.0f;
diff --git a/media/libeffects/hapticgenerator/EffectHapticGenerator.cpp b/media/libeffects/hapticgenerator/EffectHapticGenerator.cpp
index d8cf20e..e89e501 100644
--- a/media/libeffects/hapticgenerator/EffectHapticGenerator.cpp
+++ b/media/libeffects/hapticgenerator/EffectHapticGenerator.cpp
@@ -28,12 +28,12 @@
#include <errno.h>
#include <inttypes.h>
-#include <math.h>
#include <android-base/parsedouble.h>
#include <android-base/properties.h>
#include <audio_effects/effect_hapticgenerator.h>
#include <audio_utils/format.h>
+#include <audio_utils/safe_math.h>
#include <system/audio.h>
static constexpr float DEFAULT_RESONANT_FREQUENCY = 150.0f;
@@ -338,8 +338,9 @@
const float qFactor = *((float *) value + 1);
const float maxAmplitude = *((float *) value + 2);
context->param.resonantFrequency =
- isnan(resonantFrequency) ? DEFAULT_RESONANT_FREQUENCY : resonantFrequency;
- context->param.bsfZeroQ = isnan(qFactor) ? DEFAULT_BSF_POLE_Q : qFactor;
+ audio_utils::safe_isnan(resonantFrequency) ? DEFAULT_RESONANT_FREQUENCY
+ : resonantFrequency;
+ context->param.bsfZeroQ = audio_utils::safe_isnan(qFactor) ? DEFAULT_BSF_ZERO_Q : qFactor;
context->param.bsfPoleQ = context->param.bsfZeroQ / 2.0f;
context->param.maxHapticAmplitude = maxAmplitude;
ALOGD("Updating vibrator info, resonantFrequency=%f, bsfZeroQ=%f, bsfPoleQ=%f, "
diff --git a/media/libeffects/lvm/wrapper/Aidl/BundleContext.h b/media/libeffects/lvm/wrapper/Aidl/BundleContext.h
index 62bb6e4..3f31b8b 100644
--- a/media/libeffects/lvm/wrapper/Aidl/BundleContext.h
+++ b/media/libeffects/lvm/wrapper/Aidl/BundleContext.h
@@ -98,7 +98,7 @@
const Virtualizer::SpeakerAnglesPayload payload);
RetCode setVolumeStereo(const Parameter::VolumeStereo& volumeStereo) override;
- Parameter::VolumeStereo getVolumeStereo() override { return mVolumeStereo; }
+ Parameter::VolumeStereo getVolumeStereo() override { return {1.0f, 1.0f}; }
IEffect::Status lvmProcess(float* in, float* out, int samples);
diff --git a/media/libeffects/lvm/wrapper/Reverb/aidl/ReverbContext.h b/media/libeffects/lvm/wrapper/Reverb/aidl/ReverbContext.h
index 9bb0b1a..d11a081 100644
--- a/media/libeffects/lvm/wrapper/Reverb/aidl/ReverbContext.h
+++ b/media/libeffects/lvm/wrapper/Reverb/aidl/ReverbContext.h
@@ -81,7 +81,12 @@
bool getEnvironmentalReverbBypass() const { return mBypass; }
RetCode setVolumeStereo(const Parameter::VolumeStereo& volumeStereo) override;
- Parameter::VolumeStereo getVolumeStereo() override { return mVolumeStereo; }
+ Parameter::VolumeStereo getVolumeStereo() override {
+ if (isAuxiliary()) {
+ return mVolumeStereo;
+ }
+ return {1.0f, 1.0f};
+ }
RetCode setReflectionsDelay(int delay) {
mReflectionsDelayMs = delay;
diff --git a/media/libeffects/visualizer/aidl/Visualizer.cpp b/media/libeffects/visualizer/aidl/Visualizer.cpp
index 53bfb41..0303842 100644
--- a/media/libeffects/visualizer/aidl/Visualizer.cpp
+++ b/media/libeffects/visualizer/aidl/Visualizer.cpp
@@ -73,7 +73,7 @@
.proxy = std::nullopt},
.flags = {.type = Flags::Type::INSERT,
.insert = Flags::Insert::LAST,
- .volume = Flags::Volume::CTRL},
+ .volume = Flags::Volume::NONE},
.name = VisualizerImpl::kEffectName,
.implementor = "The Android Open Source Project"},
.capability = VisualizerImpl::kCapability};
diff --git a/media/libeffects/visualizer/aidl/VisualizerContext.cpp b/media/libeffects/visualizer/aidl/VisualizerContext.cpp
index a1726ad..5d2bb3a 100644
--- a/media/libeffects/visualizer/aidl/VisualizerContext.cpp
+++ b/media/libeffects/visualizer/aidl/VisualizerContext.cpp
@@ -61,6 +61,7 @@
#endif
mChannelCount = channelCount;
mCommon = common;
+ std::fill(mCaptureBuf.begin(), mCaptureBuf.end(), 0x80);
return RetCode::SUCCESS;
}
@@ -84,7 +85,7 @@
void VisualizerContext::reset() {
std::lock_guard lg(mMutex);
- std::fill_n(mCaptureBuf.begin(), kMaxCaptureBufSize, 0x80);
+ std::fill(mCaptureBuf.begin(), mCaptureBuf.end(), 0x80);
}
RetCode VisualizerContext::setCaptureSamples(int samples) {
@@ -190,13 +191,12 @@
}
std::vector<uint8_t> VisualizerContext::capture() {
- std::vector<uint8_t> result;
std::lock_guard lg(mMutex);
+ uint32_t captureSamples = mCaptureSamples;
+ std::vector<uint8_t> result(captureSamples, 0x80);
// cts android.media.audio.cts.VisualizerTest expecting silence data when effect not running
// RETURN_VALUE_IF(mState != State::ACTIVE, result, "illegalState");
if (mState != State::ACTIVE) {
- result.resize(mCaptureSamples);
- memset(result.data(), 0x80, mCaptureSamples);
return result;
}
@@ -214,7 +214,7 @@
if (latencyMs < 0) {
latencyMs = 0;
}
- uint32_t deltaSamples = mCaptureSamples + mCommon.input.base.sampleRate * latencyMs / 1000;
+ uint32_t deltaSamples = captureSamples + mCommon.input.base.sampleRate * latencyMs / 1000;
// large sample rate, latency, or capture size, could cause overflow.
// do not offset more than the size of buffer.
@@ -223,22 +223,22 @@
deltaSamples = kMaxCaptureBufSize;
}
- int32_t capturePoint, captureSamples = mCaptureSamples;
+ int32_t capturePoint;
__builtin_sub_overflow((int32_t) mCaptureIdx, deltaSamples, &capturePoint);
// a negative capturePoint means we wrap the buffer.
if (capturePoint < 0) {
uint32_t size = -capturePoint;
- if (size > mCaptureSamples) {
- size = mCaptureSamples;
+ if (size > captureSamples) {
+ size = captureSamples;
}
- // first part of two stages copy, capture to the end of buffer and reset the size/point
- result.insert(result.end(), &mCaptureBuf[kMaxCaptureBufSize + capturePoint],
- &mCaptureBuf[kMaxCaptureBufSize + capturePoint + size]);
+ std::copy(std::begin(mCaptureBuf) + kMaxCaptureBufSize - size,
+ std::begin(mCaptureBuf) + kMaxCaptureBufSize, result.begin());
captureSamples -= size;
capturePoint = 0;
}
- result.insert(result.end(), &mCaptureBuf[capturePoint],
- &mCaptureBuf[capturePoint + captureSamples]);
+ std::copy(std::begin(mCaptureBuf) + capturePoint,
+ std::begin(mCaptureBuf) + capturePoint + captureSamples,
+ result.begin() + mCaptureSamples - captureSamples);
mLastCaptureIdx = mCaptureIdx;
return result;
}
@@ -256,16 +256,15 @@
// find the peak and RMS squared for the new buffer
float rmsSqAcc = 0;
float maxSample = 0.f;
- for (size_t inIdx = 0; inIdx < (unsigned)samples; ++inIdx) {
+ for (size_t inIdx = 0; inIdx < (unsigned) samples; ++inIdx) {
maxSample = fmax(maxSample, fabs(in[inIdx]));
rmsSqAcc += in[inIdx] * in[inIdx];
}
maxSample *= 1 << 15; // scale to int16_t, with exactly 1 << 15 representing positive num.
rmsSqAcc *= 1 << 30; // scale to int16_t * 2
- mPastMeasurements[mMeasurementBufferIdx] = {
- .mPeakU16 = (uint16_t)maxSample,
- .mRmsSquared = rmsSqAcc / samples,
- .mIsValid = true };
+ mPastMeasurements[mMeasurementBufferIdx] = {.mIsValid = true,
+ .mPeakU16 = (uint16_t)maxSample,
+ .mRmsSquared = rmsSqAcc / samples};
if (++mMeasurementBufferIdx >= mMeasurementWindowSizeInBuffers) {
mMeasurementBufferIdx = 0;
}
diff --git a/media/libmediahelper/AudioParameter.cpp b/media/libmediahelper/AudioParameter.cpp
index f21ea53..e921bd2 100644
--- a/media/libmediahelper/AudioParameter.cpp
+++ b/media/libmediahelper/AudioParameter.cpp
@@ -75,6 +75,8 @@
const char * const AudioParameter::keyReconfigA2dp = AUDIO_PARAMETER_RECONFIG_A2DP;
const char * const AudioParameter::keyReconfigA2dpSupported = AUDIO_PARAMETER_A2DP_RECONFIG_SUPPORTED;
const char * const AudioParameter::keyBtLeSuspended = AUDIO_PARAMETER_KEY_BT_LE_SUSPENDED;
+const char * const AudioParameter::keyReconfigLe = AUDIO_PARAMETER_RECONFIG_LE;
+const char * const AudioParameter::keyReconfigLeSupported = AUDIO_PARAMETER_LE_RECONFIG_SUPPORTED;
// const char * const AudioParameter::keyDeviceSupportedEncapsulationModes =
// AUDIO_PARAMETER_DEVICE_SUP_ENCAPSULATION_MODES;
// const char * const AudioParameter::keyDeviceSupportedEncapsulationMetadataTypes =
diff --git a/media/libmediahelper/include/media/AudioParameter.h b/media/libmediahelper/include/media/AudioParameter.h
index 21d5117..61e6bcc 100644
--- a/media/libmediahelper/include/media/AudioParameter.h
+++ b/media/libmediahelper/include/media/AudioParameter.h
@@ -121,10 +121,14 @@
// keyReconfigA2dp: Ask HwModule to reconfigure A2DP offloaded codec
// keyReconfigA2dpSupported: Query if HwModule supports A2DP offload codec config
// keyBtLeSuspended: 'true' or 'false'
+ // keyReconfigLe: Ask HwModule to reconfigure LE offloaded codec
+ // keyReconfigLeSupported: Query if HwModule supports LE offload codec config
static const char * const keyBtA2dpSuspended;
static const char * const keyReconfigA2dp;
static const char * const keyReconfigA2dpSupported;
static const char * const keyBtLeSuspended;
+ static const char * const keyReconfigLe;
+ static const char * const keyReconfigLeSupported;
// For querying device supported encapsulation capabilities. All returned values are integer,
// which are bit fields composed from using encapsulation capability values as position bits.
diff --git a/media/libmediaplayerservice/tests/DrmSessionManager_test.cpp b/media/libmediaplayerservice/tests/DrmSessionManager_test.cpp
index 30f6a91..6da8e31 100644
--- a/media/libmediaplayerservice/tests/DrmSessionManager_test.cpp
+++ b/media/libmediaplayerservice/tests/DrmSessionManager_test.cpp
@@ -148,8 +148,8 @@
class DrmSessionManagerTest : public ::testing::Test {
public:
DrmSessionManagerTest()
- : mService(::ndk::SharedRefBase::make<ResourceManagerService>
- (new FakeProcessInfo(), new FakeSystemCallback())),
+ : mService(ResourceManagerService::Create(
+ new FakeProcessInfo(), new FakeSystemCallback())),
mDrmSessionManager(new DrmSessionManager(mService)),
mTestDrm1(::ndk::SharedRefBase::make<FakeDrm>(
kTestSessionId1, mDrmSessionManager)),
diff --git a/media/libshmem/ShmemCompat.cpp b/media/libshmem/ShmemCompat.cpp
index 246cb24..4200c2e 100644
--- a/media/libshmem/ShmemCompat.cpp
+++ b/media/libshmem/ShmemCompat.cpp
@@ -84,11 +84,11 @@
return false;
}
- const int fd = fcntl(heap->getHeapID(), F_DUPFD_CLOEXEC, 0);
- if (fd < 0) {
+ base::unique_fd fd(fcntl(heap->getHeapID(), F_DUPFD_CLOEXEC, 0));
+ if (!fd.ok()) {
return false;
}
- result->fd.reset(base::unique_fd(fd));
+ result->fd.reset(std::move(fd));
result->size = size;
result->offset = heap->getOffset() + offset;
result->writeable = (heap->getFlags() & IMemoryHeap::READ_ONLY) == 0;
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index cf29a25..2145dd9 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -670,7 +670,7 @@
msg->post();
}
-status_t ACodec::setSurface(const sp<Surface> &surface) {
+status_t ACodec::setSurface(const sp<Surface> &surface, uint32_t /*generation*/) {
sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
msg->setObject("surface", surface);
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 0285650..ee66622 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -822,6 +822,37 @@
const sp<AMessage> mNotify;
};
+class OnBufferReleasedListener : public ::android::BnProducerListener{
+private:
+ uint32_t mGeneration;
+ std::weak_ptr<BufferChannelBase> mBufferChannel;
+
+ void notifyBufferReleased() {
+ auto p = mBufferChannel.lock();
+ if (p) {
+ p->onBufferReleasedFromOutputSurface(mGeneration);
+ }
+ }
+
+public:
+ explicit OnBufferReleasedListener(
+ uint32_t generation,
+ const std::shared_ptr<BufferChannelBase> &bufferChannel)
+ : mGeneration(generation), mBufferChannel(bufferChannel) {}
+
+ virtual ~OnBufferReleasedListener() = default;
+
+ void onBufferReleased() override {
+ notifyBufferReleased();
+ }
+
+ void onBufferDetached([[maybe_unused]] int slot) override {
+ notifyBufferReleased();
+ }
+
+ bool needsReleaseNotify() override { return true; }
+};
+
class BufferCallback : public CodecBase::BufferCallback {
public:
explicit BufferCallback(const sp<AMessage> ¬ify);
@@ -4682,6 +4713,8 @@
PostReplyWithError(replyID, err);
break;
}
+ uint32_t generation = mSurfaceGeneration;
+ format->setInt32("native-window-generation", generation);
} else {
// we are not using surface so this variable is not used, but initialize sensibly anyway
mAllowFrameDroppingBySurface = false;
@@ -4810,7 +4843,8 @@
mErrorLog.log(LOG_TAG, "Unsetting surface is not supported");
err = BAD_VALUE;
} else {
- err = connectToSurface(surface);
+ uint32_t generation;
+ err = connectToSurface(surface, &generation);
if (err == ALREADY_EXISTS) {
// reconnecting to same surface
err = OK;
@@ -4825,12 +4859,13 @@
mSoftRenderer = new SoftwareRenderer(surface);
// TODO: check if this was successful
} else {
- err = mCodec->setSurface(surface);
+ err = mCodec->setSurface(surface, generation);
}
}
if (err == OK) {
(void)disconnectFromSurface();
mSurface = surface;
+ mSurfaceGeneration = generation;
}
mReliabilityContextMetrics.setOutputSurfaceCount++;
}
@@ -5068,15 +5103,17 @@
mReleaseSurface.reset(new ReleaseSurface(usage));
}
if (mSurface != mReleaseSurface->getSurface()) {
- status_t err = connectToSurface(mReleaseSurface->getSurface());
+ uint32_t generation;
+ status_t err = connectToSurface(mReleaseSurface->getSurface(), &generation);
ALOGW_IF(err != OK, "error connecting to release surface: err = %d", err);
if (err == OK && !(mFlags & kFlagUsesSoftwareRenderer)) {
- err = mCodec->setSurface(mReleaseSurface->getSurface());
+ err = mCodec->setSurface(mReleaseSurface->getSurface(), generation);
ALOGW_IF(err != OK, "error setting release surface: err = %d", err);
}
if (err == OK) {
(void)disconnectFromSurface();
mSurface = mReleaseSurface->getSurface();
+ mSurfaceGeneration = generation;
} else {
// We were not able to switch the surface, so force
// synchronous release.
@@ -6346,7 +6383,7 @@
return index;
}
-status_t MediaCodec::connectToSurface(const sp<Surface> &surface) {
+status_t MediaCodec::connectToSurface(const sp<Surface> &surface, uint32_t *generation) {
status_t err = OK;
if (surface != NULL) {
uint64_t oldId, newId;
@@ -6368,21 +6405,26 @@
// number. Rely on the fact that max supported process id by Linux is 2^22.
// PID is never 0 so we don't have to worry that we use the default generation of 0.
// TODO: come up with a unique scheme if other producers also set the generation number.
- static uint32_t mSurfaceGeneration = 0;
- uint32_t generation = (getpid() << 10) | (++mSurfaceGeneration & ((1 << 10) - 1));
- surface->setGenerationNumber(generation);
- ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), generation);
+ static uint32_t sSurfaceGeneration = 0;
+ *generation = (getpid() << 10) | (++sSurfaceGeneration & ((1 << 10) - 1));
+ surface->setGenerationNumber(*generation);
+ ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), *generation);
// HACK: clear any free buffers. Remove when connect will automatically do this.
// This is needed as the consumer may be holding onto stale frames that it can reattach
// to this surface after disconnect/connect, and those free frames would inherit the new
// generation number. Disconnecting after setting a unique generation prevents this.
nativeWindowDisconnect(surface.get(), "connectToSurface(reconnect)");
- err = nativeWindowConnect(surface.get(), "connectToSurface(reconnect)");
+ sp<IProducerListener> listener =
+ new OnBufferReleasedListener(*generation, mBufferChannel);
+ err = surfaceConnectWithListener(
+ surface, listener, "connectToSurface(reconnect-with-listener)");
}
if (err != OK) {
- ALOGE("nativeWindowConnect returned an error: %s (%d)", strerror(-err), err);
+ *generation = 0;
+ ALOGE("nativeWindowConnect/surfaceConnectWithListener returned an error: %s (%d)",
+ strerror(-err), err);
} else {
if (!mAllowFrameDroppingBySurface) {
disableLegacyBufferDropPostQ(surface);
@@ -6408,6 +6450,7 @@
}
// assume disconnected even on error
mSurface.clear();
+ mSurfaceGeneration = 0;
mIsSurfaceToDisplay = false;
}
return err;
@@ -6419,9 +6462,11 @@
(void)disconnectFromSurface();
}
if (surface != NULL) {
- err = connectToSurface(surface);
+ uint32_t generation;
+ err = connectToSurface(surface, &generation);
if (err == OK) {
mSurface = surface;
+ mSurfaceGeneration = generation;
}
}
return err;
diff --git a/media/libstagefright/MediaMuxer.cpp b/media/libstagefright/MediaMuxer.cpp
index 9768f97..aaf7465 100644
--- a/media/libstagefright/MediaMuxer.cpp
+++ b/media/libstagefright/MediaMuxer.cpp
@@ -208,6 +208,9 @@
ALOGE("WriteSampleData() get an NULL buffer.");
return -EINVAL;
}
+ if (!mWriter->isSampleMetadataValid(trackIndex, timeUs)) {
+ return -EINVAL;
+ }
{
/* As MediaMuxer's writeSampleData handles inputs from multiple tracks,
* limited the scope of mMuxerLock to this inner block so that the
diff --git a/media/libstagefright/SurfaceUtils.cpp b/media/libstagefright/SurfaceUtils.cpp
index 827052d..f16e635 100644
--- a/media/libstagefright/SurfaceUtils.cpp
+++ b/media/libstagefright/SurfaceUtils.cpp
@@ -328,6 +328,16 @@
return err;
}
+status_t surfaceConnectWithListener(
+ const sp<Surface> &surface, sp<IProducerListener> listener, const char *reason) {
+ ALOGD("connecting to surface %p, reason %s", surface.get(), reason);
+
+ status_t err = surface->connect(NATIVE_WINDOW_API_MEDIA, listener);
+ ALOGE_IF(err != OK, "Failed to connect from surface %p, err %d", surface.get(), err);
+
+ return err;
+}
+
status_t nativeWindowDisconnect(ANativeWindow *surface, const char *reason) {
ALOGD("disconnecting from surface %p, reason %s", surface, reason);
diff --git a/media/libstagefright/Utils.cpp b/media/libstagefright/Utils.cpp
index 863177d..86741a6 100644
--- a/media/libstagefright/Utils.cpp
+++ b/media/libstagefright/Utils.cpp
@@ -2483,6 +2483,11 @@
bool isStreaming, audio_stream_type_t streamType)
{
audio_offload_info_t info = AUDIO_INFO_INITIALIZER;
+ const char *mime;
+ if (meta != nullptr && meta->findCString(kKeyMIMEType, &mime)
+ && strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_OPUS) == 0) {
+ return false;
+ }
if (OK != getAudioOffloadInfo(meta, hasVideo, isStreaming, streamType, &info)) {
return false;
}
diff --git a/media/libstagefright/VideoRenderQualityTracker.cpp b/media/libstagefright/VideoRenderQualityTracker.cpp
index aca20a4..eb9ac0f 100644
--- a/media/libstagefright/VideoRenderQualityTracker.cpp
+++ b/media/libstagefright/VideoRenderQualityTracker.cpp
@@ -127,6 +127,7 @@
contentFrameRate = FRAME_RATE_UNDETERMINED;
desiredFrameRate = FRAME_RATE_UNDETERMINED;
actualFrameRate = FRAME_RATE_UNDETERMINED;
+ maxContentDroppedAfterPauseMs = 0;
freezeEventCount = 0;
freezeDurationMsHistogram.clear();
freezeDistanceMsHistogram.clear();
@@ -145,6 +146,7 @@
getFlag(maxExpectedContentFrameDurationUs, "max_expected_content_frame_duration_us");
getFlag(frameRateDetectionToleranceUs, "frame_rate_detection_tolerance_us");
getFlag(liveContentFrameDropToleranceUs, "live_content_frame_drop_tolerance_us");
+ getFlag(pauseAudioLatencyUs, "pause_audio_latency_us");
getFlag(freezeDurationMsHistogramBuckets, "freeze_duration_ms_histogram_buckets");
getFlag(freezeDurationMsHistogramToScore, "freeze_duration_ms_histogram_to_score");
getFlag(freezeDistanceMsHistogramBuckets, "freeze_distance_ms_histogram_buckets");
@@ -160,7 +162,6 @@
getFlag(traceTriggerEnabled, "trace_trigger_enabled");
getFlag(traceTriggerThrottleMs, "trace_trigger_throttle_ms");
getFlag(traceMinFreezeDurationMs, "trace_minimum_freeze_duration_ms");
- getFlag(traceMaxFreezeDurationMs, "trace_maximum_freeze_duration_ms");
#undef getFlag
return c;
}
@@ -182,6 +183,9 @@
// because of frame drops for live content, or because the user is seeking.
liveContentFrameDropToleranceUs = 200 * 1000;
+ // After a pause is initiated, audio should likely stop playback within 200ms.
+ pauseAudioLatencyUs = 200 * 1000;
+
// Freeze configuration
freezeDurationMsHistogramBuckets = {1, 20, 40, 60, 80, 100, 120, 150, 175, 225, 300, 400, 500};
freezeDurationMsHistogramToScore = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
@@ -204,7 +208,6 @@
"ro.build.type", "user") != "user"; // Enabled for non-user builds for debugging.
traceTriggerThrottleMs = 5 * 60 * 1000; // 5 mins.
traceMinFreezeDurationMs = 400;
- traceMaxFreezeDurationMs = 1500;
}
VideoRenderQualityTracker::VideoRenderQualityTracker()
@@ -296,15 +299,7 @@
int64_t actualRenderTimeUs = actualRenderTimeNs / 1000;
if (mLastRenderTimeUs != -1) {
- int64_t frameRenderDurationMs = (actualRenderTimeUs - mLastRenderTimeUs) / 1000;
- mRenderDurationMs += frameRenderDurationMs;
- if (mConfiguration.traceTriggerEnabled
- // Threshold for visible video freeze.
- && frameRenderDurationMs >= mConfiguration.traceMinFreezeDurationMs
- // Threshold for removing long render durations which could be video pause.
- && frameRenderDurationMs < mConfiguration.traceMaxFreezeDurationMs) {
- triggerTraceWithThrottle(mTraceTriggerFn, mConfiguration, actualRenderTimeUs);
- }
+ mRenderDurationMs += (actualRenderTimeUs - mLastRenderTimeUs) / 1000;
}
// Now that a frame has been rendered, the previously skipped frames can be processed as skipped
@@ -398,13 +393,13 @@
mLastRenderTimeUs = -1;
mLastFreezeEndTimeUs = -1;
mLastJudderEndTimeUs = -1;
- mWasPreviousFrameDropped = false;
+ mDroppedContentDurationUs = 0;
mFreezeEvent.valid = false;
mJudderEvent.valid = false;
- // Don't worry about tracking frame rendering times from now up until playback catches up to the
- // discontinuity. While stuttering or freezing could be found in the next few frames, the impact
- // to the user is is minimal, so better to just keep things simple and don't bother.
+ // Don't worry about tracking frame rendering times from now up until playback catches up to
+ // the discontinuity. While stuttering or freezing could be found in the next few frames, the
+ // impact to the user is is minimal, so better to just keep things simple and don't bother.
mNextExpectedRenderedFrameQueue = {};
mTunnelFrameQueuedContentTimeUs = -1;
@@ -473,7 +468,7 @@
updateFrameDurations(mDesiredFrameDurationUs, -1);
updateFrameDurations(mActualFrameDurationUs, -1);
updateFrameRate(mMetrics.contentFrameRate, mContentFrameDurationUs, mConfiguration);
- mWasPreviousFrameDropped = false;
+ mDroppedContentDurationUs = 0;
}
void VideoRenderQualityTracker::processMetricsForDroppedFrame(int64_t contentTimeUs,
@@ -484,7 +479,9 @@
updateFrameDurations(mActualFrameDurationUs, -1);
updateFrameRate(mMetrics.contentFrameRate, mContentFrameDurationUs, mConfiguration);
updateFrameRate(mMetrics.desiredFrameRate, mDesiredFrameDurationUs, mConfiguration);
- mWasPreviousFrameDropped = true;
+ if (mContentFrameDurationUs[0] != -1) {
+ mDroppedContentDurationUs += mContentFrameDurationUs[0];
+ }
}
void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTimeUs,
@@ -492,6 +489,8 @@
int64_t actualRenderTimeUs,
FreezeEvent *freezeEventOut,
JudderEvent *judderEventOut) {
+ const Configuration& c = mConfiguration;
+
// Capture the timestamp at which the first frame was rendered
if (mMetrics.firstRenderTimeUs == 0) {
mMetrics.firstRenderTimeUs = actualRenderTimeUs;
@@ -514,11 +513,36 @@
updateFrameRate(mMetrics.desiredFrameRate, mDesiredFrameDurationUs, mConfiguration);
updateFrameRate(mMetrics.actualFrameRate, mActualFrameDurationUs, mConfiguration);
- // If the previous frame was dropped, there was a freeze if we've already rendered a frame
- if (mWasPreviousFrameDropped && mLastRenderTimeUs != -1) {
- processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent,
- mMetrics, mConfiguration);
- mLastFreezeEndTimeUs = actualRenderTimeUs;
+ // A freeze occurs if frames were dropped NOT after a discontinuity
+ if (mDroppedContentDurationUs != 0 && mLastRenderTimeUs != -1) {
+ // When pausing, audio playback may continue for a brief period of time after video
+ // pauses while the audio buffers drain. When resuming, a small number of video frames
+ // might be dropped to catch up to the audio position. This is acceptable behacvior and
+ // should not count as a freeze.
+ bool isLikelyCatchingUpAfterPause = false;
+ // A pause can be detected if a freeze occurs for a longer period of time than the
+ // content duration of the dropped frames. This strategy works because, for freeze
+ // events (no video pause), the content duration of the dropped frames will closely track
+ // the wall clock time (freeze duration). When pausing, however, the wall clock time
+ // (freeze duration) will be longer than the content duration of the dropped frames
+ // required to catch up to the audio position.
+ const int64_t wallClockDurationUs = actualRenderTimeUs - mLastRenderTimeUs;
+ // 200ms is chosen because it is larger than what a hiccup in the display pipeline could
+ // likely be, but shorter than the duration for which a user could pause for.
+ static const int32_t MAX_PIPELINE_HICCUP_DURATION_US = 200 * 1000;
+ if (wallClockDurationUs > mDroppedContentDurationUs + MAX_PIPELINE_HICCUP_DURATION_US) {
+ // Capture the amount of content that is dropped after pause, so we can push apps to be
+ // better about this behavior.
+ if (mDroppedContentDurationUs / 1000 > mMetrics.maxContentDroppedAfterPauseMs) {
+ mMetrics.maxContentDroppedAfterPauseMs = int32_t(mDroppedContentDurationUs / 1000);
+ }
+ isLikelyCatchingUpAfterPause = mDroppedContentDurationUs <= c.pauseAudioLatencyUs;
+ }
+ if (!isLikelyCatchingUpAfterPause) {
+ processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent,
+ mMetrics, mConfiguration, mTraceTriggerFn);
+ mLastFreezeEndTimeUs = actualRenderTimeUs;
+ }
}
maybeCaptureFreezeEvent(actualRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent, mMetrics,
mConfiguration, freezeEventOut);
@@ -537,13 +561,13 @@
maybeCaptureJudderEvent(actualRenderTimeUs, mLastJudderEndTimeUs, mJudderEvent, mMetrics,
mConfiguration, judderEventOut);
- mWasPreviousFrameDropped = false;
+ mDroppedContentDurationUs = 0;
}
void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
int64_t lastFreezeEndTimeUs, FreezeEvent &e,
- VideoRenderQualityMetrics &m,
- const Configuration &c) {
+ VideoRenderQualityMetrics &m, const Configuration &c,
+ const TraceTriggerFn traceTriggerFn) {
int32_t durationMs = int32_t((actualRenderTimeUs - lastRenderTimeUs) / 1000);
m.freezeDurationMsHistogram.insert(durationMs);
int32_t distanceMs = -1;
@@ -579,6 +603,11 @@
e.details.distanceMs.push_back(distanceMs); // -1 for first detail in the first event
}
}
+
+ if (c.traceTriggerEnabled && durationMs >= c.traceMinFreezeDurationMs) {
+ ALOGI("Video freezed %lld ms", (long long) durationMs);
+ triggerTraceWithThrottle(traceTriggerFn, c, actualRenderTimeUs);
+ }
}
void VideoRenderQualityTracker::maybeCaptureFreezeEvent(int64_t actualRenderTimeUs,
@@ -785,17 +814,20 @@
static int64_t lastTriggerUs = -1;
static Mutex updateLastTriggerLock;
- Mutex::Autolock autoLock(updateLastTriggerLock);
- if (lastTriggerUs != -1) {
- int32_t sinceLastTriggerMs = int32_t((triggerTimeUs - lastTriggerUs) / 1000);
- // Throttle the trace trigger calls to reduce continuous PID fork calls in a short time
- // to impact device performance, and reduce spamming trace reports.
- if (sinceLastTriggerMs < c.traceTriggerThrottleMs) {
- ALOGI("Not triggering trace - not enough time since last trigger");
- return;
+ {
+ Mutex::Autolock autoLock(updateLastTriggerLock);
+ if (lastTriggerUs != -1) {
+ int32_t sinceLastTriggerMs = int32_t((triggerTimeUs - lastTriggerUs) / 1000);
+ // Throttle the trace trigger calls to reduce continuous PID fork calls in a short time
+ // to impact device performance, and reduce spamming trace reports.
+ if (sinceLastTriggerMs < c.traceTriggerThrottleMs) {
+ ALOGI("Not triggering trace - not enough time since last trigger");
+ return;
+ }
}
+ lastTriggerUs = triggerTimeUs;
}
- lastTriggerUs = triggerTimeUs;
+
(*traceTriggerFn)();
}
diff --git a/media/libstagefright/include/media/stagefright/ACodec.h b/media/libstagefright/include/media/stagefright/ACodec.h
index f876bc6..a4d82ab 100644
--- a/media/libstagefright/include/media/stagefright/ACodec.h
+++ b/media/libstagefright/include/media/stagefright/ACodec.h
@@ -83,7 +83,7 @@
const char* mime, bool isEncoder,
MediaCodecInfo::CapabilitiesWriter* caps);
- virtual status_t setSurface(const sp<Surface> &surface);
+ virtual status_t setSurface(const sp<Surface> &surface, uint32_t /*generation*/);
virtual void signalFlush();
virtual void signalResume();
diff --git a/media/libstagefright/include/media/stagefright/CodecBase.h b/media/libstagefright/include/media/stagefright/CodecBase.h
index 6f06bc6..0927653 100644
--- a/media/libstagefright/include/media/stagefright/CodecBase.h
+++ b/media/libstagefright/include/media/stagefright/CodecBase.h
@@ -239,7 +239,9 @@
// require an explicit message handler
virtual void onMessageReceived(const sp<AMessage> &msg) = 0;
- virtual status_t setSurface(const sp<Surface>& /*surface*/) { return INVALID_OPERATION; }
+ virtual status_t setSurface(const sp<Surface>& /*surface*/, uint32_t /*generation*/) {
+ return INVALID_OPERATION;
+ }
virtual void signalFlush() = 0;
virtual void signalResume() = 0;
@@ -424,6 +426,15 @@
virtual void pollForRenderedBuffers() = 0;
/**
+ * Notify a buffer is released from output surface.
+ *
+ * @param generation MediaCodec's surface specifier
+ */
+ virtual void onBufferReleasedFromOutputSurface(uint32_t /*generation*/) {
+ // default: no-op
+ };
+
+ /**
* Discard a buffer to the underlying CodecBase object.
*
* TODO: remove once this operation can be handled by just clearing the
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index 879c624..f99a78b 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -449,6 +449,7 @@
int64_t mPresentationTimeUs = 0;
status_t mStickyError;
sp<Surface> mSurface;
+ uint32_t mSurfaceGeneration = 0;
SoftwareRenderer *mSoftRenderer;
Mutex mMetricsLock;
@@ -617,7 +618,7 @@
status_t queueCSDInputBuffer(size_t bufferIndex);
status_t handleSetSurface(const sp<Surface> &surface);
- status_t connectToSurface(const sp<Surface> &surface);
+ status_t connectToSurface(const sp<Surface> &surface, uint32_t *generation);
status_t disconnectFromSurface();
bool hasCryptoOrDescrambler() {
diff --git a/media/libstagefright/include/media/stagefright/MediaWriter.h b/media/libstagefright/include/media/stagefright/MediaWriter.h
index 2b14811..04dcfc0 100644
--- a/media/libstagefright/include/media/stagefright/MediaWriter.h
+++ b/media/libstagefright/include/media/stagefright/MediaWriter.h
@@ -54,6 +54,12 @@
return true;
}
+ // Returns true if the sample data is valid.
+ virtual bool isSampleMetadataValid([[maybe_unused]] size_t trackIndex,
+ [[maybe_unused]] int64_t timeUs) {
+ return true;
+ }
+
virtual status_t addSource(const sp<MediaSource> &source) = 0;
virtual bool reachedEOS() = 0;
virtual status_t start(MetaData *params = NULL) = 0;
diff --git a/media/libstagefright/include/media/stagefright/SurfaceUtils.h b/media/libstagefright/include/media/stagefright/SurfaceUtils.h
index 35b3fa2..eccb413 100644
--- a/media/libstagefright/include/media/stagefright/SurfaceUtils.h
+++ b/media/libstagefright/include/media/stagefright/SurfaceUtils.h
@@ -27,6 +27,7 @@
namespace android {
struct HDRStaticInfo;
+class IProducerListener;
/**
* Configures |nativeWindow| for given |width|x|height|, pixel |format|, |rotation| and |usage|.
@@ -43,6 +44,8 @@
status_t pushBlankBuffersToNativeWindow(ANativeWindow *nativeWindow /* nonnull */);
status_t nativeWindowConnect(ANativeWindow *surface, const char *reason);
status_t nativeWindowDisconnect(ANativeWindow *surface, const char *reason);
+status_t surfaceConnectWithListener(const sp<Surface> &surface,
+ sp<IProducerListener> listener, const char *reason);
/**
* Disable buffer dropping behavior of BufferQueue if target sdk of application
diff --git a/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h b/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h
index cf53f27..7139deb 100644
--- a/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h
+++ b/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h
@@ -63,6 +63,11 @@
// post-render.
float actualFrameRate;
+ // The amount of content duration skipped by the app after a pause when video was trying to
+ // resume. This sometimes happen when catching up to the audio position which continued playing
+ // after video pauses.
+ int32_t maxContentDroppedAfterPauseMs;
+
// A histogram of the durations of freezes due to dropped/skipped frames.
MediaHistogram<int32_t> freezeDurationMsHistogram;
// The computed overall freeze score using the above histogram and score conversion table. The
@@ -155,6 +160,11 @@
// seeking forward.
int32_t liveContentFrameDropToleranceUs;
+ // The amount of time it takes for audio to stop playback after a pause is initiated. Used
+ // for providing some allowance of dropped video frames to catch back up to the audio
+ // position when resuming playback.
+ int32_t pauseAudioLatencyUs;
+
// Freeze configuration
//
// The values used to distribute freeze durations across a histogram.
@@ -210,11 +220,6 @@
//
// The minimum frame render duration to recognize video freeze event to collect trace.
int32_t traceMinFreezeDurationMs;
- //
- // The maximum frame render duration to recognize video freeze event. A frame render
- // duration that is larger than the max duration would not trigger trace collection for
- // video freeze because it's highly possible a video pause.
- int32_t traceMaxFreezeDurationMs;
};
struct FreezeEvent {
@@ -370,7 +375,8 @@
// Process a frame freeze.
static void processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
int64_t lastFreezeEndTimeUs, FreezeEvent &e,
- VideoRenderQualityMetrics &m, const Configuration &c);
+ VideoRenderQualityMetrics &m, const Configuration &c,
+ const TraceTriggerFn traceTriggerFn);
// Retrieve a freeze event if an event just finished.
static void maybeCaptureFreezeEvent(int64_t actualRenderTimeUs, int64_t lastFreezeEndTimeUs,
@@ -441,8 +447,8 @@
// The render duration of the playback.
int64_t mRenderDurationMs;
- // True if the previous frame was dropped.
- bool mWasPreviousFrameDropped;
+ // The duration of the content that was dropped.
+ int64_t mDroppedContentDurationUs;
// The freeze event that's currently being tracked.
FreezeEvent mFreezeEvent;
diff --git a/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp b/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp
index 78140dd..16f8294 100644
--- a/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp
+++ b/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp
@@ -140,6 +140,7 @@
EXPECT_EQ(c.maxExpectedContentFrameDurationUs, d.maxExpectedContentFrameDurationUs);
EXPECT_EQ(c.frameRateDetectionToleranceUs, d.frameRateDetectionToleranceUs);
EXPECT_EQ(c.liveContentFrameDropToleranceUs, d.liveContentFrameDropToleranceUs);
+ EXPECT_EQ(c.pauseAudioLatencyUs, d.pauseAudioLatencyUs);
EXPECT_EQ(c.freezeDurationMsHistogramBuckets, d.freezeDurationMsHistogramBuckets);
EXPECT_EQ(c.freezeDurationMsHistogramToScore, d.freezeDurationMsHistogramToScore);
EXPECT_EQ(c.freezeDistanceMsHistogramBuckets, d.freezeDistanceMsHistogramBuckets);
@@ -155,7 +156,6 @@
EXPECT_EQ(c.traceTriggerEnabled, d.traceTriggerEnabled);
EXPECT_EQ(c.traceTriggerThrottleMs, d.traceTriggerThrottleMs);
EXPECT_EQ(c.traceMinFreezeDurationMs, d.traceMinFreezeDurationMs);
- EXPECT_EQ(c.traceMaxFreezeDurationMs, d.traceMaxFreezeDurationMs);
}
TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withEmpty) {
@@ -171,6 +171,7 @@
EXPECT_EQ(c.maxExpectedContentFrameDurationUs, d.maxExpectedContentFrameDurationUs);
EXPECT_EQ(c.frameRateDetectionToleranceUs, d.frameRateDetectionToleranceUs);
EXPECT_EQ(c.liveContentFrameDropToleranceUs, d.liveContentFrameDropToleranceUs);
+ EXPECT_EQ(c.pauseAudioLatencyUs, d.pauseAudioLatencyUs);
EXPECT_EQ(c.freezeDurationMsHistogramBuckets, d.freezeDurationMsHistogramBuckets);
EXPECT_EQ(c.freezeDurationMsHistogramToScore, d.freezeDurationMsHistogramToScore);
EXPECT_EQ(c.freezeDistanceMsHistogramBuckets, d.freezeDistanceMsHistogramBuckets);
@@ -186,7 +187,6 @@
EXPECT_EQ(c.traceTriggerEnabled, d.traceTriggerEnabled);
EXPECT_EQ(c.traceTriggerThrottleMs, d.traceTriggerThrottleMs);
EXPECT_EQ(c.traceMinFreezeDurationMs, d.traceMinFreezeDurationMs);
- EXPECT_EQ(c.traceMaxFreezeDurationMs, d.traceMaxFreezeDurationMs);
}
TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withInvalid) {
@@ -202,6 +202,7 @@
EXPECT_EQ(c.maxExpectedContentFrameDurationUs, d.maxExpectedContentFrameDurationUs);
EXPECT_EQ(c.frameRateDetectionToleranceUs, d.frameRateDetectionToleranceUs);
EXPECT_EQ(c.liveContentFrameDropToleranceUs, d.liveContentFrameDropToleranceUs);
+ EXPECT_EQ(c.pauseAudioLatencyUs, d.pauseAudioLatencyUs);
EXPECT_EQ(c.freezeDurationMsHistogramBuckets, d.freezeDurationMsHistogramBuckets);
EXPECT_EQ(c.freezeDurationMsHistogramToScore, d.freezeDurationMsHistogramToScore);
EXPECT_EQ(c.freezeDistanceMsHistogramBuckets, d.freezeDistanceMsHistogramBuckets);
@@ -217,7 +218,6 @@
EXPECT_EQ(c.traceTriggerEnabled, d.traceTriggerEnabled);
EXPECT_EQ(c.traceTriggerThrottleMs, d.traceTriggerThrottleMs);
EXPECT_EQ(c.traceMinFreezeDurationMs, d.traceMinFreezeDurationMs);
- EXPECT_EQ(c.traceMaxFreezeDurationMs, d.traceMaxFreezeDurationMs);
}
TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withAlmostValid) {
@@ -233,6 +233,8 @@
return "10b0";
} else if (flag == "render_metrics_live_content_frame_drop_tolerance_us") {
return "c100";
+ } else if (flag == "render_metrics_pause_audio_latency_us") {
+ return "1ab0";
} else if (flag == "render_metrics_freeze_duration_ms_histogram_buckets") {
return "1,5300,3b400,123";
} else if (flag == "render_metrics_freeze_duration_ms_histogram_to_score") {
@@ -276,6 +278,7 @@
EXPECT_EQ(c.maxExpectedContentFrameDurationUs, d.maxExpectedContentFrameDurationUs);
EXPECT_EQ(c.frameRateDetectionToleranceUs, d.frameRateDetectionToleranceUs);
EXPECT_EQ(c.liveContentFrameDropToleranceUs, d.liveContentFrameDropToleranceUs);
+ EXPECT_EQ(c.pauseAudioLatencyUs, d.pauseAudioLatencyUs);
EXPECT_EQ(c.freezeDurationMsHistogramBuckets, d.freezeDurationMsHistogramBuckets);
EXPECT_EQ(c.freezeDurationMsHistogramToScore, d.freezeDurationMsHistogramToScore);
EXPECT_EQ(c.freezeDistanceMsHistogramBuckets, d.freezeDistanceMsHistogramBuckets);
@@ -291,7 +294,6 @@
EXPECT_EQ(c.traceTriggerEnabled, d.traceTriggerEnabled);
EXPECT_EQ(c.traceTriggerThrottleMs, d.traceTriggerThrottleMs);
EXPECT_EQ(c.traceMinFreezeDurationMs, d.traceMinFreezeDurationMs);
- EXPECT_EQ(c.traceMaxFreezeDurationMs, d.traceMaxFreezeDurationMs);
}
TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withValid) {
@@ -307,6 +309,8 @@
return "3000";
} else if (flag == "render_metrics_live_content_frame_drop_tolerance_us") {
return "4000";
+ } else if (flag == "render_metrics_pause_audio_latency_us") {
+ return "300000";
} else if (flag == "render_metrics_freeze_duration_ms_histogram_buckets") {
return "100,200,300,400";
} else if (flag == "render_metrics_freeze_duration_ms_histogram_to_score") {
@@ -359,6 +363,8 @@
EXPECT_NE(c.frameRateDetectionToleranceUs, d.frameRateDetectionToleranceUs);
EXPECT_EQ(c.liveContentFrameDropToleranceUs, 4000);
EXPECT_NE(c.liveContentFrameDropToleranceUs, d.liveContentFrameDropToleranceUs);
+ EXPECT_EQ(c.pauseAudioLatencyUs, 300000);
+ EXPECT_NE(c.pauseAudioLatencyUs, d.pauseAudioLatencyUs);
{
std::vector<int32_t> expected({100,200,300,400});
EXPECT_EQ(c.freezeDurationMsHistogramBuckets, expected);
@@ -402,7 +408,6 @@
EXPECT_EQ(c.traceTriggerEnabled, true);
EXPECT_EQ(c.traceTriggerThrottleMs, 50000);
EXPECT_EQ(c.traceMinFreezeDurationMs, 1000);
- EXPECT_EQ(c.traceMaxFreezeDurationMs, 5000);
}
TEST_F(VideoRenderQualityTrackerTest, countsReleasedFrames) {
@@ -1116,53 +1121,34 @@
c.enabled = true;
c.traceTriggerEnabled = true; // The trigger is enabled, so traces should be triggered.
// The value of traceTriggerThrottleMs must be larger than traceMinFreezeDurationMs. Otherwise,
- // the throttle does work.
+ // the throttle does not work.
c.traceTriggerThrottleMs = 200;
- c.traceMinFreezeDurationMs = 40;
- int32_t freeze = c.traceMinFreezeDurationMs;
+ c.traceMinFreezeDurationMs = 4 * 20; // 4 frames.
Helper h(20, c);
- // Freeze triggers separated by 80ms which is less than the threshold.
- h.render({
- freeze, // Freeze duration does not check trace trigger.
- 20, // Trace triggered.
- 20, // Throttle time: 20/200ms
- 20, // Throttle time: 40/200ms
- freeze, // Throttle time: 80/200ms
- 20, // Throttle time: 100/200ms (Trace not triggered)
- });
+ // Freeze triggers separated by 100ms which is less than the threshold.
+ h.render(1); // Video start.
+ h.drop(3); // Freeze.
+ h.render(1); // Trace triggered.
+ h.render(1); // Throttle time: 20/200ms
+ h.drop(3); // Throttle time: 80/200ms
+ h.render(1); // Throttle time: 100/200ms (Trace not triggered)
EXPECT_EQ(h.getTraceTriggeredCount(), 1);
// Next freeze trigger is separated by 200ms which breaks the throttle threshold.
- h.render({
- 20, // Throttle time: 120/200ms
- 20, // Throttle time: 140/200ms
- 20, // Throttle time: 160/200ms
- freeze, // Throttle time: 200/200ms
- 20, // Trace triggered.
- });
+ h.render(1); // Throttle time: 120/200ms
+ h.drop(3); // Throttle time: 180/200ms
+ h.render(1); // Throttle time: 200/200ms (Trace triggered)
EXPECT_EQ(h.getTraceTriggeredCount(), 2);
- // Next freeze trigger is separated by 80ms which is less than the threshold.
- h.render({
- 20, // Throttle time: 20/200ms
- 20, // Throttle time: 40/200ms
- freeze, // Throttle time: 80/200ms
- 20, // Throttle time: 100/200ms (Trace not triggered)
- });
+ // Next freeze trigger is separated by 100ms which is less than the threshold.
+ h.render(1); // Throttle time: 20/200ms
+ h.drop(3); // Throttle time: 80/200ms
+ h.render(1); // Throttle time: 100/200ms (Trace not triggered)
EXPECT_EQ(h.getTraceTriggeredCount(), 2);
-}
-
-TEST_F(VideoRenderQualityTrackerTest, freezeForTraceDuration_triggersTrace) {
- Configuration c;
- c.enabled = true;
- c.traceTriggerEnabled = true; // The trigger is enabled, so traces should be triggered.
- c.traceTriggerThrottleMs = 0; // Disable throttle in the test case.
- int32_t freeze1 = c.traceMinFreezeDurationMs;
- int32_t freeze2 = c.traceMaxFreezeDurationMs - 1;
- int32_t couldBeAPause = c.traceMaxFreezeDurationMs + 1;
-
- Helper h(20, c);
- h.render({freeze1, 20, freeze2, 20, couldBeAPause, 20});
-
+ // Freeze duration is less than traceMinFreezeDurationMs and throttle ends.
+ h.render(1); // Throttle time: 120/200ms
+ h.render(1); // Throttle time: 140/200ms
+ h.drop(2); // Throttle time: 180/200ms
+ h.render(1); // Throttle time: 200/200ms (Trace not triggered, freeze duration = 60ms)
EXPECT_EQ(h.getTraceTriggeredCount(), 2);
}
@@ -1172,13 +1158,64 @@
c.enabled = true;
c.traceTriggerEnabled = false; // The trigger is disabled, so no traces should be triggered.
c.traceTriggerThrottleMs = 0; // Disable throttle in the test case.
- int32_t freeze1 = c.traceMinFreezeDurationMs;
- int32_t freeze2 = c.traceMaxFreezeDurationMs - 1;
- int32_t couldBeAPause = c.traceMaxFreezeDurationMs + 1;
+ c.traceMinFreezeDurationMs = 4 * 20; // 4 frames.
Helper h(20, c);
- h.render({freeze1, 20, freeze2, 20, couldBeAPause, 20});
+ h.render(1);
+ h.drop(3);
+ h.render(1); // Render duration is 80 ms.
+ h.drop(4);
+ h.render(1); // Render duration is 100 ms.
EXPECT_EQ(h.getTraceTriggeredCount(), 0);
}
+
+TEST_F(VideoRenderQualityTrackerTest, doesNotCountCatchUpAfterPauseAsFreeze) {
+ Configuration c;
+ c.enabled = true;
+ c.pauseAudioLatencyUs = 200 * 1000; // allows for up to 10 frames to be dropped to catch up
+ // to the audio position
+ Helper h(20, c);
+ // A few frames followed by a long pause
+ h.render({20, 20, 1000});
+ h.drop(10); // simulate catching up to audio
+ h.render({20, 20, 1000});
+ h.drop(11); // simulate catching up to audio but then also dropping frames
+ h.render({20});
+
+ // Only 1 freeze is counted because the first freeze (200ms) because it's equal to or below the
+ // pause latency allowance, and the algorithm assumes a legitimate case of the video trying to
+ // catch up to the audio position, which continued to play for a short period of time (less than
+ // 200ms) after the pause was initiated
+ EXPECT_EQ(h.getMetrics().freezeDurationMsHistogram.getCount(), 1);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, capturesMaximumContentDroppedAfterPause) {
+ Configuration c;
+ c.enabled = true;
+ c.pauseAudioLatencyUs = 200 * 1000; // allows for up to 10 frames to be dropped to catch up
+ // to the audio position
+ Helper h(20, c);
+
+ // Freezes are below the pause latency are captured
+ h.render({20, 20, 1000});
+ h.drop(6);
+ h.render({20, 20, 1000});
+ h.drop(8);
+ h.render({20, 20, 1000});
+ h.drop(7);
+ h.render({20});
+ EXPECT_EQ(h.getMetrics().maxContentDroppedAfterPauseMs, 8 * 20);
+
+ // Freezes are above the pause latency are also captured
+ h.render({20, 20, 1000});
+ h.drop(10);
+ h.render({20, 20, 1000});
+ h.drop(12);
+ h.render({20, 20, 1000});
+ h.drop(11);
+ h.render({20});
+ EXPECT_EQ(h.getMetrics().maxContentDroppedAfterPauseMs, 12 * 20);
+}
+
} // android
diff --git a/media/libstagefright/tests/mediacodec/MediaCodecTest.cpp b/media/libstagefright/tests/mediacodec/MediaCodecTest.cpp
index 71ddbe5..ed01e36 100644
--- a/media/libstagefright/tests/mediacodec/MediaCodecTest.cpp
+++ b/media/libstagefright/tests/mediacodec/MediaCodecTest.cpp
@@ -89,7 +89,8 @@
MOCK_METHOD(void, initiateStart, (), (override));
MOCK_METHOD(void, initiateShutdown, (bool keepComponentAllocated), (override));
MOCK_METHOD(void, onMessageReceived, (const sp<AMessage> &msg), (override));
- MOCK_METHOD(status_t, setSurface, (const sp<Surface> &surface), (override));
+ MOCK_METHOD(
+ status_t, setSurface, (const sp<Surface> &surface, uint32_t generation), (override));
MOCK_METHOD(void, signalFlush, (), (override));
MOCK_METHOD(void, signalResume, (), (override));
MOCK_METHOD(void, signalRequestIDRFrame, (), (override));
diff --git a/media/libstagefright/webm/WebmWriter.cpp b/media/libstagefright/webm/WebmWriter.cpp
index 3823c36..ca862b0 100644
--- a/media/libstagefright/webm/WebmWriter.cpp
+++ b/media/libstagefright/webm/WebmWriter.cpp
@@ -67,6 +67,25 @@
return true;
}
+bool WebmWriter::isSampleMetadataValid(size_t trackIndex, int64_t timeUs) {
+ int64_t prevTimeUs = 0;
+ if (mLastTimestampUsByTrackIndex.find(trackIndex) != mLastTimestampUsByTrackIndex.end()) {
+ prevTimeUs = mLastTimestampUsByTrackIndex[trackIndex];
+ }
+ // WebM has monotonically increasing timestamps
+ if (timeUs < 0 || timeUs < prevTimeUs) {
+ return false;
+ }
+ int64_t lastDurationUs = timeUs - prevTimeUs;
+ // Ensure that the timeUs value does not overflow,
+ // when adding lastDurationUs in the WebmFrameMediaSourceThread.
+ if (timeUs > (INT64_MAX / 1000) - lastDurationUs) {
+ return false;
+ }
+ mLastTimestampUsByTrackIndex[trackIndex] = timeUs;
+ return true;
+}
+
WebmWriter::WebmWriter(int fd)
: mFd(dup(fd)),
mInitCheck(mFd < 0 ? NO_INIT : OK),
diff --git a/media/libstagefright/webm/include/webm/WebmWriter.h b/media/libstagefright/webm/include/webm/WebmWriter.h
index e339add..4c51f0e 100644
--- a/media/libstagefright/webm/include/webm/WebmWriter.h
+++ b/media/libstagefright/webm/include/webm/WebmWriter.h
@@ -40,6 +40,9 @@
// which is compatible with WebmWriter.
// Note that this overloads that method in the base class.
static bool isFdOpenModeValid(int fd);
+ // Returns true if the timestamp is valid which is compatible with the WebmWriter.
+ // Note that this overloads that method in the base class.
+ bool isSampleMetadataValid(size_t trackIndex, int64_t timeUs);
explicit WebmWriter(int fd);
~WebmWriter() { reset(); }
@@ -67,6 +70,7 @@
uint64_t mInfoSize;
uint64_t mTracksOffset;
uint64_t mCuesOffset;
+ std::map<size_t, int64_t> mLastTimestampUsByTrackIndex;
bool mPaused;
bool mStarted;
diff --git a/media/libstagefright/writer_fuzzers/WriterFuzzerBase.cpp b/media/libstagefright/writer_fuzzers/WriterFuzzerBase.cpp
index fee5c94..7235ba9 100644
--- a/media/libstagefright/writer_fuzzers/WriterFuzzerBase.cpp
+++ b/media/libstagefright/writer_fuzzers/WriterFuzzerBase.cpp
@@ -209,6 +209,9 @@
}
vector<FrameData> bufferInfo = mBufferSource->getFrameList(trackIndex);
for (int idx = startFrameIndex; idx < endFrameIndex; ++idx) {
+ if (!mWriter->isSampleMetadataValid(trackIndex, bufferInfo[idx].timeUs)) {
+ continue;
+ }
sp<ABuffer> buffer = new ABuffer((void *)bufferInfo[idx].buf, bufferInfo[idx].size);
MediaBuffer *mediaBuffer = new MediaBuffer(buffer);
diff --git a/media/module/TEST_MAPPING b/media/module/TEST_MAPPING
index c59df72..1b572da 100644
--- a/media/module/TEST_MAPPING
+++ b/media/module/TEST_MAPPING
@@ -1,8 +1,7 @@
{
- "postsubmit": [
+ "presubmit": [
{
"name": "hal_implementation_test"
}
]
}
-
diff --git a/media/module/extractors/fuzzers/Android.bp b/media/module/extractors/fuzzers/Android.bp
index 91ca7b1..0a8d2ab 100644
--- a/media/module/extractors/fuzzers/Android.bp
+++ b/media/module/extractors/fuzzers/Android.bp
@@ -69,9 +69,9 @@
fuzz_config: {
cc: [
- "android-media-fuzzing-reports@google.com",
+ "android-media-playback+bugs@google.com",
],
- componentid: 155276,
+ componentid: 817235,
hotlists: [
"4593311",
],
diff --git a/media/mtp/MtpPacket.cpp b/media/mtp/MtpPacket.cpp
index f196d87..af67fc0 100644
--- a/media/mtp/MtpPacket.cpp
+++ b/media/mtp/MtpPacket.cpp
@@ -168,8 +168,10 @@
return;
}
int offset = MTP_CONTAINER_PARAMETER_OFFSET + (index - 1) * sizeof(uint32_t);
- if (mPacketSize < offset + sizeof(uint32_t))
+ if (mPacketSize < offset + sizeof(uint32_t)) {
mPacketSize = offset + sizeof(uint32_t);
+ allocate(mPacketSize);
+ }
putUInt32(offset, value);
}
diff --git a/media/utils/TimeCheck.cpp b/media/utils/TimeCheck.cpp
index a5378e6..ec68de7 100644
--- a/media/utils/TimeCheck.cpp
+++ b/media/utils/TimeCheck.cpp
@@ -287,6 +287,11 @@
.append(halPids).append("\n")
.append(snapshotAnalysis.toString());
+ // In many cases, the initial timeout stack differs from the abort backtrace because
+ // (1) the time difference between initial timeout and the final abort signal
+ // and (2) signalling the HAL audio service may cause
+ // the thread to unblock and continue.
+
// Note: LOG_ALWAYS_FATAL limits the size of the string - per log/log.h:
// Log message text may be truncated to less than an
// implementation-specific limit (1023 bytes).
diff --git a/media/utils/TimerThread.cpp b/media/utils/TimerThread.cpp
index 3966103..ad27af3 100644
--- a/media/utils/TimerThread.cpp
+++ b/media/utils/TimerThread.cpp
@@ -21,6 +21,7 @@
#include <unistd.h>
#include <vector>
+#include <audio_utils/mutex.h>
#include <mediautils/MediaUtilsDelayed.h>
#include <mediautils/TidWrapper.h>
#include <mediautils/TimerThread.h>
@@ -59,14 +60,14 @@
return true;
}
-
-std::string TimerThread::SnapshotAnalysis::toString() const {
+std::string TimerThread::SnapshotAnalysis::toString(bool showTimeoutStack) const {
// Note: These request queues are snapshot very close together but
// not at "identical" times as we don't use a class-wide lock.
std::string analysisSummary = std::string("\nanalysis [ ").append(description).append(" ]");
std::string timeoutStack;
std::string blockedStack;
- if (timeoutTid != -1) {
+ std::string mutexWaitChainStack;
+ if (showTimeoutStack && timeoutTid != -1) {
timeoutStack = std::string(suspectTid == timeoutTid ? "\ntimeout/blocked(" : "\ntimeout(")
.append(std::to_string(timeoutTid)).append(") callstack [\n")
.append(getCallStackStringForTid(timeoutTid)).append("]");
@@ -78,6 +79,29 @@
.append(getCallStackStringForTid(suspectTid)).append("]");
}
+ if (!mutexWaitChain.empty()) {
+ mutexWaitChainStack.append("\nmutex wait chain [\n");
+ // the wait chain omits the initial timeout tid (which is good as we don't
+ // need to suppress it).
+ for (size_t i = 0; i < mutexWaitChain.size(); ++i) {
+ const auto& [tid, name] = mutexWaitChain[i];
+ mutexWaitChainStack.append("{ tid: ").append(std::to_string(tid))
+ .append(" (holding ").append(name).append(")");
+ if (tid == timeoutTid) {
+ mutexWaitChainStack.append(" TIMEOUT_STACK }\n");
+ } else if (tid == suspectTid) {
+ mutexWaitChainStack.append(" BLOCKED_STACK }\n");
+ } else if (hasMutexCycle && i == mutexWaitChain.size() - 1) {
+ // for a cycle, the last pid in the chain is repeated.
+ mutexWaitChainStack.append(" CYCLE_STACK }\n");
+ } else {
+ mutexWaitChainStack.append(" callstack [\n")
+ .append(getCallStackStringForTid(tid)).append("] }\n");
+ }
+ }
+ mutexWaitChainStack.append("]");
+ }
+
return std::string("now ")
.append(formatTime(std::chrono::system_clock::now()))
.append("\nsecondChanceCount ")
@@ -91,7 +115,8 @@
.append(requestsToString(retiredRequests))
.append(" ]")
.append(timeoutStack)
- .append(blockedStack);
+ .append(blockedStack)
+ .append(mutexWaitChainStack);
}
// A HAL method is where the substring "Hidl" is in the class name.
@@ -121,13 +146,33 @@
analysis.pendingRequests = getPendingRequests();
analysis.secondChanceCount = mMonitorThread.getSecondChanceCount();
// No call has timed out, so there is no analysis to be done.
- if (analysis.timeoutRequests.empty())
+ if (analysis.timeoutRequests.empty()) {
return analysis;
+ }
+
// for now look at last timeout (in our case, the only timeout)
const std::shared_ptr<const Request> timeout = analysis.timeoutRequests.back();
analysis.timeoutTid = timeout->tid;
- if (analysis.pendingRequests.empty())
- return analysis;
+
+ std::string& description = analysis.description;
+
+ // audio mutex specific wait chain analysis
+ auto deadlockInfo = audio_utils::mutex::deadlock_detection(analysis.timeoutTid);
+ ALOGD("%s: deadlockInfo: %s", __func__, deadlockInfo.to_string().c_str());
+
+ if (!deadlockInfo.empty()) {
+ if (!description.empty()) description.append("\n");
+ description.append(deadlockInfo.to_string());
+ }
+
+ analysis.hasMutexCycle = deadlockInfo.has_cycle;
+ analysis.mutexWaitChain = std::move(deadlockInfo.chain);
+
+ // no pending requests, we don't attempt to use temporal correlation between a recent call.
+ if (analysis.pendingRequests.empty()) {
+ return analysis;
+ }
+
// pending Requests that are problematic.
std::vector<std::shared_ptr<const Request>> pendingExact;
std::vector<std::shared_ptr<const Request>> pendingPossible;
@@ -151,15 +196,15 @@
}
}
- std::string& description = analysis.description;
if (!pendingExact.empty()) {
const auto& request = pendingExact.front();
const bool hal = isRequestFromHal(request);
if (hal) {
- description = std::string("Blocked directly due to HAL call: ")
+ if (!description.empty()) description.append("\n");
+ description.append("Blocked directly due to HAL call: ")
.append(request->toString());
- analysis.suspectTid= request->tid;
+ analysis.suspectTid = request->tid;
}
}
if (description.empty() && !pendingPossible.empty()) {
diff --git a/media/utils/include/mediautils/FixedString.h b/media/utils/include/mediautils/FixedString.h
index 047aa82..c316813 100644
--- a/media/utils/include/mediautils/FixedString.h
+++ b/media/utils/include/mediautils/FixedString.h
@@ -101,10 +101,15 @@
return strncmp(c_str(), s, capacity() + 1) == 0;
}
- bool operator==(std::string_view s) const {
+ bool operator==(const std::string_view s) const {
return size() == s.size() && memcmp(data(), s.data(), size()) == 0;
}
+ template <uint32_t N_>
+ bool operator==(const FixedString<N_>& s) const {
+ return operator==(s.asStringView());
+ }
+
// operator not-equals
template <typename T>
bool operator!=(const T& other) const {
diff --git a/media/utils/include/mediautils/TimerThread.h b/media/utils/include/mediautils/TimerThread.h
index d84d682..320567b 100644
--- a/media/utils/include/mediautils/TimerThread.h
+++ b/media/utils/include/mediautils/TimerThread.h
@@ -269,9 +269,12 @@
std::vector<std::shared_ptr<const Request>> timeoutRequests;
// List of retired requests
std::vector<std::shared_ptr<const Request>> retiredRequests;
+ // mutex deadlock / wait detection information.
+ bool hasMutexCycle = false;
+ std::vector<std::pair<pid_t, std::string>> mutexWaitChain;
// Dumps the information contained above as well as additional call
// stacks where applicable.
- std::string toString() const;
+ std::string toString(bool showTimeoutStack = true) const;
};
private:
diff --git a/media/utils/tests/Android.bp b/media/utils/tests/Android.bp
index d1675fd..3fdc6eb 100644
--- a/media/utils/tests/Android.bp
+++ b/media/utils/tests/Android.bp
@@ -13,7 +13,6 @@
host_supported: true,
- cpp_std: "gnu++17",
cflags: [
"-Wall",
"-Werror",
diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp
index 2aa7cbf..54880f8 100644
--- a/services/audioflinger/Threads.cpp
+++ b/services/audioflinger/Threads.cpp
@@ -85,7 +85,6 @@
#include <utils/Trace.h>
#include <fcntl.h>
-#include <future>
#include <linux/futex.h>
#include <math.h>
#include <memory>
@@ -3900,25 +3899,15 @@
{
aflog::setThreadWriter(mNBLogWriter.get());
- std::future<void> priorityBoostFuture; // joined on dtor; this is a one-shot boost.
if (mType == SPATIALIZER) {
const pid_t tid = getTid();
if (tid == -1) { // odd: we are here, we must be a running thread.
ALOGW("%s: Cannot update Spatializer mixer thread priority, no tid", __func__);
} else {
- // We launch the priority boost request in a separate thread because
- // the SchedulingPolicyService may not be available during early
- // boot time, with a wait causing boot delay.
- // There is also a PrioConfigEvent that does this, but it will also
- // block other config events. This command should be able
- // to run concurrent with other stream commands.
- priorityBoostFuture = std::async(std::launch::async,
- [tid, output_sp = stream()]() {
- const int priorityBoost = requestSpatializerPriority(getpid(), tid);
- if (priorityBoost > 0) {
- output_sp->setHalThreadPriority(priorityBoost);
- }
- });
+ const int priorityBoost = requestSpatializerPriority(getpid(), tid);
+ if (priorityBoost > 0) {
+ stream()->setHalThreadPriority(priorityBoost);
+ }
}
}
@@ -7758,7 +7747,8 @@
uint32_t DuplicatingThread::activeSleepTimeUs() const
{
- return (mWaitTimeMs * 1000) / 2;
+ // return half the wait time in microseconds.
+ return std::min(mWaitTimeMs * 500ULL, (unsigned long long)UINT32_MAX); // prevent overflow.
}
void DuplicatingThread::cacheParameters_l()
diff --git a/services/audioflinger/afutils/NBAIO_Tee.h b/services/audioflinger/afutils/NBAIO_Tee.h
index 13335fe..a5c544e 100644
--- a/services/audioflinger/afutils/NBAIO_Tee.h
+++ b/services/audioflinger/afutils/NBAIO_Tee.h
@@ -24,6 +24,7 @@
#include <mutex>
#include <set>
+#include <audio_utils/clock.h>
#include <cutils/properties.h>
#include <media/nbaio/NBAIO.h>
diff --git a/services/audioflinger/datapath/SpdifStreamOut.h b/services/audioflinger/datapath/SpdifStreamOut.h
index 321b172..56d57f6 100644
--- a/services/audioflinger/datapath/SpdifStreamOut.h
+++ b/services/audioflinger/datapath/SpdifStreamOut.h
@@ -120,7 +120,6 @@
audio_config_base_t mApplicationConfig = AUDIO_CONFIG_BASE_INITIALIZER;
ssize_t writeDataBurst(const void* data, size_t bytes);
- ssize_t writeInternal(const void* buffer, size_t bytes);
#ifdef TEE_SINK
NBAIO_Tee mTee;
diff --git a/services/audiopolicy/common/include/policy.h b/services/audiopolicy/common/include/policy.h
index d266e63..4643bd1 100644
--- a/services/audiopolicy/common/include/policy.h
+++ b/services/audiopolicy/common/include/policy.h
@@ -115,7 +115,7 @@
*/
static inline bool device_has_encoding_capability(audio_devices_t device)
{
- return audio_is_a2dp_out_device(device);
+ return audio_is_a2dp_out_device(device) || audio_is_ble_out_device(device);
}
/**
diff --git a/services/audiopolicy/engine/common/Android.bp b/services/audiopolicy/engine/common/Android.bp
index 6c46c54..0034a04 100644
--- a/services/audiopolicy/engine/common/Android.bp
+++ b/services/audiopolicy/engine/common/Android.bp
@@ -57,4 +57,8 @@
"libaudiofoundation",
"libaudiopolicycomponents",
],
+ whole_static_libs: [
+ "server_configurable_flags",
+ "com.android.media.audio-aconfig-cc",
+ ],
}
diff --git a/services/audiopolicy/engine/common/src/EngineBase.cpp b/services/audiopolicy/engine/common/src/EngineBase.cpp
index 218aff8..e259e6e 100644
--- a/services/audiopolicy/engine/common/src/EngineBase.cpp
+++ b/services/audiopolicy/engine/common/src/EngineBase.cpp
@@ -24,6 +24,7 @@
#include "EngineBase.h"
#include "EngineDefaultConfig.h"
#include <TypeConverter.h>
+#include <com_android_media_audio.h>
namespace android {
namespace audio_policy {
@@ -178,6 +179,24 @@
ALOGE("%s: Invalid %s", __FUNCTION__, configCurve.deviceCategory.c_str());
continue;
}
+ if (!com::android::media::audio::alarm_min_volume_zero()) {
+ /*
+ * This special handling is done because the audio_policy_volumes.xml
+ * is updated but if the flag is disabled, the min alarm volume can
+ * still be zero because the audio_policy_volumes.xml is the source of
+ * truth. So the index value is modified here to match the flag setting
+ */
+ if (volumeConfig.name.compare(audio_stream_type_to_string(AUDIO_STREAM_ALARM)) == 0
+ && deviceCat == DEVICE_CATEGORY_SPEAKER) {
+ for (auto &point : configCurve.curvePoints) {
+ if (point.index == 1) {
+ ALOGD("Mute alarm disabled: Found point with index 1. setting it to 0");
+ point.index = 0;
+ break;
+ }
+ }
+ }
+ }
sp<VolumeCurve> curve = new VolumeCurve(deviceCat);
for (auto &point : configCurve.curvePoints) {
curve->add({point.index, point.attenuationInMb});
diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
index e8066fb..69d3b5d 100644
--- a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
+++ b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
@@ -511,11 +511,6 @@
const char *device_name,
audio_format_t encodedFormat)
{
- status_t status;
- String8 reply;
- AudioParameter param;
- int isReconfigA2dpSupported = 0;
-
ALOGV("handleDeviceConfigChange(() device: 0x%X, address %s name %s encodedFormat: 0x%X",
device, device_address, device_name, encodedFormat);
@@ -537,19 +532,22 @@
// Case 1: A2DP active device switches from primary to primary
// module
// Case 2: A2DP device config changes on primary module.
- if (audio_is_a2dp_out_device(device) && hasPrimaryOutput()) {
+ if (device_has_encoding_capability(device) && hasPrimaryOutput()) {
sp<HwModule> module = mHwModules.getModuleForDeviceType(device, encodedFormat);
audio_module_handle_t primaryHandle = mPrimaryOutput->getModuleHandle();
if (availablePrimaryOutputDevices().contains(devDesc) &&
(module != 0 && module->getHandle() == primaryHandle)) {
- reply = mpClientInterface->getParameters(
- AUDIO_IO_HANDLE_NONE,
- String8(AudioParameter::keyReconfigA2dpSupported));
+ bool isA2dp = audio_is_a2dp_out_device(device);
+ const String8 supportKey = isA2dp ? String8(AudioParameter::keyReconfigA2dpSupported)
+ : String8(AudioParameter::keyReconfigLeSupported);
+ String8 reply = mpClientInterface->getParameters(AUDIO_IO_HANDLE_NONE, supportKey);
AudioParameter repliedParameters(reply);
- repliedParameters.getInt(
- String8(AudioParameter::keyReconfigA2dpSupported), isReconfigA2dpSupported);
- if (isReconfigA2dpSupported) {
- const String8 key(AudioParameter::keyReconfigA2dp);
+ int isReconfigSupported;
+ repliedParameters.getInt(supportKey, isReconfigSupported);
+ if (isReconfigSupported) {
+ const String8 key = isA2dp ? String8(AudioParameter::keyReconfigA2dp)
+ : String8(AudioParameter::keyReconfigLe);
+ AudioParameter param;
param.add(key, String8("true"));
mpClientInterface->setParameters(AUDIO_IO_HANDLE_NONE, param.toString());
devDesc->setEncodedFormat(encodedFormat);
@@ -569,7 +567,7 @@
}
// Toggle the device state: UNAVAILABLE -> AVAILABLE
// This will force reading again the device configuration
- status = setDeviceConnectionState(device,
+ status_t status = setDeviceConnectionState(device,
AUDIO_POLICY_DEVICE_STATE_UNAVAILABLE,
device_address, device_name,
devDesc->getEncodedFormat());
diff --git a/services/audiopolicy/service/Android.bp b/services/audiopolicy/service/Android.bp
index c674909..fb55225 100644
--- a/services/audiopolicy/service/Android.bp
+++ b/services/audiopolicy/service/Android.bp
@@ -31,6 +31,7 @@
"libmediametrics",
"libmediautils",
"libpermission",
+ "libPlatformProperties",
"libsensor",
"libsensorprivacy",
"libshmemcompat",
@@ -42,6 +43,7 @@
"audiopolicy-aidl-cpp",
"audiopolicy-types-aidl-cpp",
"capture_state_listener-aidl-cpp",
+ "com.android.media.audio-aconfig-cc",
"framework-permission-aidl-cpp",
"packagemanager_aidl-cpp",
"spatializer-aidl-cpp",
diff --git a/services/audiopolicy/service/Spatializer.cpp b/services/audiopolicy/service/Spatializer.cpp
index 7859c2c..90418a5 100644
--- a/services/audiopolicy/service/Spatializer.cpp
+++ b/services/audiopolicy/service/Spatializer.cpp
@@ -27,7 +27,9 @@
#include <sys/types.h>
#include <android/content/AttributionSourceState.h>
+#include <android/sysprop/BluetoothProperties.sysprop.h>
#include <audio_utils/fixedfft.h>
+#include <com_android_media_audio.h>
#include <cutils/bitops.h>
#include <hardware/sensors.h>
#include <media/stagefright/foundation/AHandler.h>
@@ -213,6 +215,38 @@
Spatializer::EngineCallbackHandler::kRotation2Key,
};
+// Mapping table between strings read form property bluetooth.core.le.dsa_transport_preference
+// and low latency modes emums.
+//TODO b/273373363: use AIDL enum when available
+const std::map<std::string, audio_latency_mode_t> Spatializer::sStringToLatencyModeMap = {
+ {"le-acl", AUDIO_LATENCY_MODE_LOW},
+ {"iso-sw", AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_SOFTWARE},
+ {"iso-hw", AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_HARDWARE},
+};
+
+void Spatializer::loadOrderedLowLatencyModes() {
+ if (!com::android::media::audio::dsa_over_bt_le_audio()) {
+ return;
+ }
+ auto latencyModesStrs = android::sysprop::BluetoothProperties::dsa_transport_preference();
+ std::lock_guard lock(mLock);
+ // First load preferred low latency modes ordered from the property
+ for (auto str : latencyModesStrs) {
+ if (!str.has_value()) continue;
+ if (auto it = sStringToLatencyModeMap.find(str.value());
+ it != sStringToLatencyModeMap.end()) {
+ mOrderedLowLatencyModes.push_back(it->second);
+ }
+ }
+ // Then add unlisted latency modes at the end of the ordered list
+ for (auto it : sStringToLatencyModeMap) {
+ if (std::find(mOrderedLowLatencyModes.begin(), mOrderedLowLatencyModes.end(), it.second)
+ == mOrderedLowLatencyModes.end()) {
+ mOrderedLowLatencyModes.push_back(it.second);
+ }
+ }
+}
+
// ---------------------------------------------------------------------------
sp<Spatializer> Spatializer::create(SpatializerPolicyCallback* callback,
const sp<EffectsFactoryHalInterface>& effectsFactoryHal) {
@@ -244,6 +278,7 @@
spatializer.clear();
ALOGW("%s loadEngine error: %d effect %p", __func__, status, effect.get());
} else {
+ spatializer->loadOrderedLowLatencyModes();
spatializer->mLocalLog.log("%s with effect Id %p", __func__, effect.get());
}
}
@@ -371,6 +406,30 @@
return BAD_VALUE;
}
+ //TODO b/273373363: use AIDL enum when available
+ if (com::android::media::audio::dsa_over_bt_le_audio()
+ && mSupportsHeadTracking) {
+ mHeadtrackingConnectionMode = HEADTRACKING_CONNECTION_FRAMEWORK_PROCESSED;
+ std::vector<uint8_t> headtrackingConnectionModes;
+ status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_HEADTRACKING_CONNECTION,
+ &headtrackingConnectionModes);
+ if (status == NO_ERROR) {
+ for (const auto htConnectionMode : headtrackingConnectionModes) {
+ if (htConnectionMode < HEADTRACKING_CONNECTION_FRAMEWORK_PROCESSED ||
+ htConnectionMode > HEADTRACKING_CONNECTION_DIRECT_TO_SENSOR_TUNNEL) {
+ ALOGW("%s: ignoring HT connection mode:%d", __func__, (int)htConnectionMode);
+ continue;
+ }
+ mSupportedHeadtrackingConnectionModes.insert(
+ static_cast<headtracking_connection_t> (htConnectionMode));
+ }
+ ALOGW_IF(mSupportedHeadtrackingConnectionModes.find(
+ HEADTRACKING_CONNECTION_FRAMEWORK_PROCESSED)
+ == mSupportedHeadtrackingConnectionModes.end(),
+ "%s: HEADTRACKING_CONNECTION_FRAMEWORK_PROCESSED not reported", __func__);
+ }
+ }
+
// Currently we expose only RELATIVE_WORLD.
// This is a limitation of the head tracking library based on a UX choice.
mHeadTrackingModes.push_back(HeadTracking::Mode::DISABLED);
@@ -831,6 +890,7 @@
} else {
setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
std::vector<HeadTracking::Mode>{spatializerMode});
+ setEngineHeadtrackingConnectionMode_l();
}
}
callback = mHeadTrackingCallback;
@@ -841,6 +901,32 @@
}
}
+void Spatializer::setEngineHeadtrackingConnectionMode_l() {
+ if (!com::android::media::audio::dsa_over_bt_le_audio()) {
+ return;
+ }
+ if (mActualHeadTrackingMode != HeadTracking::Mode::DISABLED
+ && !mSupportedHeadtrackingConnectionModes.empty()) {
+ setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_CONNECTION,
+ static_cast<uint8_t>(mHeadtrackingConnectionMode),
+ static_cast<uint32_t>(mHeadSensor));
+ }
+}
+
+void Spatializer::sortSupportedLatencyModes_l() {
+ if (!com::android::media::audio::dsa_over_bt_le_audio()) {
+ return;
+ }
+ std::sort(mSupportedLatencyModes.begin(), mSupportedLatencyModes.end(),
+ [this](audio_latency_mode_t x, audio_latency_mode_t y) {
+ auto itX = std::find(mOrderedLowLatencyModes.begin(),
+ mOrderedLowLatencyModes.end(), x);
+ auto itY = std::find(mOrderedLowLatencyModes.begin(),
+ mOrderedLowLatencyModes.end(), y);
+ return itX < itY;
+ });
+}
+
status_t Spatializer::attachOutput(audio_io_handle_t output, size_t numActiveTracks) {
bool outputChanged = false;
sp<media::INativeSpatializerCallback> callback;
@@ -881,6 +967,7 @@
status = AudioSystem::getSupportedLatencyModes(mOutput, &latencyModes);
if (status == OK) {
mSupportedLatencyModes = latencyModes;
+ sortSupportedLatencyModes_l();
}
checkEngineState_l();
@@ -950,6 +1037,7 @@
__func__, (int)output, (int)mOutput, modes.size());
if (output == mOutput) {
mSupportedLatencyModes = std::move(modes);
+ sortSupportedLatencyModes_l();
checkSensorsState_l();
}
}
@@ -964,26 +1052,75 @@
}
}
+//TODO b/273373363: use AIDL enum when available
+audio_latency_mode_t Spatializer::selectHeadtrackingConnectionMode_l() {
+ if (!com::android::media::audio::dsa_over_bt_le_audio()) {
+ return AUDIO_LATENCY_MODE_LOW;
+ }
+ // mSupportedLatencyModes is ordered according to system preferences loaded in
+ // mOrderedLowLatencyModes
+ mHeadtrackingConnectionMode = HEADTRACKING_CONNECTION_FRAMEWORK_PROCESSED;
+ audio_latency_mode_t requestedLatencyMode = mSupportedLatencyModes[0];
+ if (requestedLatencyMode == AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_HARDWARE) {
+ if (mSupportedHeadtrackingConnectionModes.find(
+ HEADTRACKING_CONNECTION_DIRECT_TO_SENSOR_TUNNEL)
+ != mSupportedHeadtrackingConnectionModes.end()) {
+ mHeadtrackingConnectionMode = HEADTRACKING_CONNECTION_DIRECT_TO_SENSOR_TUNNEL;
+ } else if (mSupportedHeadtrackingConnectionModes.find(
+ HEADTRACKING_CONNECTION_DIRECT_TO_SENSOR_SW)
+ != mSupportedHeadtrackingConnectionModes.end()) {
+ mHeadtrackingConnectionMode = HEADTRACKING_CONNECTION_DIRECT_TO_SENSOR_SW;
+ } else {
+ // if the engine does not support direct reading of IMU data, do not allow
+ // DYNAMIC_SPATIAL_AUDIO_HARDWARE mode and fallback to next mode
+ if (mSupportedLatencyModes.size() > 1) {
+ requestedLatencyMode = mSupportedLatencyModes[1];
+ } else {
+ // If only DYNAMIC_SPATIAL_AUDIO_HARDWARE mode is reported by the
+ // HAL and the engine does not support it, assert as this is a
+ // product configuration error
+ LOG_ALWAYS_FATAL("%s: the audio HAL reported only low latency with"
+ "HW HID tunneling but the spatializer does not support it",
+ __func__);
+ }
+ }
+ }
+ return requestedLatencyMode;
+}
+
void Spatializer::checkSensorsState_l() {
audio_latency_mode_t requestedLatencyMode = AUDIO_LATENCY_MODE_FREE;
const bool supportsSetLatencyMode = !mSupportedLatencyModes.empty();
- const bool supportsLowLatencyMode = supportsSetLatencyMode && std::find(
+ bool supportsLowLatencyMode;
+ if (com::android::media::audio::dsa_over_bt_le_audio()) {
+ // mSupportedLatencyModes is ordered with MODE_FREE always at the end:
+ // the first entry is never MODE_FREE if at least one low ltency mode is supported.
+ supportsLowLatencyMode = supportsSetLatencyMode
+ && mSupportedLatencyModes[0] != AUDIO_LATENCY_MODE_FREE;
+ } else {
+ supportsLowLatencyMode = supportsSetLatencyMode && std::find(
mSupportedLatencyModes.begin(), mSupportedLatencyModes.end(),
AUDIO_LATENCY_MODE_LOW) != mSupportedLatencyModes.end();
+ }
if (mSupportsHeadTracking) {
if (mPoseController != nullptr) {
// TODO(b/253297301, b/255433067) reenable low latency condition check
// for Head Tracking after Bluetooth HAL supports it correctly.
if (mNumActiveTracks > 0 && mLevel != Spatialization::Level::NONE
- && mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
- && mHeadSensor != SpatializerPoseController::INVALID_SENSOR) {
+ && mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
+ && mHeadSensor != SpatializerPoseController::INVALID_SENSOR) {
+ if (supportsLowLatencyMode) {
+ requestedLatencyMode = selectHeadtrackingConnectionMode_l();
+ }
if (mEngine != nullptr) {
setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
std::vector<HeadTracking::Mode>{mActualHeadTrackingMode});
+ setEngineHeadtrackingConnectionMode_l();
}
+ // TODO: b/307588546: configure mPoseController according to selected
+ // mHeadtrackingConnectionMode
mPoseController->setHeadSensor(mHeadSensor);
mPoseController->setScreenSensor(mScreenSensor);
- if (supportsLowLatencyMode) requestedLatencyMode = AUDIO_LATENCY_MODE_LOW;
} else {
mPoseController->setHeadSensor(SpatializerPoseController::INVALID_SENSOR);
mPoseController->setScreenSensor(SpatializerPoseController::INVALID_SENSOR);
diff --git a/services/audiopolicy/service/Spatializer.h b/services/audiopolicy/service/Spatializer.h
index 4ef07ce..123517e 100644
--- a/services/audiopolicy/service/Spatializer.h
+++ b/services/audiopolicy/service/Spatializer.h
@@ -32,6 +32,7 @@
#include <media/stagefright/foundation/ALooper.h>
#include <system/audio_effects/effect_spatializer.h>
#include <string>
+#include <unordered_set>
#include "SpatializerPoseController.h"
@@ -281,6 +282,33 @@
}
/**
+ * Set a parameter to spatializer engine by calling setParameter on mEngine AudioEffect object.
+ * The variant is for compound parameters with two values of different base types
+ */
+ template<typename P1, typename P2>
+ status_t setEffectParameter_l(uint32_t type, const P1 val1, const P2 val2) REQUIRES(mLock) {
+ static_assert(sizeof(P1) <= sizeof(uint32_t), "The size of P1 must less than 32 bits");
+ static_assert(sizeof(P2) <= sizeof(uint32_t), "The size of P2 must less than 32 bits");
+
+ uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 3];
+ effect_param_t *p = (effect_param_t *)cmd;
+ p->psize = sizeof(uint32_t);
+ p->vsize = 2 * sizeof(uint32_t);
+ *(uint32_t *)p->data = type;
+ *((uint32_t *)p->data + 1) = static_cast<uint32_t>(val1);
+ *((uint32_t *)p->data + 2) = static_cast<uint32_t>(val2);
+
+ status_t status = mEngine->setParameter(p);
+ if (status != NO_ERROR) {
+ return status;
+ }
+ if (p->status != NO_ERROR) {
+ return p->status;
+ }
+ return NO_ERROR;
+ }
+
+ /**
* Get a parameter from spatializer engine by calling getParameter on AudioEffect object.
* It is possible to read more than one value of type T according to the parameter type
* by specifying values vector size.
@@ -312,6 +340,34 @@
return NO_ERROR;
}
+ /**
+ * Get a parameter from spatializer engine by calling getParameter on AudioEffect object.
+ * The variant is for compound parameters with two values of different base types
+ */
+ template<typename P1, typename P2>
+ status_t getEffectParameter_l(uint32_t type, P1 *val1, P2 *val2) REQUIRES(mLock) {
+ static_assert(sizeof(P1) <= sizeof(uint32_t), "The size of P1 must less than 32 bits");
+ static_assert(sizeof(P2) <= sizeof(uint32_t), "The size of P2 must less than 32 bits");
+
+ uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 3];
+ effect_param_t *p = (effect_param_t *)cmd;
+ p->psize = sizeof(uint32_t);
+ p->vsize = 2 * sizeof(uint32_t);
+ *(uint32_t *)p->data = type;
+
+ status_t status = mEngine->getParameter(p);
+
+ if (status != NO_ERROR) {
+ return status;
+ }
+ if (p->status != NO_ERROR) {
+ return p->status;
+ }
+ *val1 = static_cast<P1>(*((uint32_t *)p->data + 1));
+ *val2 = static_cast<P2>(*((uint32_t *)p->data + 2));
+ return NO_ERROR;
+ }
+
virtual void onFramesProcessed(int32_t framesProcessed) override;
/**
@@ -339,6 +395,35 @@
*/
void resetEngineHeadPose_l() REQUIRES(mLock);
+ /** Read bluetooth.core.le.dsa_transport_preference property and populate the ordered list of
+ * preferred low latency modes in mOrderedLowLatencyModes.
+ */
+ void loadOrderedLowLatencyModes();
+
+ /**
+ * Sort mSupportedLatencyModes list according to the preference order stored in
+ * mOrderedLowLatencyModes.
+ * Note: Because MODE_FREE is not in mOrderedLowLatencyModes, it will always be at
+ * the end of the list.
+ */
+ void sortSupportedLatencyModes_l() REQUIRES(mLock);
+
+ /**
+ * Called after enabling head tracking in the spatializer engine to indicate which
+ * connection mode should be used among those supported. The selection depends on
+ * currently supported latency modes reported by the audio HAL.
+ * When the connection mode is direct to the sensor, the sensor ID is also communicated
+ * to the spatializer engine.
+ */
+ void setEngineHeadtrackingConnectionMode_l() REQUIRES(mLock);
+
+ /**
+ * Select the desired head tracking connection mode for the spatializer engine among the list
+ * stored in mSupportedHeadtrackingConnectionModes at init time.
+ * Also returns the desired low latency mode according to selected connection mode.
+ */
+ audio_latency_mode_t selectHeadtrackingConnectionMode_l() REQUIRES(mLock);
+
/** Effect engine descriptor */
const effect_descriptor_t mEngineDescriptor;
/** Callback interface to parent audio policy service */
@@ -398,6 +483,13 @@
std::vector<media::audio::common::Spatialization::Mode> mSpatializationModes;
std::vector<audio_channel_mask_t> mChannelMasks;
bool mSupportsHeadTracking;
+ /** List of supported headtracking connection modes reported by the spatializer.
+ * If the list is empty, the spatializer does not support any optional connection
+ * mode and mode HEADTRACKING_CONNECTION_FRAMEWORK_PROCESSED is assumed.
+ */
+ std::unordered_set<headtracking_connection_t> mSupportedHeadtrackingConnectionModes;
+ /** Selected HT connection mode when several modes are supported by the spatializer */
+ headtracking_connection_t mHeadtrackingConnectionMode;
// Looper thread for mEngine callbacks
class EngineCallbackHandler;
@@ -407,7 +499,10 @@
size_t mNumActiveTracks GUARDED_BY(mLock) = 0;
std::vector<audio_latency_mode_t> mSupportedLatencyModes GUARDED_BY(mLock);
-
+ /** preference order for low latency modes according to persist.bluetooth.hid.transport */
+ std::vector<audio_latency_mode_t> mOrderedLowLatencyModes;
+ /** string to latency mode map used to parse bluetooth.core.le.dsa_transport_preference */
+ static const std::map<std::string, audio_latency_mode_t> sStringToLatencyModeMap;
static const std::vector<const char*> sHeadPoseKeys;
// Local log for command messages.
diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp
index 0a13ab2..fe8ac1b 100644
--- a/services/camera/libcameraservice/Android.bp
+++ b/services/camera/libcameraservice/Android.bp
@@ -66,6 +66,7 @@
"libmedia_codeclist",
"libmedia_omx",
"libmemunreachable",
+ "libnativewindow",
"libprocessgroup",
"libprocinfo",
"libsensorprivacy",
@@ -73,6 +74,7 @@
"libstagefright_foundation",
"libxml2",
"libyuv",
+ "android.companion.virtualdevice.flags-aconfig-cc",
"android.hardware.camera.common@1.0",
"android.hardware.camera.device@1.0",
"android.hardware.camera.device@3.2",
@@ -95,8 +97,8 @@
"android.frameworks.cameraservice.device@2.0",
"android.frameworks.cameraservice.device@2.1",
"android.frameworks.cameraservice.common-V1-ndk",
- "android.frameworks.cameraservice.service-V1-ndk",
- "android.frameworks.cameraservice.device-V1-ndk",
+ "android.frameworks.cameraservice.service-V2-ndk",
+ "android.frameworks.cameraservice.device-V2-ndk",
"android.hardware.camera.common-V1-ndk",
"android.hardware.camera.device-V2-ndk",
"android.hardware.camera.metadata-V2-ndk",
diff --git a/services/camera/libcameraservice/aidl/AidlUtils.cpp b/services/camera/libcameraservice/aidl/AidlUtils.cpp
index 2225cfe..f5d68eb 100644
--- a/services/camera/libcameraservice/aidl/AidlUtils.cpp
+++ b/services/camera/libcameraservice/aidl/AidlUtils.cpp
@@ -73,24 +73,49 @@
UOutputConfiguration convertFromAidl(const SOutputConfiguration &src) {
std::vector<sp<IGraphicBufferProducer>> iGBPs;
- auto &windowHandles = src.windowHandles;
- iGBPs.reserve(windowHandles.size());
+ if (!src.surfaces.empty()) {
+ auto& surfaces = src.surfaces;
+ iGBPs.reserve(surfaces.size());
- for (auto &handle : windowHandles) {
- native_handle_t* nh = makeFromAidl(handle);
- auto igbp = AImageReader_getHGBPFromHandle(nh);
- if (igbp == nullptr) {
- ALOGE("%s: Could not get HGBP from NativeHandle: %s. Skipping.",
- __FUNCTION__, handle.toString().c_str());
- continue;
+ for (auto& sSurface : surfaces) {
+ sp<IGraphicBufferProducer> igbp =
+ Surface::getIGraphicBufferProducer(sSurface.get());
+ if (igbp == nullptr) {
+ ALOGE("%s: ANativeWindow (%p) not backed by a Surface.",
+ __FUNCTION__, sSurface.get());
+ continue;
+ }
+ iGBPs.push_back(igbp);
}
- iGBPs.push_back(new H2BGraphicBufferProducer(igbp));
- native_handle_delete(nh);
+ } else {
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+ // HIDL token manager (and consequently 'windowHandles') is deprecated and will be removed
+ // in the future. However, cameraservice must still support old NativeHandle pathway until
+ // all vendors have moved away from using NativeHandles
+ auto &windowHandles = src.windowHandles;
+#pragma clang diagnostic pop
+
+ iGBPs.reserve(windowHandles.size());
+
+ for (auto &handle : windowHandles) {
+ native_handle_t* nh = makeFromAidl(handle);
+ auto igbp = AImageReader_getHGBPFromHandle(nh);
+ if (igbp == nullptr) {
+ ALOGE("%s: Could not get HGBP from NativeHandle: %s. Skipping.",
+ __FUNCTION__, handle.toString().c_str());
+ continue;
+ }
+
+ iGBPs.push_back(new H2BGraphicBufferProducer(igbp));
+ native_handle_delete(nh);
+ }
}
+
UOutputConfiguration outputConfiguration(
iGBPs, convertFromAidl(src.rotation), src.physicalCameraId,
src.windowGroupId, OutputConfiguration::SURFACE_TYPE_UNKNOWN, 0, 0,
- (windowHandles.size() > 1));
+ (iGBPs.size() > 1));
return outputConfiguration;
}
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp
index 54dc2bc..adcb523 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.cpp
+++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp
@@ -32,6 +32,7 @@
#include <dlfcn.h>
#include <future>
#include <inttypes.h>
+#include <android_companion_virtualdevice_flags.h>
#include <android/binder_manager.h>
#include <android/hidl/manager/1.2/IServiceManager.h>
#include <hidl/ServiceManagement.h>
@@ -59,6 +60,7 @@
using hardware::camera2::utils::CameraIdAndSessionConfiguration;
namespace flags = com::android::internal::camera::flags;
+namespace vd_flags = android::companion::virtualdevice::flags;
namespace {
const bool kEnableLazyHal(property_get_bool("ro.camera.enableLazyHal", false));
@@ -3162,7 +3164,7 @@
}
bool CameraProviderManager::isVirtualCameraHalEnabled() {
- return flags::virtual_camera_service_discovery();
+ return vd_flags::virtual_camera_service_discovery();
}
} // namespace android
diff --git a/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp b/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp
index 257103f..c2b8bf4 100644
--- a/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp
+++ b/services/camera/libcameraservice/common/aidl/AidlProviderInfo.cpp
@@ -21,6 +21,7 @@
#include <cutils/properties.h>
#include <aidlcommonsupport/NativeHandle.h>
+#include <android_companion_virtualdevice_flags.h>
#include <android/binder_manager.h>
#include <android/hardware/ICameraService.h>
#include <camera_metadata_hidden.h>
@@ -35,9 +36,9 @@
namespace android {
-namespace flags = com::android::internal::camera::flags;
namespace SessionConfigurationUtils = ::android::camera3::SessionConfigurationUtils;
namespace flags = com::android::internal::camera::flags;
+namespace vd_flags = android::companion::virtualdevice::flags;
using namespace aidl::android::hardware;
using namespace hardware::camera;
@@ -131,7 +132,7 @@
mDeathRecipient = ndk::ScopedAIBinder_DeathRecipient(AIBinder_DeathRecipient_new(binderDied));
- if (!flags::virtual_camera_service_discovery() || interface->isRemote()) {
+ if (!vd_flags::virtual_camera_service_discovery() || interface->isRemote()) {
binder_status_t link =
AIBinder_linkToDeath(interface->asBinder().get(), mDeathRecipient.get(), this);
if (link != STATUS_OK) {
diff --git a/services/camera/libcameraservice/libcameraservice_fuzzer/Android.bp b/services/camera/libcameraservice/libcameraservice_fuzzer/Android.bp
index 921ad7d..efc58b4 100644
--- a/services/camera/libcameraservice/libcameraservice_fuzzer/Android.bp
+++ b/services/camera/libcameraservice/libcameraservice_fuzzer/Android.bp
@@ -60,15 +60,21 @@
],
fuzz_config: {
cc: [
- "android-media-fuzzing-reports@google.com",
"android-camera-fwk-eng@google.com",
],
- componentid: 155276,
+ componentid: 41727,
libfuzzer_options: [
//based on b/187360866
"timeout=770",
],
-
+ hotlists: [
+ "4593311",
+ ],
+ description: "The fuzzer targets the APIs of libcameraservice",
+ vector: "local_no_privileges_required",
+ service_privilege: "privileged",
+ users: "multi_user",
+ fuzzed_code_usage: "shipped",
},
}
diff --git a/services/camera/libcameraservice/tests/Android.bp b/services/camera/libcameraservice/tests/Android.bp
index 60e1a88..e9ecc86 100644
--- a/services/camera/libcameraservice/tests/Android.bp
+++ b/services/camera/libcameraservice/tests/Android.bp
@@ -47,6 +47,7 @@
"libutils",
"libjpeg",
"libexif",
+ "android.companion.virtualdevice.flags-aconfig-cc",
"android.hardware.camera.common@1.0",
"android.hardware.camera.device@1.0",
"android.hardware.camera.device@3.2",
diff --git a/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp b/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp
index 151c5ce..afc8b94 100644
--- a/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp
+++ b/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp
@@ -20,6 +20,7 @@
#include "../common/CameraProviderManager.h"
#include <aidl/android/hardware/camera/device/BnCameraDevice.h>
#include <aidl/android/hardware/camera/provider/BnCameraProvider.h>
+#include <android_companion_virtualdevice_flags.h>
#include <android/binder_auto_utils.h>
#include <android/binder_ibinder.h>
#include <android/binder_interface_utils.h>
@@ -52,6 +53,7 @@
using ::testing::ElementsAre;
namespace flags = com::android::internal::camera::flags;
+namespace vd_flags = android::companion::virtualdevice::flags;
/**
* Basic test implementation of a camera ver. 3.2 device interface
@@ -836,7 +838,7 @@
<< "Unavailable physical camera Ids not set properly.";
}
TEST_WITH_FLAGS(CameraProviderManagerTest, AidlVirtualCameraProviderDiscovered,
- REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(flags, virtual_camera_service_discovery))) {
+ REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(vd_flags, virtual_camera_service_discovery))) {
sp<CameraProviderManager> providerManager = new CameraProviderManager();
sp<TestStatusListener> statusListener = new TestStatusListener();
TestInteractionProxy serviceProxy;
@@ -862,7 +864,7 @@
}
TEST_WITH_FLAGS(CameraProviderManagerTest, AidlVirtualCameraProviderDiscoveredOnInit,
- REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(flags, virtual_camera_service_discovery))) {
+ REQUIRES_FLAGS_ENABLED(ACONFIG_FLAG(vd_flags, virtual_camera_service_discovery))) {
sp<CameraProviderManager> providerManager = new CameraProviderManager();
sp<TestStatusListener> statusListener = new TestStatusListener();
TestInteractionProxy serviceProxy;
diff --git a/services/camera/virtualcamera/.clang-format b/services/camera/virtualcamera/.clang-format
new file mode 100644
index 0000000..f397454
--- /dev/null
+++ b/services/camera/virtualcamera/.clang-format
@@ -0,0 +1,12 @@
+BasedOnStyle: Google
+AllowShortBlocksOnASingleLine: false
+AllowShortFunctionsOnASingleLine: false
+
+ColumnLimit: 80
+ContinuationIndentWidth: 4
+CommentPragmas: NOLINT:.*
+DerivePointerAlignment: false
+IndentWidth: 2
+PointerAlignment: Left
+UseTab: Never
+PenaltyExcessCharacter: 32
\ No newline at end of file
diff --git a/services/camera/virtualcamera/Android.bp b/services/camera/virtualcamera/Android.bp
new file mode 100644
index 0000000..c8fa84e
--- /dev/null
+++ b/services/camera/virtualcamera/Android.bp
@@ -0,0 +1,95 @@
+package {
+ // See: http://go/android-license-faq
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+cc_defaults {
+ name: "libvirtualcamera_defaults",
+ shared_libs: [
+ "android.hardware.common-V2-ndk",
+ "android.hardware.common.fmq-V1-ndk",
+ "libbinder",
+ "libbinder_ndk",
+ "libcamera_metadata",
+ "liblog",
+ "libfmq",
+ "libgui",
+ "libjpeg",
+ "libnativewindow",
+ "libbase",
+ "libcutils",
+ "libui",
+ "libutils",
+ "libEGL",
+ "libGLESv2",
+ "libGLESv3",
+ ],
+ static_libs: [
+ "android.hardware.camera.common@1.0-helper",
+ "android.hardware.camera.common-V1-ndk",
+ "android.hardware.camera.device-V2-ndk",
+ "android.hardware.camera.metadata-V2-ndk",
+ "android.hardware.camera.provider-V2-ndk",
+ "libaidlcommonsupport",
+ "virtual_camera_service_aidl-ndk",
+ ],
+ cflags: [
+ "-DANDROID_UTILS_REF_BASE_DISABLE_IMPLICIT_CONSTRUCTION",
+ "-Wall",
+ "-Werror",
+ "-Wformat",
+ "-Wthread-safety",
+ ],
+}
+
+cc_library_static {
+ name: "libvirtualcamera_utils",
+ srcs: [
+ "util/JpegUtil.cc",
+ "util/MetadataBuilder.cc",
+ "util/Util.cc",
+ "util/TestPatternHelper.cc",
+ "util/EglDisplayContext.cc",
+ "util/EglFramebuffer.cc",
+ "util/EglProgram.cc",
+ "util/EglSurfaceTexture.cc",
+ "util/EglUtil.cc",
+ ],
+ defaults: [
+ "libvirtualcamera_defaults",
+ ],
+}
+
+cc_library_static {
+ name: "libvirtualcamera",
+ srcs: [
+ "VirtualCameraProvider.cc",
+ "VirtualCameraDevice.cc",
+ "VirtualCameraSession.cc",
+ "VirtualCameraStream.cc",
+ "VirtualCameraService.cc",
+ "VirtualCameraSessionContext.cc",
+ "VirtualCameraRenderThread.cc",
+ ],
+ defaults: [
+ "libvirtualcamera_defaults",
+ ],
+ static_libs: [
+ "libvirtualcamera_utils",
+ ],
+ export_include_dirs: ["."],
+ min_sdk_version: "current",
+}
+
+cc_binary {
+ name: "virtual_camera",
+ srcs: ["main.cc"],
+ defaults: [
+ "libvirtualcamera_defaults",
+ ],
+ static_libs: [
+ "libvirtualcamera",
+ "libvirtualcamera_utils",
+ ],
+ init_rc: ["virtual_camera.hal.rc"],
+}
diff --git a/services/camera/virtualcamera/OWNERS b/services/camera/virtualcamera/OWNERS
new file mode 100644
index 0000000..db34336
--- /dev/null
+++ b/services/camera/virtualcamera/OWNERS
@@ -0,0 +1,4 @@
+# Bug component: 1171888
+include platform/frameworks/base:/services/companion/java/com/android/server/companion/virtual/OWNERS
+caen@google.com
+jsebechlebsky@google.com
diff --git a/services/camera/virtualcamera/TEST_MAPPING b/services/camera/virtualcamera/TEST_MAPPING
new file mode 100644
index 0000000..66c5e52
--- /dev/null
+++ b/services/camera/virtualcamera/TEST_MAPPING
@@ -0,0 +1,17 @@
+{
+ "presubmit" : [
+ {
+ "name": "virtual_camera_tests"
+ }
+ ],
+ "postsubmit": [
+ {
+ "name": "CtsVirtualDevicesCameraTestCases",
+ "options": [
+ {
+ "exclude-annotation": "androidx.test.filters.FlakyTest"
+ }
+ ]
+ }
+ ]
+}
diff --git a/services/camera/virtualcamera/VirtualCameraDevice.cc b/services/camera/virtualcamera/VirtualCameraDevice.cc
new file mode 100644
index 0000000..f5fa16e
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraDevice.cc
@@ -0,0 +1,371 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "VirtualCameraDevice"
+#include "VirtualCameraDevice.h"
+
+#include <algorithm>
+#include <array>
+#include <chrono>
+#include <cstdint>
+#include <iterator>
+#include <optional>
+#include <string>
+
+#include "VirtualCameraSession.h"
+#include "aidl/android/companion/virtualcamera/SupportedStreamConfiguration.h"
+#include "aidl/android/hardware/camera/common/Status.h"
+#include "aidl/android/hardware/camera/device/CameraMetadata.h"
+#include "aidl/android/hardware/camera/device/StreamConfiguration.h"
+#include "android/binder_auto_utils.h"
+#include "android/binder_status.h"
+#include "log/log.h"
+#include "system/camera_metadata.h"
+#include "util/MetadataBuilder.h"
+#include "util/Util.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+using ::aidl::android::companion::virtualcamera::Format;
+using ::aidl::android::companion::virtualcamera::IVirtualCameraCallback;
+using ::aidl::android::companion::virtualcamera::SupportedStreamConfiguration;
+using ::aidl::android::hardware::camera::common::CameraResourceCost;
+using ::aidl::android::hardware::camera::common::Status;
+using ::aidl::android::hardware::camera::device::CameraMetadata;
+using ::aidl::android::hardware::camera::device::ICameraDeviceCallback;
+using ::aidl::android::hardware::camera::device::ICameraDeviceSession;
+using ::aidl::android::hardware::camera::device::ICameraInjectionSession;
+using ::aidl::android::hardware::camera::device::Stream;
+using ::aidl::android::hardware::camera::device::StreamConfiguration;
+using ::aidl::android::hardware::camera::device::StreamRotation;
+using ::aidl::android::hardware::camera::device::StreamType;
+using ::aidl::android::hardware::graphics::common::PixelFormat;
+
+namespace {
+
+using namespace std::chrono_literals;
+
+// Prefix of camera name - "device@1.1/virtual/{numerical_id}"
+const char* kDevicePathPrefix = "device@1.1/virtual/";
+
+constexpr std::chrono::nanoseconds kMinFrameDuration30Fps = 1s / 30;
+constexpr int32_t kMaxJpegSize = 3 * 1024 * 1024 /*3MiB*/;
+
+constexpr MetadataBuilder::ControlRegion kDefaultEmptyControlRegion{};
+
+struct Resolution {
+ Resolution(const int w, const int h) : width(w), height(h) {
+ }
+
+ bool operator<(const Resolution& other) const {
+ return width * height < other.width * other.height;
+ }
+
+ bool operator==(const Resolution& other) const {
+ return width == other.width && height == other.height;
+ }
+
+ const int width;
+ const int height;
+};
+
+std::optional<Resolution> getMaxResolution(
+ const std::vector<SupportedStreamConfiguration>& configs) {
+ auto itMax = std::max_element(configs.begin(), configs.end(),
+ [](const SupportedStreamConfiguration& a,
+ const SupportedStreamConfiguration& b) {
+ return a.width * b.height < a.width * b.height;
+ });
+ if (itMax == configs.end()) {
+ ALOGE(
+ "%s: empty vector of supported configurations, cannot find largest "
+ "resolution.",
+ __func__);
+ return std::nullopt;
+ }
+
+ return Resolution(itMax->width, itMax->height);
+}
+
+std::set<Resolution> getUniqueResolutions(
+ const std::vector<SupportedStreamConfiguration>& configs) {
+ std::set<Resolution> uniqueResolutions;
+ std::transform(configs.begin(), configs.end(),
+ std::inserter(uniqueResolutions, uniqueResolutions.begin()),
+ [](const SupportedStreamConfiguration& config) {
+ return Resolution(config.width, config.height);
+ });
+ return uniqueResolutions;
+}
+
+// TODO(b/301023410) - Populate camera characteristics according to camera configuration.
+std::optional<CameraMetadata> initCameraCharacteristics(
+ const std::vector<SupportedStreamConfiguration>& supportedInputConfig) {
+ if (!std::all_of(supportedInputConfig.begin(), supportedInputConfig.end(),
+ [](const SupportedStreamConfiguration& config) {
+ return config.pixelFormat == Format::YUV_420_888;
+ })) {
+ ALOGE("%s: input configuration contains unsupported pixel format", __func__);
+ return std::nullopt;
+ }
+
+ MetadataBuilder builder =
+ MetadataBuilder()
+ .setSupportedHardwareLevel(
+ ANDROID_INFO_SUPPORTED_HARDWARE_LEVEL_EXTERNAL)
+ .setFlashAvailable(false)
+ .setLensFacing(ANDROID_LENS_FACING_EXTERNAL)
+ .setSensorOrientation(0)
+ .setAvailableFaceDetectModes({ANDROID_STATISTICS_FACE_DETECT_MODE_OFF})
+ .setAvailableMaxDigitalZoom(1.0)
+ .setControlAvailableModes({ANDROID_CONTROL_MODE_AUTO})
+ .setControlAfAvailableModes({ANDROID_CONTROL_AF_MODE_OFF})
+ .setControlAeAvailableFpsRange(10, 30)
+ .setControlMaxRegions(0, 0, 0)
+ .setControlAfRegions({kDefaultEmptyControlRegion})
+ .setControlAeRegions({kDefaultEmptyControlRegion})
+ .setControlAwbRegions({kDefaultEmptyControlRegion})
+ .setControlAeCompensationRange(0, 1)
+ .setControlAeCompensationStep(camera_metadata_rational_t{0, 1})
+ .setMaxJpegSize(kMaxJpegSize)
+ .setAvailableRequestKeys({ANDROID_CONTROL_AF_MODE})
+ .setAvailableResultKeys({ANDROID_CONTROL_AF_MODE})
+ .setAvailableCapabilities(
+ {ANDROID_REQUEST_AVAILABLE_CAPABILITIES_BACKWARD_COMPATIBLE})
+ .setAvailableCharacteristicKeys();
+
+ // Active array size must correspond to largest supported input resolution.
+ std::optional<Resolution> maxResolution =
+ getMaxResolution(supportedInputConfig);
+ if (!maxResolution.has_value()) {
+ return std::nullopt;
+ }
+ builder.setSensorActiveArraySize(0, 0, maxResolution->width,
+ maxResolution->height);
+
+ std::vector<MetadataBuilder::StreamConfiguration> outputConfigurations;
+
+ // TODO(b/301023410) Add also all "standard" resolutions we can rescale the
+ // streams to (all standard resolutions with same aspect ratio).
+
+ // Add IMPLEMENTATION_DEFINED format for all supported input resolutions.
+ std::set<Resolution> uniqueResolutions =
+ getUniqueResolutions(supportedInputConfig);
+ std::transform(
+ uniqueResolutions.begin(), uniqueResolutions.end(),
+ std::back_inserter(outputConfigurations),
+ [](const Resolution& resolution) {
+ return MetadataBuilder::StreamConfiguration{
+ .width = resolution.width,
+ .height = resolution.height,
+ .format = ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED,
+ .minFrameDuration = kMinFrameDuration30Fps,
+ .minStallDuration = 0s};
+ });
+
+ // Add all supported configuration with explicit pixel format.
+ std::transform(supportedInputConfig.begin(), supportedInputConfig.end(),
+ std::back_inserter(outputConfigurations),
+ [](const SupportedStreamConfiguration& config) {
+ return MetadataBuilder::StreamConfiguration{
+ .width = config.width,
+ .height = config.height,
+ .format = static_cast<int>(config.pixelFormat),
+ .minFrameDuration = kMinFrameDuration30Fps,
+ .minStallDuration = 0s};
+ });
+
+ // TODO(b/301023410) We currently don't support rescaling for still capture,
+ // so only announce BLOB support for formats exactly matching the input.
+ std::transform(uniqueResolutions.begin(), uniqueResolutions.end(),
+ std::back_inserter(outputConfigurations),
+ [](const Resolution& resolution) {
+ return MetadataBuilder::StreamConfiguration{
+ .width = resolution.width,
+ .height = resolution.height,
+ .format = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB,
+ .minFrameDuration = kMinFrameDuration30Fps,
+ .minStallDuration = 0s};
+ });
+
+ ALOGV("Adding %zu output configurations", outputConfigurations.size());
+ builder.setAvailableOutputStreamConfigurations(outputConfigurations);
+
+ auto metadata = builder.build();
+ if (metadata == nullptr) {
+ ALOGE("Failed to build metadata!");
+ return CameraMetadata();
+ }
+
+ return std::move(*metadata);
+}
+
+} // namespace
+
+VirtualCameraDevice::VirtualCameraDevice(
+ const uint32_t cameraId,
+ const std::vector<SupportedStreamConfiguration>& supportedInputConfig,
+ std::shared_ptr<IVirtualCameraCallback> virtualCameraClientCallback)
+ : mCameraId(cameraId),
+ mVirtualCameraClientCallback(virtualCameraClientCallback),
+ mSupportedInputConfigurations(supportedInputConfig) {
+ std::optional<CameraMetadata> metadata =
+ initCameraCharacteristics(mSupportedInputConfigurations);
+ if (metadata.has_value()) {
+ mCameraCharacteristics = *metadata;
+ } else {
+ ALOGE(
+ "%s: Failed to initialize camera characteristic based on provided "
+ "configuration.",
+ __func__);
+ }
+}
+
+ndk::ScopedAStatus VirtualCameraDevice::getCameraCharacteristics(
+ CameraMetadata* _aidl_return) {
+ ALOGV("%s", __func__);
+ if (_aidl_return == nullptr) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ *_aidl_return = mCameraCharacteristics;
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraDevice::getPhysicalCameraCharacteristics(
+ const std::string& in_physicalCameraId, CameraMetadata* _aidl_return) {
+ ALOGV("%s: physicalCameraId %s", __func__, in_physicalCameraId.c_str());
+ (void)_aidl_return;
+
+ // VTS tests expect this call to fail with illegal argument status for
+ // all publicly advertised camera ids.
+ // Because we don't support physical camera ids, we just always
+ // fail with illegal argument (there's no valid argument to provide).
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+}
+
+ndk::ScopedAStatus VirtualCameraDevice::getResourceCost(
+ CameraResourceCost* _aidl_return) {
+ ALOGV("%s", __func__);
+ if (_aidl_return == nullptr) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+ _aidl_return->resourceCost = 100; // ¯\_(ツ)_/¯
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraDevice::isStreamCombinationSupported(
+ const StreamConfiguration& in_streams, bool* _aidl_return) {
+ ALOGV("%s", __func__);
+
+ if (_aidl_return == nullptr) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ *_aidl_return = isStreamCombinationSupported(in_streams);
+ return ndk::ScopedAStatus::ok();
+};
+
+bool VirtualCameraDevice::isStreamCombinationSupported(
+ const StreamConfiguration& streamConfiguration) const {
+ for (const Stream& stream : streamConfiguration.streams) {
+ ALOGV("%s: Configuration queried: %s", __func__, stream.toString().c_str());
+
+ if (stream.streamType == StreamType::INPUT) {
+ ALOGW("%s: Input stream type is not supported", __func__);
+ return false;
+ }
+
+ // TODO(b/301023410) remove hardcoded format checks, verify against configuration.
+ if (stream.rotation != StreamRotation::ROTATION_0 ||
+ (stream.format != PixelFormat::IMPLEMENTATION_DEFINED &&
+ stream.format != PixelFormat::YCBCR_420_888 &&
+ stream.format != PixelFormat::BLOB)) {
+ ALOGV("Unsupported output stream type");
+ return false;
+ }
+
+ auto matchesSupportedInputConfig =
+ [&stream](const SupportedStreamConfiguration& config) {
+ return stream.width == config.width && stream.height == config.height;
+ };
+ if (std::none_of(mSupportedInputConfigurations.begin(),
+ mSupportedInputConfigurations.end(),
+ matchesSupportedInputConfig)) {
+ ALOGV("Requested config doesn't match any supported input config");
+ return false;
+ }
+ }
+ return true;
+}
+
+ndk::ScopedAStatus VirtualCameraDevice::open(
+ const std::shared_ptr<ICameraDeviceCallback>& in_callback,
+ std::shared_ptr<ICameraDeviceSession>* _aidl_return) {
+ ALOGV("%s", __func__);
+
+ *_aidl_return = ndk::SharedRefBase::make<VirtualCameraSession>(
+ *this, in_callback, mVirtualCameraClientCallback);
+
+ return ndk::ScopedAStatus::ok();
+};
+
+ndk::ScopedAStatus VirtualCameraDevice::openInjectionSession(
+ const std::shared_ptr<ICameraDeviceCallback>& in_callback,
+ std::shared_ptr<ICameraInjectionSession>* _aidl_return) {
+ ALOGV("%s", __func__);
+
+ (void)in_callback;
+ (void)_aidl_return;
+ return cameraStatus(Status::OPERATION_NOT_SUPPORTED);
+}
+
+ndk::ScopedAStatus VirtualCameraDevice::setTorchMode(bool in_on) {
+ ALOGV("%s: on = %s", __func__, in_on ? "on" : "off");
+ return cameraStatus(Status::OPERATION_NOT_SUPPORTED);
+}
+
+ndk::ScopedAStatus VirtualCameraDevice::turnOnTorchWithStrengthLevel(
+ int32_t in_torchStrength) {
+ ALOGV("%s: torchStrength = %d", __func__, in_torchStrength);
+ return cameraStatus(Status::OPERATION_NOT_SUPPORTED);
+}
+
+ndk::ScopedAStatus VirtualCameraDevice::getTorchStrengthLevel(
+ int32_t* _aidl_return) {
+ (void)_aidl_return;
+ return cameraStatus(Status::OPERATION_NOT_SUPPORTED);
+}
+
+binder_status_t VirtualCameraDevice::dump(int fd, const char** args,
+ uint32_t numArgs) {
+ // TODO(b/301023410) Implement.
+ (void)fd;
+ (void)args;
+ (void)numArgs;
+ return STATUS_OK;
+}
+
+std::string VirtualCameraDevice::getCameraName() const {
+ return std::string(kDevicePathPrefix) + std::to_string(mCameraId);
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/VirtualCameraDevice.h b/services/camera/virtualcamera/VirtualCameraDevice.h
new file mode 100644
index 0000000..ee7a3a7
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraDevice.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERADEVICE_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERADEVICE_H
+
+#include <cstdint>
+#include <memory>
+
+#include "aidl/android/companion/virtualcamera/IVirtualCameraCallback.h"
+#include "aidl/android/companion/virtualcamera/SupportedStreamConfiguration.h"
+#include "aidl/android/hardware/camera/device/BnCameraDevice.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Representation of single virtual camera device, implements
+// ICameraDevice AIDL to expose camera to camera framework.
+class VirtualCameraDevice
+ : public ::aidl::android::hardware::camera::device::BnCameraDevice {
+ public:
+ explicit VirtualCameraDevice(
+ uint32_t cameraId,
+ const std::vector<
+ aidl::android::companion::virtualcamera::SupportedStreamConfiguration>&
+ supportedInputConfig,
+ std::shared_ptr<
+ ::aidl::android::companion::virtualcamera::IVirtualCameraCallback>
+ virtualCameraClientCallback = nullptr);
+
+ virtual ~VirtualCameraDevice() override = default;
+
+ ndk::ScopedAStatus getCameraCharacteristics(
+ ::aidl::android::hardware::camera::device::CameraMetadata* _aidl_return)
+ override;
+
+ ndk::ScopedAStatus getPhysicalCameraCharacteristics(
+ const std::string& in_physicalCameraId,
+ ::aidl::android::hardware::camera::device::CameraMetadata* _aidl_return)
+ override;
+
+ ndk::ScopedAStatus getResourceCost(
+ ::aidl::android::hardware::camera::common::CameraResourceCost*
+ _aidl_return) override;
+
+ ndk::ScopedAStatus isStreamCombinationSupported(
+ const ::aidl::android::hardware::camera::device::StreamConfiguration&
+ in_streams,
+ bool* _aidl_return) override;
+
+ bool isStreamCombinationSupported(
+ const ::aidl::android::hardware::camera::device::StreamConfiguration&
+ in_streams) const;
+
+ ndk::ScopedAStatus open(
+ const std::shared_ptr<
+ ::aidl::android::hardware::camera::device::ICameraDeviceCallback>&
+ in_callback,
+ std::shared_ptr<
+ ::aidl::android::hardware::camera::device::ICameraDeviceSession>*
+ _aidl_return) override;
+
+ ndk::ScopedAStatus openInjectionSession(
+ const std::shared_ptr<
+ ::aidl::android::hardware::camera::device::ICameraDeviceCallback>&
+ in_callback,
+ std::shared_ptr<
+ ::aidl::android::hardware::camera::device::ICameraInjectionSession>*
+ _aidl_return) override;
+
+ ndk::ScopedAStatus setTorchMode(bool in_on) override;
+
+ ndk::ScopedAStatus turnOnTorchWithStrengthLevel(
+ int32_t in_torchStrength) override;
+
+ ndk::ScopedAStatus getTorchStrengthLevel(int32_t* _aidl_return) override;
+
+ binder_status_t dump(int fd, const char** args, uint32_t numArgs) override;
+
+ // Returns unique virtual camera name in form
+ // "device@{major}.{minor}/virtual/{numerical_id}"
+ std::string getCameraName() const;
+
+ uint32_t getCameraId() const { return mCameraId; }
+
+ private:
+ const uint32_t mCameraId;
+ const std::shared_ptr<
+ ::aidl::android::companion::virtualcamera::IVirtualCameraCallback>
+ mVirtualCameraClientCallback;
+
+ ::aidl::android::hardware::camera::device::CameraMetadata mCameraCharacteristics;
+
+ const std::vector<
+ aidl::android::companion::virtualcamera::SupportedStreamConfiguration>
+ mSupportedInputConfigurations;
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERADEVICE_H
diff --git a/services/camera/virtualcamera/VirtualCameraProvider.cc b/services/camera/virtualcamera/VirtualCameraProvider.cc
new file mode 100644
index 0000000..25a43d6
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraProvider.cc
@@ -0,0 +1,221 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "VirtualCameraProvider"
+#include "VirtualCameraProvider.h"
+
+#include <atomic>
+#include <memory>
+#include <mutex>
+#include <tuple>
+#include <utility>
+
+#include "VirtualCameraDevice.h"
+#include "aidl/android/hardware/camera/common/Status.h"
+#include "log/log.h"
+#include "util/Util.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+using ::aidl::android::companion::virtualcamera::IVirtualCameraCallback;
+using ::aidl::android::companion::virtualcamera::SupportedStreamConfiguration;
+using ::aidl::android::hardware::camera::common::CameraDeviceStatus;
+using ::aidl::android::hardware::camera::common::Status;
+using ::aidl::android::hardware::camera::common::VendorTagSection;
+using ::aidl::android::hardware::camera::device::ICameraDevice;
+using ::aidl::android::hardware::camera::provider::CameraIdAndStreamCombination;
+using ::aidl::android::hardware::camera::provider::ConcurrentCameraIdCombination;
+using ::aidl::android::hardware::camera::provider::ICameraProviderCallback;
+
+// TODO(b/301023410) Make camera id range configurable / dynamic
+// based on already registered devices.
+std::atomic_int VirtualCameraProvider::sNextId{42};
+
+ndk::ScopedAStatus VirtualCameraProvider::setCallback(
+ const std::shared_ptr<ICameraProviderCallback>& in_callback) {
+ ALOGV("%s", __func__);
+
+ if (in_callback == nullptr) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ {
+ const std::lock_guard<std::mutex> lock(mLock);
+ mCameraProviderCallback = in_callback;
+
+ for (const auto& [cameraName, _] : mCameras) {
+ auto ret = mCameraProviderCallback->cameraDeviceStatusChange(
+ cameraName, CameraDeviceStatus::PRESENT);
+ if (!ret.isOk()) {
+ ALOGE("Failed to announce camera status change: %s",
+ ret.getDescription().c_str());
+ }
+ }
+ }
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraProvider::getVendorTags(
+ std::vector<VendorTagSection>* _aidl_return) {
+ ALOGV("%s", __func__);
+
+ if (_aidl_return == nullptr) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ // No vendor tags supported.
+ _aidl_return->clear();
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraProvider::getCameraIdList(
+ std::vector<std::string>* _aidl_return) {
+ ALOGV("%s", __func__);
+
+ if (_aidl_return == nullptr) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ {
+ const std::lock_guard<std::mutex> lock(mLock);
+ _aidl_return->clear();
+ _aidl_return->reserve(mCameras.size());
+ for (const auto& [cameraName, _] : mCameras) {
+ _aidl_return->emplace_back(cameraName);
+ }
+ }
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraProvider::getCameraDeviceInterface(
+ const std::string& in_cameraDeviceName,
+ std::shared_ptr<ICameraDevice>* _aidl_return) {
+ ALOGV("%s cameraDeviceName %s", __func__, in_cameraDeviceName.c_str());
+
+ if (_aidl_return == nullptr) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ {
+ const std::lock_guard<std::mutex> lock(mLock);
+ const auto it = mCameras.find(in_cameraDeviceName);
+ *_aidl_return = (it == mCameras.end()) ? nullptr : it->second;
+ }
+
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraProvider::notifyDeviceStateChange(
+ int64_t in_deviceState) {
+ ALOGV("%s", __func__);
+ (void)in_deviceState;
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraProvider::getConcurrentCameraIds(
+ std::vector<ConcurrentCameraIdCombination>* _aidl_return) {
+ ALOGV("%s", __func__);
+ if (_aidl_return == nullptr) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ // No support for any concurrent combination.
+ _aidl_return->clear();
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraProvider::isConcurrentStreamCombinationSupported(
+ const std::vector<CameraIdAndStreamCombination>& in_configs,
+ bool* _aidl_return) {
+ ALOGV("%s", __func__);
+ (void)in_configs;
+ if (_aidl_return == nullptr) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ // No support for any stream combination at the moment.
+ *_aidl_return = false;
+ return ndk::ScopedAStatus::ok();
+}
+
+std::shared_ptr<VirtualCameraDevice> VirtualCameraProvider::createCamera(
+ const std::vector<SupportedStreamConfiguration>& supportedInputConfig,
+ std::shared_ptr<IVirtualCameraCallback> virtualCameraClientCallback) {
+ auto camera = ndk::SharedRefBase::make<VirtualCameraDevice>(
+ sNextId++, supportedInputConfig, virtualCameraClientCallback);
+ std::shared_ptr<ICameraProviderCallback> callback;
+ {
+ const std::lock_guard<std::mutex> lock(mLock);
+ if (mCameras.find(camera->getCameraName()) != mCameras.end()) {
+ ALOGE("Camera with identical name already exists.");
+ return nullptr;
+ }
+ mCameras.emplace(std::piecewise_construct,
+ std::forward_as_tuple(camera->getCameraName()),
+ std::forward_as_tuple(camera));
+ callback = mCameraProviderCallback;
+ }
+
+ if (callback != nullptr) {
+ auto ret = callback->cameraDeviceStatusChange(camera->getCameraName(),
+ CameraDeviceStatus::PRESENT);
+ if (!ret.isOk()) {
+ ALOGE("Failed to announce camera %s status change (PRESENT): %s",
+ camera->getCameraName().c_str(), ret.getDescription().c_str());
+ }
+ }
+ return camera;
+}
+
+std::shared_ptr<VirtualCameraDevice> VirtualCameraProvider::getCamera(
+ const std::string& cameraName) {
+ const std::lock_guard<std::mutex> lock(mLock);
+ auto it = mCameras.find(cameraName);
+ return it == mCameras.end() ? nullptr : it->second;
+}
+
+bool VirtualCameraProvider::removeCamera(const std::string& name) {
+ std::shared_ptr<ICameraProviderCallback> callback;
+ {
+ const std::lock_guard<std::mutex> lock(mLock);
+ auto it = mCameras.find(name);
+ if (it == mCameras.end()) {
+ ALOGE("Cannot remove camera %s: no such camera", name.c_str());
+ return false;
+ }
+ // TODO(b/301023410) Gracefully shut down camera.
+ mCameras.erase(it);
+ callback = mCameraProviderCallback;
+ }
+
+ if (callback != nullptr) {
+ auto ret = callback->cameraDeviceStatusChange(
+ name, CameraDeviceStatus::NOT_PRESENT);
+ if (!ret.isOk()) {
+ ALOGE("Failed to announce camera %s status change (NOT_PRESENT): %s",
+ name.c_str(), ret.getDescription().c_str());
+ }
+ }
+
+ return true;
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/VirtualCameraProvider.h b/services/camera/virtualcamera/VirtualCameraProvider.h
new file mode 100644
index 0000000..d41a005
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraProvider.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERAPROVIDER_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERAPROVIDER_H
+
+#include <atomic>
+#include <map>
+#include <memory>
+#include <mutex>
+
+#include "VirtualCameraDevice.h"
+#include "aidl/android/companion/virtualcamera/BnVirtualCameraCallback.h"
+#include "aidl/android/hardware/camera/common/VendorTagSection.h"
+#include "aidl/android/hardware/camera/device/ICameraDevice.h"
+#include "aidl/android/hardware/camera/provider/BnCameraProvider.h"
+#include "aidl/android/hardware/camera/provider/CameraIdAndStreamCombination.h"
+#include "aidl/android/hardware/camera/provider/ConcurrentCameraIdCombination.h"
+#include "aidl/android/hardware/camera/provider/ICameraProviderCallback.h"
+#include "utils/Mutex.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Entry point for virtual camera HAL.
+// Allows to create and keep track of virtual camera and implements
+// IAudioProvider AIDL interface to expose virtual camera devices to camera framework.
+class VirtualCameraProvider
+ : public ::aidl::android::hardware::camera::provider::BnCameraProvider {
+ public:
+ ~VirtualCameraProvider() override = default;
+
+ ndk::ScopedAStatus setCallback(
+ const std::shared_ptr<
+ ::aidl::android::hardware::camera::provider::ICameraProviderCallback>&
+ in_callback) override;
+
+ ndk::ScopedAStatus getVendorTags(
+ std::vector<::aidl::android::hardware::camera::common::VendorTagSection>*
+ _aidl_return) override;
+
+ ndk::ScopedAStatus getCameraIdList(
+ std::vector<std::string>* _aidl_return) override;
+
+ ndk::ScopedAStatus getCameraDeviceInterface(
+ const std::string& in_cameraDeviceName,
+ std::shared_ptr<::aidl::android::hardware::camera::device::ICameraDevice>*
+ _aidl_return) override;
+
+ ndk::ScopedAStatus notifyDeviceStateChange(int64_t in_deviceState) override;
+
+ ndk::ScopedAStatus getConcurrentCameraIds(
+ std::vector<::aidl::android::hardware::camera::provider::
+ ConcurrentCameraIdCombination>* _aidl_return) override;
+
+ ndk::ScopedAStatus isConcurrentStreamCombinationSupported(
+ const std::vector<::aidl::android::hardware::camera::provider::
+ CameraIdAndStreamCombination>& in_configs,
+ bool* _aidl_return) override;
+
+ // Create new virtual camera devices
+ // Returns nullptr if creation was not successful.
+ //
+ // TODO(b/301023410) - Add camera configuration.
+ std::shared_ptr<VirtualCameraDevice> createCamera(
+ const std::vector<
+ aidl::android::companion::virtualcamera::SupportedStreamConfiguration>&
+ supportedInputConfig,
+ std::shared_ptr<aidl::android::companion::virtualcamera::IVirtualCameraCallback>
+ virtualCameraClientCallback = nullptr);
+
+ std::shared_ptr<VirtualCameraDevice> getCamera(const std::string& name);
+
+ bool removeCamera(const std::string& name);
+
+ private:
+ std::mutex mLock;
+
+ std::shared_ptr<
+ ::aidl::android::hardware::camera::provider::ICameraProviderCallback>
+ mCameraProviderCallback GUARDED_BY(mLock);
+
+ std::map<std::string, std::shared_ptr<VirtualCameraDevice>> mCameras
+ GUARDED_BY(mLock);
+
+ // Numerical id to assign to next created camera.
+ static std::atomic_int sNextId;
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_SERVICES_VIRTUAL_CAMERA_VIRTUALCAMERAPROVIDER_H
diff --git a/services/camera/virtualcamera/VirtualCameraRenderThread.cc b/services/camera/virtualcamera/VirtualCameraRenderThread.cc
new file mode 100644
index 0000000..8a2db1c
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraRenderThread.cc
@@ -0,0 +1,453 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "VirtualCameraRenderThread"
+#include "VirtualCameraRenderThread.h"
+
+#include <chrono>
+#include <cstddef>
+#include <future>
+#include <memory>
+#include <mutex>
+#include <thread>
+
+#include "VirtualCameraSessionContext.h"
+#include "aidl/android/hardware/camera/common/Status.h"
+#include "aidl/android/hardware/camera/device/BufferStatus.h"
+#include "aidl/android/hardware/camera/device/CameraMetadata.h"
+#include "aidl/android/hardware/camera/device/CaptureResult.h"
+#include "aidl/android/hardware/camera/device/ErrorCode.h"
+#include "aidl/android/hardware/camera/device/ICameraDeviceCallback.h"
+#include "aidl/android/hardware/camera/device/NotifyMsg.h"
+#include "aidl/android/hardware/camera/device/ShutterMsg.h"
+#include "aidl/android/hardware/camera/device/StreamBuffer.h"
+#include "android-base/thread_annotations.h"
+#include "android/binder_auto_utils.h"
+#include "android/hardware_buffer.h"
+#include "util/EglFramebuffer.h"
+#include "util/JpegUtil.h"
+#include "util/MetadataBuilder.h"
+#include "util/TestPatternHelper.h"
+#include "util/Util.h"
+#include "utils/Errors.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+using ::aidl::android::hardware::camera::common::Status;
+using ::aidl::android::hardware::camera::device::BufferStatus;
+using ::aidl::android::hardware::camera::device::CameraMetadata;
+using ::aidl::android::hardware::camera::device::CaptureResult;
+using ::aidl::android::hardware::camera::device::ErrorCode;
+using ::aidl::android::hardware::camera::device::ErrorMsg;
+using ::aidl::android::hardware::camera::device::ICameraDeviceCallback;
+using ::aidl::android::hardware::camera::device::NotifyMsg;
+using ::aidl::android::hardware::camera::device::ShutterMsg;
+using ::aidl::android::hardware::camera::device::Stream;
+using ::aidl::android::hardware::camera::device::StreamBuffer;
+using ::aidl::android::hardware::graphics::common::PixelFormat;
+using ::android::base::ScopedLockAssertion;
+
+namespace {
+
+using namespace std::chrono_literals;
+
+static constexpr std::chrono::milliseconds kAcquireFenceTimeout = 500ms;
+
+CameraMetadata createCaptureResultMetadata(
+ const std::chrono::nanoseconds timestamp) {
+ std::unique_ptr<CameraMetadata> metadata =
+ MetadataBuilder().setSensorTimestamp(timestamp).build();
+ if (metadata == nullptr) {
+ ALOGE("%s: Failed to build capture result metadata", __func__);
+ return CameraMetadata();
+ }
+ return std::move(*metadata);
+}
+
+NotifyMsg createShutterNotifyMsg(int frameNumber,
+ std::chrono::nanoseconds timestamp) {
+ NotifyMsg msg;
+ msg.set<NotifyMsg::Tag::shutter>(ShutterMsg{
+ .frameNumber = frameNumber,
+ .timestamp = timestamp.count(),
+ });
+ return msg;
+}
+
+NotifyMsg createBufferErrorNotifyMsg(int frameNumber, int streamId) {
+ NotifyMsg msg;
+ msg.set<NotifyMsg::Tag::error>(ErrorMsg{.frameNumber = frameNumber,
+ .errorStreamId = streamId,
+ .errorCode = ErrorCode::ERROR_BUFFER});
+ return msg;
+}
+
+NotifyMsg createRequestErrorNotifyMsg(int frameNumber) {
+ NotifyMsg msg;
+ msg.set<NotifyMsg::Tag::error>(ErrorMsg{
+ .frameNumber = frameNumber,
+ // errorStreamId needs to be set to -1 for ERROR_REQUEST
+ // (not tied to specific stream).
+ .errorStreamId = -1,
+ .errorCode = ErrorCode::ERROR_REQUEST});
+ return msg;
+}
+
+} // namespace
+
+CaptureRequestBuffer::CaptureRequestBuffer(int streamId, int bufferId,
+ sp<Fence> fence)
+ : mStreamId(streamId), mBufferId(bufferId), mFence(fence) {
+}
+
+int CaptureRequestBuffer::getStreamId() const {
+ return mStreamId;
+}
+
+int CaptureRequestBuffer::getBufferId() const {
+ return mBufferId;
+}
+
+sp<Fence> CaptureRequestBuffer::getFence() const {
+ return mFence;
+}
+
+VirtualCameraRenderThread::VirtualCameraRenderThread(
+ VirtualCameraSessionContext& sessionContext, const int mWidth,
+ const int mHeight,
+ std::shared_ptr<ICameraDeviceCallback> cameraDeviceCallback, bool testMode)
+ : mCameraDeviceCallback(cameraDeviceCallback),
+ mInputSurfaceWidth(mWidth),
+ mInputSurfaceHeight(mHeight),
+ mTestMode(testMode),
+ mSessionContext(sessionContext) {
+}
+
+VirtualCameraRenderThread::~VirtualCameraRenderThread() {
+ stop();
+ if (mThread.joinable()) {
+ mThread.join();
+ }
+}
+
+ProcessCaptureRequestTask::ProcessCaptureRequestTask(
+ int frameNumber, const std::vector<CaptureRequestBuffer>& requestBuffers)
+ : mFrameNumber(frameNumber), mBuffers(requestBuffers) {
+}
+
+int ProcessCaptureRequestTask::getFrameNumber() const {
+ return mFrameNumber;
+}
+
+const std::vector<CaptureRequestBuffer>& ProcessCaptureRequestTask::getBuffers()
+ const {
+ return mBuffers;
+}
+
+void VirtualCameraRenderThread::enqueueTask(
+ std::unique_ptr<ProcessCaptureRequestTask> task) {
+ std::lock_guard<std::mutex> lock(mLock);
+ mQueue.emplace_back(std::move(task));
+ mCondVar.notify_one();
+}
+
+void VirtualCameraRenderThread::flush() {
+ std::lock_guard<std::mutex> lock(mLock);
+ while (!mQueue.empty()) {
+ std::unique_ptr<ProcessCaptureRequestTask> task = std::move(mQueue.front());
+ mQueue.pop_front();
+ flushCaptureRequest(*task);
+ }
+}
+
+void VirtualCameraRenderThread::start() {
+ mThread = std::thread(&VirtualCameraRenderThread::threadLoop, this);
+}
+
+void VirtualCameraRenderThread::stop() {
+ {
+ std::lock_guard<std::mutex> lock(mLock);
+ mPendingExit = true;
+ mCondVar.notify_one();
+ }
+}
+
+sp<Surface> VirtualCameraRenderThread::getInputSurface() {
+ return mInputSurfacePromise.get_future().get();
+}
+
+std::unique_ptr<ProcessCaptureRequestTask>
+VirtualCameraRenderThread::dequeueTask() {
+ std::unique_lock<std::mutex> lock(mLock);
+ // Clang's thread safety analysis doesn't perform alias analysis,
+ // so it doesn't support moveable std::unique_lock.
+ //
+ // Lock assertion below is basically explicit declaration that
+ // the lock is held in this scope, which is true, since it's only
+ // released during waiting inside mCondVar.wait calls.
+ ScopedLockAssertion lockAssertion(mLock);
+
+ mCondVar.wait(lock, [this]() REQUIRES(mLock) {
+ return mPendingExit || !mQueue.empty();
+ });
+ if (mPendingExit) {
+ return nullptr;
+ }
+ std::unique_ptr<ProcessCaptureRequestTask> task = std::move(mQueue.front());
+ mQueue.pop_front();
+ return task;
+}
+
+void VirtualCameraRenderThread::threadLoop() {
+ ALOGV("Render thread starting");
+
+ mEglDisplayContext = std::make_unique<EglDisplayContext>();
+ mEglTextureProgram = std::make_unique<EglTextureProgram>();
+ mEglSurfaceTexture = std::make_unique<EglSurfaceTexture>(mInputSurfaceWidth,
+ mInputSurfaceHeight);
+ mInputSurfacePromise.set_value(mEglSurfaceTexture->getSurface());
+
+ while (std::unique_ptr<ProcessCaptureRequestTask> task = dequeueTask()) {
+ processCaptureRequest(*task);
+ }
+
+ ALOGV("Render thread exiting");
+}
+
+void VirtualCameraRenderThread::processCaptureRequest(
+ const ProcessCaptureRequestTask& request) {
+ const std::chrono::nanoseconds timestamp =
+ std::chrono::duration_cast<std::chrono::nanoseconds>(
+ std::chrono::steady_clock::now().time_since_epoch());
+
+ CaptureResult captureResult;
+ captureResult.fmqResultSize = 0;
+ captureResult.frameNumber = request.getFrameNumber();
+ // Partial result needs to be set to 1 when metadata are present.
+ captureResult.partialResult = 1;
+ captureResult.inputBuffer.streamId = -1;
+ captureResult.physicalCameraMetadata.resize(0);
+ captureResult.result = createCaptureResultMetadata(timestamp);
+
+ const std::vector<CaptureRequestBuffer>& buffers = request.getBuffers();
+ captureResult.outputBuffers.resize(buffers.size());
+
+ if (mTestMode) {
+ // In test mode let's just render something to the Surface ourselves.
+ renderTestPatternYCbCr420(mEglSurfaceTexture->getSurface(),
+ request.getFrameNumber());
+ }
+
+ mEglSurfaceTexture->updateTexture();
+
+ for (int i = 0; i < buffers.size(); ++i) {
+ const CaptureRequestBuffer& reqBuffer = buffers[i];
+ StreamBuffer& resBuffer = captureResult.outputBuffers[i];
+ resBuffer.streamId = reqBuffer.getStreamId();
+ resBuffer.bufferId = reqBuffer.getBufferId();
+ resBuffer.status = BufferStatus::OK;
+
+ const std::optional<Stream> streamConfig =
+ mSessionContext.getStreamConfig(reqBuffer.getStreamId());
+
+ if (!streamConfig.has_value()) {
+ resBuffer.status = BufferStatus::ERROR;
+ continue;
+ }
+
+ auto status = streamConfig->format == PixelFormat::BLOB
+ ? renderIntoBlobStreamBuffer(
+ reqBuffer.getStreamId(), reqBuffer.getBufferId(),
+ streamConfig->bufferSize, reqBuffer.getFence())
+ : renderIntoImageStreamBuffer(reqBuffer.getStreamId(),
+ reqBuffer.getBufferId(),
+ reqBuffer.getFence());
+ if (!status.isOk()) {
+ resBuffer.status = BufferStatus::ERROR;
+ }
+ }
+
+ std::vector<NotifyMsg> notifyMsg{
+ createShutterNotifyMsg(request.getFrameNumber(), timestamp)};
+ for (const StreamBuffer& resBuffer : captureResult.outputBuffers) {
+ if (resBuffer.status != BufferStatus::OK) {
+ notifyMsg.push_back(createBufferErrorNotifyMsg(request.getFrameNumber(),
+ resBuffer.streamId));
+ }
+ }
+
+ auto status = mCameraDeviceCallback->notify(notifyMsg);
+ if (!status.isOk()) {
+ ALOGE("%s: notify call failed: %s", __func__,
+ status.getDescription().c_str());
+ return;
+ }
+
+ std::vector<::aidl::android::hardware::camera::device::CaptureResult>
+ captureResults(1);
+ captureResults[0] = std::move(captureResult);
+
+ status = mCameraDeviceCallback->processCaptureResult(captureResults);
+ if (!status.isOk()) {
+ ALOGE("%s: processCaptureResult call failed: %s", __func__,
+ status.getDescription().c_str());
+ return;
+ }
+
+ ALOGD("%s: Successfully called processCaptureResult", __func__);
+}
+
+void VirtualCameraRenderThread::flushCaptureRequest(
+ const ProcessCaptureRequestTask& request) {
+ CaptureResult captureResult;
+ captureResult.fmqResultSize = 0;
+ captureResult.frameNumber = request.getFrameNumber();
+ captureResult.inputBuffer.streamId = -1;
+
+ const std::vector<CaptureRequestBuffer>& buffers = request.getBuffers();
+ captureResult.outputBuffers.resize(buffers.size());
+
+ for (int i = 0; i < buffers.size(); ++i) {
+ const CaptureRequestBuffer& reqBuffer = buffers[i];
+ StreamBuffer& resBuffer = captureResult.outputBuffers[i];
+ resBuffer.streamId = reqBuffer.getStreamId();
+ resBuffer.bufferId = reqBuffer.getBufferId();
+ resBuffer.status = BufferStatus::ERROR;
+ sp<Fence> fence = reqBuffer.getFence();
+ if (fence != nullptr && fence->isValid()) {
+ resBuffer.releaseFence.fds.emplace_back(fence->dup());
+ }
+ }
+
+ auto status = mCameraDeviceCallback->notify(
+ {createRequestErrorNotifyMsg(request.getFrameNumber())});
+ if (!status.isOk()) {
+ ALOGE("%s: notify call failed: %s", __func__,
+ status.getDescription().c_str());
+ return;
+ }
+
+ std::vector<::aidl::android::hardware::camera::device::CaptureResult>
+ captureResults(1);
+ captureResults[0] = std::move(captureResult);
+
+ status = mCameraDeviceCallback->processCaptureResult(captureResults);
+ if (!status.isOk()) {
+ ALOGE("%s: processCaptureResult call failed: %s", __func__,
+ status.getDescription().c_str());
+ }
+}
+
+ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoBlobStreamBuffer(
+ const int streamId, const int bufferId, const size_t bufferSize,
+ sp<Fence> fence) {
+ ALOGV("%s", __func__);
+ sp<GraphicBuffer> gBuffer = mEglSurfaceTexture->getCurrentBuffer();
+ if (gBuffer == nullptr) {
+ // Most probably nothing was yet written to input surface if we reached this.
+ ALOGE("%s: Cannot fetch most recent buffer from SurfaceTexture", __func__);
+ return cameraStatus(Status::INTERNAL_ERROR);
+ }
+ std::shared_ptr<AHardwareBuffer> hwBuffer =
+ mSessionContext.fetchHardwareBuffer(streamId, bufferId);
+
+ AHardwareBuffer_Planes planes_info;
+
+ int32_t rawFence = fence != nullptr ? fence->get() : -1;
+ int result = AHardwareBuffer_lockPlanes(hwBuffer.get(),
+ AHARDWAREBUFFER_USAGE_CPU_READ_RARELY,
+ rawFence, nullptr, &planes_info);
+ if (result != OK) {
+ ALOGE("%s: Failed to lock planes for BLOB buffer: %d", __func__, result);
+ return cameraStatus(Status::INTERNAL_ERROR);
+ }
+
+ android_ycbcr ycbcr;
+ status_t status =
+ gBuffer->lockYCbCr(AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, &ycbcr);
+ ALOGV("Locked buffers");
+ if (status != NO_ERROR) {
+ AHardwareBuffer_unlock(hwBuffer.get(), nullptr);
+ ALOGE("%s: Failed to lock graphic buffer: %d", __func__, status);
+ return cameraStatus(Status::INTERNAL_ERROR);
+ }
+
+ bool success = compressJpeg(gBuffer->getWidth(), gBuffer->getHeight(), ycbcr,
+ bufferSize, planes_info.planes[0].data);
+
+ status_t res = gBuffer->unlock();
+ if (res != NO_ERROR) {
+ ALOGE("Failed to unlock graphic buffer: %d", res);
+ }
+ AHardwareBuffer_unlock(hwBuffer.get(), nullptr);
+ ALOGV("Unlocked buffers");
+ return success ? ndk::ScopedAStatus::ok()
+ : cameraStatus(Status::INTERNAL_ERROR);
+}
+
+ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoImageStreamBuffer(
+ int streamId, int bufferId, sp<Fence> fence) {
+ ALOGV("%s", __func__);
+
+ const std::chrono::nanoseconds before =
+ std::chrono::duration_cast<std::chrono::nanoseconds>(
+ std::chrono::steady_clock::now().time_since_epoch());
+
+ // Render test pattern using EGL.
+ std::shared_ptr<EglFrameBuffer> framebuffer =
+ mSessionContext.fetchOrCreateEglFramebuffer(
+ mEglDisplayContext->getEglDisplay(), streamId, bufferId);
+ if (framebuffer == nullptr) {
+ ALOGE(
+ "%s: Failed to get EGL framebuffer corresponding to buffer id "
+ "%d for streamId %d",
+ __func__, bufferId, streamId);
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ // Wait for fence to clear.
+ if (fence != nullptr && fence->isValid()) {
+ status_t ret = fence->wait(kAcquireFenceTimeout.count());
+ if (ret != 0) {
+ ALOGE(
+ "Timeout while waiting for the acquire fence for buffer %d"
+ " for streamId %d",
+ bufferId, streamId);
+ return cameraStatus(Status::INTERNAL_ERROR);
+ }
+ }
+
+ mEglDisplayContext->makeCurrent();
+ framebuffer->beforeDraw();
+
+ mEglTextureProgram->draw(mEglSurfaceTexture->updateTexture());
+ framebuffer->afterDraw();
+
+ const std::chrono::nanoseconds after =
+ std::chrono::duration_cast<std::chrono::nanoseconds>(
+ std::chrono::steady_clock::now().time_since_epoch());
+
+ ALOGV("Rendering to buffer %d, stream %d took %lld ns", bufferId, streamId,
+ after.count() - before.count());
+
+ return ndk::ScopedAStatus::ok();
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/VirtualCameraRenderThread.h b/services/camera/virtualcamera/VirtualCameraRenderThread.h
new file mode 100644
index 0000000..30de7c2
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraRenderThread.h
@@ -0,0 +1,170 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H
+
+#include <deque>
+#include <future>
+#include <memory>
+#include <thread>
+
+#include "VirtualCameraSessionContext.h"
+#include "aidl/android/hardware/camera/device/ICameraDeviceCallback.h"
+#include "util/EglDisplayContext.h"
+#include "util/EglProgram.h"
+#include "util/EglSurfaceTexture.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Represents single output buffer of capture request.
+class CaptureRequestBuffer {
+ public:
+ CaptureRequestBuffer(int streamId, int bufferId, sp<Fence> fence = nullptr);
+
+ int getStreamId() const;
+ int getBufferId() const;
+ sp<Fence> getFence() const;
+
+ private:
+ const int mStreamId;
+ const int mBufferId;
+ const sp<Fence> mFence;
+};
+
+// Represents single capture request to fill set of buffers.
+class ProcessCaptureRequestTask {
+ public:
+ ProcessCaptureRequestTask(
+ int frameNumber, const std::vector<CaptureRequestBuffer>& requestBuffers);
+
+ // Returns frame number corresponding to the request.
+ int getFrameNumber() const;
+
+ // Get reference to vector describing output buffers corresponding
+ // to this request.
+ //
+ // Note that the vector is owned by the ProcessCaptureRequestTask instance,
+ // so it cannot be access outside of its lifetime.
+ const std::vector<CaptureRequestBuffer>& getBuffers() const;
+
+ private:
+ const int mFrameNumber;
+ const std::vector<CaptureRequestBuffer> mBuffers;
+};
+
+// Wraps dedicated rendering thread and rendering business with corresponding
+// input surface.
+class VirtualCameraRenderThread {
+ public:
+ // Create VirtualCameraRenderThread instance:
+ // * sessionContext - VirtualCameraSessionContext reference for shared access
+ // to mapped buffers.
+ // * inputWidth - requested width of input surface ("virtual camera sensor")
+ // * inputHeight - requested height of input surface ("virtual camera sensor")
+ // * cameraDeviceCallback - callback for corresponding camera instance
+ // * testMode - when set to true, test pattern is rendered to input surface
+ // before each capture request is processed to simulate client input.
+ VirtualCameraRenderThread(
+ VirtualCameraSessionContext& sessionContext, int inputWidth,
+ int inputHeight,
+ std::shared_ptr<
+ ::aidl::android::hardware::camera::device::ICameraDeviceCallback>
+ cameraDeviceCallback,
+ bool testMode = false);
+
+ ~VirtualCameraRenderThread();
+
+ // Start rendering thread.
+ void start();
+ // Stop rendering thread.
+ void stop();
+
+ // Equeue capture task for processing on render thread.
+ void enqueueTask(std::unique_ptr<ProcessCaptureRequestTask> task)
+ EXCLUDES(mLock);
+
+ // Flush all in-flight requests.
+ void flush() EXCLUDES(mLock);
+
+ // Returns input surface corresponding to "virtual camera sensor".
+ sp<Surface> getInputSurface();
+
+ private:
+ std::unique_ptr<ProcessCaptureRequestTask> dequeueTask() EXCLUDES(mLock);
+
+ // Rendering thread entry point.
+ void threadLoop();
+
+ // Process single capture request task (always called on render thread).
+ void processCaptureRequest(const ProcessCaptureRequestTask& captureRequestTask);
+
+ // Flush single capture request task returning the error status immediately.
+ void flushCaptureRequest(const ProcessCaptureRequestTask& captureRequestTask);
+
+ // TODO(b/301023410) - Refactor the actual rendering logic off this class for
+ // easier testability.
+
+ // Render current image to the BLOB buffer.
+ // If fence is specified, this function will block until the fence is cleared
+ // before writing to the buffer.
+ // Always called on render thread.
+ ndk::ScopedAStatus renderIntoBlobStreamBuffer(const int streamId,
+ const int bufferId,
+ const size_t bufferSize,
+ sp<Fence> fence = nullptr);
+
+ // Render current image to the YCbCr buffer.
+ // If fence is specified, this function will block until the fence is cleared
+ // before writing to the buffer.
+ // Always called on render thread.
+ ndk::ScopedAStatus renderIntoImageStreamBuffer(int streamId, int bufferId,
+ sp<Fence> fence = nullptr);
+
+ // Camera callback
+ const std::shared_ptr<
+ ::aidl::android::hardware::camera::device::ICameraDeviceCallback>
+ mCameraDeviceCallback;
+
+ const int mInputSurfaceWidth;
+ const int mInputSurfaceHeight;
+ const int mTestMode;
+
+ VirtualCameraSessionContext& mSessionContext;
+
+ std::thread mThread;
+
+ // Blocking queue implementation.
+ std::mutex mLock;
+ std::deque<std::unique_ptr<ProcessCaptureRequestTask>> mQueue GUARDED_BY(mLock);
+ std::condition_variable mCondVar;
+ volatile bool mPendingExit GUARDED_BY(mLock);
+
+ // EGL helpers - constructed and accessed only from rendering thread.
+ std::unique_ptr<EglDisplayContext> mEglDisplayContext;
+ std::unique_ptr<EglTextureProgram> mEglTextureProgram;
+ std::unique_ptr<EglSurfaceTexture> mEglSurfaceTexture;
+
+ std::promise<sp<Surface>> mInputSurfacePromise;
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H
diff --git a/services/camera/virtualcamera/VirtualCameraService.cc b/services/camera/virtualcamera/VirtualCameraService.cc
new file mode 100644
index 0000000..08bfff7
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraService.cc
@@ -0,0 +1,241 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "VirtualCameraService"
+#include "VirtualCameraService.h"
+
+#include <cinttypes>
+#include <cstdint>
+#include <cstdio>
+#include <memory>
+#include <mutex>
+
+#include "VirtualCameraDevice.h"
+#include "VirtualCameraProvider.h"
+#include "aidl/android/companion/virtualcamera/Format.h"
+#include "aidl/android/companion/virtualcamera/VirtualCameraConfiguration.h"
+#include "android/binder_auto_utils.h"
+#include "android/binder_libbinder.h"
+#include "binder/Status.h"
+#include "util/Util.h"
+
+using ::android::binder::Status;
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+using ::aidl::android::companion::virtualcamera::Format;
+using ::aidl::android::companion::virtualcamera::SupportedStreamConfiguration;
+using ::aidl::android::companion::virtualcamera::VirtualCameraConfiguration;
+
+namespace {
+
+constexpr int kVgaWidth = 640;
+constexpr int kVgaHeight = 480;
+constexpr char kEnableTestCameraCmd[] = "enable_test_camera";
+constexpr char kDisableTestCameraCmd[] = "disable_test_camera";
+constexpr char kShellCmdHelp[] = R"(
+Available commands:
+ * enable_test_camera
+ * disable_test_camera
+)";
+
+ndk::ScopedAStatus validateConfiguration(
+ const VirtualCameraConfiguration& configuration) {
+ if (configuration.supportedStreamConfigs.empty()) {
+ ALOGE("%s: No supported input configuration specified", __func__);
+ return ndk::ScopedAStatus::fromServiceSpecificError(
+ Status::EX_ILLEGAL_ARGUMENT);
+ }
+
+ for (const SupportedStreamConfiguration& config :
+ configuration.supportedStreamConfigs) {
+ if (!isFormatSupportedForInput(config.width, config.height,
+ config.pixelFormat)) {
+ ALOGE("%s: Requested unsupported input format: %d x %d (%d)", __func__,
+ config.width, config.height, static_cast<int>(config.pixelFormat));
+ return ndk::ScopedAStatus::fromServiceSpecificError(
+ Status::EX_ILLEGAL_ARGUMENT);
+ }
+ }
+ return ndk::ScopedAStatus::ok();
+}
+
+} // namespace
+
+VirtualCameraService::VirtualCameraService(
+ std::shared_ptr<VirtualCameraProvider> virtualCameraProvider)
+ : mVirtualCameraProvider(virtualCameraProvider) {
+}
+
+ndk::ScopedAStatus VirtualCameraService::registerCamera(
+ const ::ndk::SpAIBinder& token,
+ const VirtualCameraConfiguration& configuration, bool* _aidl_return) {
+ if (_aidl_return == nullptr) {
+ return ndk::ScopedAStatus::fromServiceSpecificError(
+ Status::EX_ILLEGAL_ARGUMENT);
+ }
+ *_aidl_return = true;
+
+ auto status = validateConfiguration(configuration);
+ if (!status.isOk()) {
+ *_aidl_return = false;
+ return status;
+ }
+
+ std::lock_guard lock(mLock);
+ if (mTokenToCameraName.find(token) != mTokenToCameraName.end()) {
+ ALOGE(
+ "Attempt to register camera corresponding to already registered binder "
+ "token: "
+ "0x%" PRIxPTR,
+ reinterpret_cast<uintptr_t>(token.get()));
+ *_aidl_return = false;
+ return ndk::ScopedAStatus::ok();
+ }
+
+ // TODO(b/301023410) Validate configuration and pass it to the camera.
+ std::shared_ptr<VirtualCameraDevice> camera =
+ mVirtualCameraProvider->createCamera(configuration.supportedStreamConfigs,
+ configuration.virtualCameraCallback);
+ if (camera == nullptr) {
+ ALOGE("Failed to create camera for binder token 0x%" PRIxPTR,
+ reinterpret_cast<uintptr_t>(token.get()));
+ *_aidl_return = false;
+ return ndk::ScopedAStatus::fromServiceSpecificError(
+ Status::EX_SERVICE_SPECIFIC);
+ }
+
+ mTokenToCameraName[token] = camera->getCameraName();
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraService::unregisterCamera(
+ const ::ndk::SpAIBinder& token) {
+ std::lock_guard lock(mLock);
+
+ auto it = mTokenToCameraName.find(token);
+ if (it == mTokenToCameraName.end()) {
+ ALOGE(
+ "Attempt to unregister camera corresponding to unknown binder token: "
+ "0x%" PRIxPTR,
+ reinterpret_cast<uintptr_t>(token.get()));
+ return ndk::ScopedAStatus::ok();
+ }
+
+ mVirtualCameraProvider->removeCamera(it->second);
+
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraService::getCameraId(
+ const ::ndk::SpAIBinder& token, int32_t* _aidl_return) {
+ if (_aidl_return == nullptr) {
+ return ndk::ScopedAStatus::fromServiceSpecificError(
+ Status::EX_ILLEGAL_ARGUMENT);
+ }
+
+ auto camera = getCamera(token);
+ if (camera == nullptr) {
+ ALOGE(
+ "Attempt to get camera id corresponding to unknown binder token: "
+ "0x%" PRIxPTR,
+ reinterpret_cast<uintptr_t>(token.get()));
+ return ndk::ScopedAStatus::ok();
+ }
+
+ *_aidl_return = camera->getCameraId();
+
+ return ndk::ScopedAStatus::ok();
+}
+
+std::shared_ptr<VirtualCameraDevice> VirtualCameraService::getCamera(
+ const ::ndk::SpAIBinder& token) {
+ if (token == nullptr) {
+ return nullptr;
+ }
+
+ std::lock_guard lock(mLock);
+ auto it = mTokenToCameraName.find(token);
+ if (it == mTokenToCameraName.end()) {
+ return nullptr;
+ }
+
+ return mVirtualCameraProvider->getCamera(it->second);
+}
+
+binder_status_t VirtualCameraService::handleShellCommand(int in, int out,
+ int err,
+ const char** args,
+ uint32_t numArgs) {
+ if (numArgs <= 0) {
+ dprintf(out, kShellCmdHelp);
+ fsync(out);
+ return STATUS_OK;
+ }
+
+ if (args == nullptr || args[0] == nullptr) {
+ return STATUS_BAD_VALUE;
+ }
+ const char* const cmd = args[0];
+ if (strcmp(kEnableTestCameraCmd, cmd) == 0) {
+ enableTestCameraCmd(in, err);
+ } else if (strcmp(kDisableTestCameraCmd, cmd) == 0) {
+ disableTestCameraCmd(in);
+ } else {
+ dprintf(out, kShellCmdHelp);
+ }
+
+ fsync(out);
+ return STATUS_OK;
+}
+
+void VirtualCameraService::enableTestCameraCmd(const int out, const int err) {
+ if (mTestCameraToken != nullptr) {
+ dprintf(out, "Test camera is already enabled (%s).",
+ getCamera(mTestCameraToken)->getCameraName().c_str());
+ return;
+ }
+
+ sp<BBinder> token = sp<BBinder>::make();
+ mTestCameraToken.set(AIBinder_fromPlatformBinder(token));
+
+ bool ret;
+ VirtualCameraConfiguration configuration;
+ configuration.supportedStreamConfigs.push_back(
+ {.width = kVgaWidth, .height = kVgaHeight, Format::YUV_420_888});
+ registerCamera(mTestCameraToken, configuration, &ret);
+ if (ret) {
+ dprintf(out, "Successfully registered test camera %s",
+ getCamera(mTestCameraToken)->getCameraName().c_str());
+ } else {
+ dprintf(err, "Failed to create test camera");
+ }
+}
+
+void VirtualCameraService::disableTestCameraCmd(const int out) {
+ if (mTestCameraToken == nullptr) {
+ dprintf(out, "Test camera is not registered.");
+ }
+ unregisterCamera(mTestCameraToken);
+ mTestCameraToken.set(nullptr);
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/VirtualCameraService.h b/services/camera/virtualcamera/VirtualCameraService.h
new file mode 100644
index 0000000..b68d43a
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraService.h
@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASERVICE_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASERVICE_H
+
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+
+#include "VirtualCameraDevice.h"
+#include "VirtualCameraProvider.h"
+#include "aidl/android/companion/virtualcamera/BnVirtualCameraService.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Implementation of Virtual Camera Service for managing virtual camera devices.
+class VirtualCameraService
+ : public aidl::android::companion::virtualcamera::BnVirtualCameraService {
+ public:
+ VirtualCameraService(
+ std::shared_ptr<VirtualCameraProvider> virtualCameraProvider);
+
+ // Register camera corresponding to the binder token.
+ ndk::ScopedAStatus registerCamera(
+ const ::ndk::SpAIBinder& token,
+ const ::aidl::android::companion::virtualcamera::VirtualCameraConfiguration&
+ configuration,
+ bool* _aidl_return) override EXCLUDES(mLock);
+
+ // Unregisters camera corresponding to the binder token.
+ ndk::ScopedAStatus unregisterCamera(const ::ndk::SpAIBinder& token) override
+ EXCLUDES(mLock);
+
+ // Returns the camera id corresponding to the binder token.
+ ndk::ScopedAStatus getCameraId(
+ const ::ndk::SpAIBinder& token, int32_t* _aidl_return) override EXCLUDES(mLock);
+
+ // Returns VirtualCameraDevice corresponding to binder token or nullptr if
+ // there's no camera asociated with the token.
+ std::shared_ptr<VirtualCameraDevice> getCamera(const ::ndk::SpAIBinder& token)
+ EXCLUDES(mLock);
+
+ // Handle cmd shell commands `adb shell cmd virtual_camera_service` [args].
+ binder_status_t handleShellCommand(int in, int out, int err, const char** args,
+ uint32_t numArgs) override;
+
+ private:
+ // Create and enable test camera instance if there's none.
+ void enableTestCameraCmd(int out, int err);
+ // Disable and destroy test camera instance if there's one.
+ void disableTestCameraCmd(int out);
+
+ std::shared_ptr<VirtualCameraProvider> mVirtualCameraProvider;
+
+ std::mutex mLock;
+ struct BinderTokenHash {
+ std::size_t operator()(const ::ndk::SpAIBinder& key) const {
+ return std::hash<void*>{}(reinterpret_cast<void*>(key.get()));
+ }
+ };
+ // Map Binder tokens to names of cameras managed by camera provider.
+ std::unordered_map<::ndk::SpAIBinder, std::string, BinderTokenHash>
+ mTokenToCameraName GUARDED_BY(mLock);
+
+ // Local binder token for test camera instance, or nullptr if there's none.
+ ::ndk::SpAIBinder mTestCameraToken;
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASERVICE_H
diff --git a/services/camera/virtualcamera/VirtualCameraSession.cc b/services/camera/virtualcamera/VirtualCameraSession.cc
new file mode 100644
index 0000000..9e15871
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraSession.cc
@@ -0,0 +1,443 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "VirtualCameraSession"
+#include "VirtualCameraSession.h"
+
+#include <atomic>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <tuple>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "CameraMetadata.h"
+#include "EGL/egl.h"
+#include "VirtualCameraDevice.h"
+#include "VirtualCameraRenderThread.h"
+#include "VirtualCameraStream.h"
+#include "aidl/android/hardware/camera/common/Status.h"
+#include "aidl/android/hardware/camera/device/BufferCache.h"
+#include "aidl/android/hardware/camera/device/BufferStatus.h"
+#include "aidl/android/hardware/camera/device/CaptureRequest.h"
+#include "aidl/android/hardware/camera/device/HalStream.h"
+#include "aidl/android/hardware/camera/device/NotifyMsg.h"
+#include "aidl/android/hardware/camera/device/ShutterMsg.h"
+#include "aidl/android/hardware/camera/device/StreamBuffer.h"
+#include "aidl/android/hardware/camera/device/StreamConfiguration.h"
+#include "aidl/android/hardware/camera/device/StreamRotation.h"
+#include "aidl/android/hardware/graphics/common/BufferUsage.h"
+#include "aidl/android/hardware/graphics/common/PixelFormat.h"
+#include "android/hardware_buffer.h"
+#include "android/native_window_aidl.h"
+#include "fmq/AidlMessageQueue.h"
+#include "system/camera_metadata.h"
+#include "ui/GraphicBuffer.h"
+#include "util/EglDisplayContext.h"
+#include "util/EglFramebuffer.h"
+#include "util/EglProgram.h"
+#include "util/JpegUtil.h"
+#include "util/MetadataBuilder.h"
+#include "util/TestPatternHelper.h"
+#include "util/Util.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+using ::aidl::android::companion::virtualcamera::Format;
+using ::aidl::android::companion::virtualcamera::IVirtualCameraCallback;
+using ::aidl::android::hardware::camera::common::Status;
+using ::aidl::android::hardware::camera::device::BufferCache;
+using ::aidl::android::hardware::camera::device::CameraMetadata;
+using ::aidl::android::hardware::camera::device::CameraOfflineSessionInfo;
+using ::aidl::android::hardware::camera::device::CaptureRequest;
+using ::aidl::android::hardware::camera::device::HalStream;
+using ::aidl::android::hardware::camera::device::ICameraDeviceCallback;
+using ::aidl::android::hardware::camera::device::ICameraOfflineSession;
+using ::aidl::android::hardware::camera::device::RequestTemplate;
+using ::aidl::android::hardware::camera::device::Stream;
+using ::aidl::android::hardware::camera::device::StreamBuffer;
+using ::aidl::android::hardware::camera::device::StreamConfiguration;
+using ::aidl::android::hardware::camera::device::StreamRotation;
+using ::aidl::android::hardware::common::fmq::MQDescriptor;
+using ::aidl::android::hardware::common::fmq::SynchronizedReadWrite;
+using ::aidl::android::hardware::graphics::common::BufferUsage;
+using ::aidl::android::hardware::graphics::common::PixelFormat;
+using ::android::base::unique_fd;
+
+namespace {
+
+using metadata_ptr =
+ std::unique_ptr<camera_metadata_t, void (*)(camera_metadata_t*)>;
+
+using namespace std::chrono_literals;
+
+// Size of request/result metadata fast message queue.
+// Setting to 0 to always disables FMQ.
+static constexpr size_t kMetadataMsgQueueSize = 0;
+
+// Maximum number of buffers to use per single stream.
+static constexpr size_t kMaxStreamBuffers = 2;
+
+CameraMetadata createDefaultRequestSettings(RequestTemplate type) {
+ hardware::camera::common::V1_0::helper::CameraMetadata metadataHelper;
+
+ camera_metadata_enum_android_control_capture_intent_t intent =
+ ANDROID_CONTROL_CAPTURE_INTENT_PREVIEW;
+ switch (type) {
+ case RequestTemplate::PREVIEW:
+ intent = ANDROID_CONTROL_CAPTURE_INTENT_PREVIEW;
+ break;
+ case RequestTemplate::STILL_CAPTURE:
+ intent = ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE;
+ break;
+ case RequestTemplate::VIDEO_RECORD:
+ intent = ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_RECORD;
+ break;
+ case RequestTemplate::VIDEO_SNAPSHOT:
+ intent = ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_SNAPSHOT;
+ break;
+ default:
+ // Leave default.
+ break;
+ }
+
+ auto metadata = MetadataBuilder().setControlCaptureIntent(intent).build();
+ return (metadata != nullptr) ? std::move(*metadata) : CameraMetadata();
+}
+
+HalStream getHalStream(const Stream& stream) {
+ HalStream halStream;
+ halStream.id = stream.id;
+ halStream.physicalCameraId = stream.physicalCameraId;
+ halStream.maxBuffers = kMaxStreamBuffers;
+
+ if (stream.format == PixelFormat::IMPLEMENTATION_DEFINED) {
+ // If format is implementation defined we need it to override
+ // it with actual format.
+ // TODO(b/301023410) Override with the format based on the
+ // camera configuration, once we support more formats.
+ halStream.overrideFormat = PixelFormat::YCBCR_420_888;
+ } else {
+ halStream.overrideFormat = stream.format;
+ }
+ halStream.overrideDataSpace = stream.dataSpace;
+
+ halStream.producerUsage = BufferUsage::GPU_RENDER_TARGET;
+ halStream.supportOffline = false;
+ return halStream;
+}
+
+} // namespace
+
+VirtualCameraSession::VirtualCameraSession(
+ VirtualCameraDevice& cameraDevice,
+ std::shared_ptr<ICameraDeviceCallback> cameraDeviceCallback,
+ std::shared_ptr<IVirtualCameraCallback> virtualCameraClientCallback)
+ : mCameraDevice(cameraDevice),
+ mCameraDeviceCallback(cameraDeviceCallback),
+ mVirtualCameraClientCallback(virtualCameraClientCallback) {
+ mRequestMetadataQueue = std::make_unique<RequestMetadataQueue>(
+ kMetadataMsgQueueSize, false /* non blocking */);
+ if (!mRequestMetadataQueue->isValid()) {
+ ALOGE("%s: invalid request fmq", __func__);
+ }
+
+ mResultMetadataQueue = std::make_shared<ResultMetadataQueue>(
+ kMetadataMsgQueueSize, false /* non blocking */);
+ if (!mResultMetadataQueue->isValid()) {
+ ALOGE("%s: invalid result fmq", __func__);
+ }
+}
+
+ndk::ScopedAStatus VirtualCameraSession::close() {
+ ALOGV("%s", __func__);
+
+ if (mVirtualCameraClientCallback != nullptr) {
+ mVirtualCameraClientCallback->onStreamClosed(/*streamId=*/0);
+ }
+
+ {
+ std::lock_guard<std::mutex> lock(mLock);
+ if (mRenderThread != nullptr) {
+ mRenderThread->stop();
+ mRenderThread = nullptr;
+ }
+ }
+
+ mSessionContext.closeAllStreams();
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraSession::configureStreams(
+ const StreamConfiguration& in_requestedConfiguration,
+ std::vector<HalStream>* _aidl_return) {
+ ALOGV("%s: requestedConfiguration: %s", __func__,
+ in_requestedConfiguration.toString().c_str());
+
+ if (_aidl_return == nullptr) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ mSessionContext.removeStreamsNotInStreamConfiguration(
+ in_requestedConfiguration);
+
+ auto& streams = in_requestedConfiguration.streams;
+ auto& halStreams = *_aidl_return;
+ halStreams.clear();
+ halStreams.resize(in_requestedConfiguration.streams.size());
+
+ sp<Surface> inputSurface = nullptr;
+ int inputWidth;
+ int inputHeight;
+
+ if (!mCameraDevice.isStreamCombinationSupported(in_requestedConfiguration)) {
+ ALOGE("%s: Requested stream configuration is not supported", __func__);
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ {
+ std::lock_guard<std::mutex> lock(mLock);
+ for (int i = 0; i < in_requestedConfiguration.streams.size(); ++i) {
+ halStreams[i] = getHalStream(streams[i]);
+ if (mSessionContext.initializeStream(streams[i])) {
+ ALOGV("Configured new stream: %s", streams[i].toString().c_str());
+ }
+ }
+
+ inputWidth = streams[0].width;
+ inputHeight = streams[0].height;
+ if (mRenderThread == nullptr) {
+ // If there's no client callback, start camera in test mode.
+ const bool testMode = mVirtualCameraClientCallback == nullptr;
+ mRenderThread = std::make_unique<VirtualCameraRenderThread>(
+ mSessionContext, inputWidth, inputHeight, mCameraDeviceCallback,
+ testMode);
+ mRenderThread->start();
+ inputSurface = mRenderThread->getInputSurface();
+ }
+ }
+
+ if (mVirtualCameraClientCallback != nullptr && inputSurface != nullptr) {
+ // TODO(b/301023410) Pass streamId based on client input stream id once
+ // support for multiple input streams is implemented. For now we always
+ // create single texture.
+ mVirtualCameraClientCallback->onStreamConfigured(
+ /*streamId=*/0, aidl::android::view::Surface(inputSurface.get()),
+ inputWidth, inputHeight, Format::YUV_420_888);
+ }
+
+ mFirstRequest.store(true);
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraSession::constructDefaultRequestSettings(
+ RequestTemplate in_type, CameraMetadata* _aidl_return) {
+ ALOGV("%s: type %d", __func__, static_cast<int32_t>(in_type));
+
+ switch (in_type) {
+ case RequestTemplate::PREVIEW:
+ case RequestTemplate::STILL_CAPTURE:
+ case RequestTemplate::VIDEO_RECORD:
+ case RequestTemplate::VIDEO_SNAPSHOT: {
+ *_aidl_return = createDefaultRequestSettings(in_type);
+ return ndk::ScopedAStatus::ok();
+ }
+ case RequestTemplate::MANUAL:
+ case RequestTemplate::ZERO_SHUTTER_LAG:
+ // Don't support VIDEO_SNAPSHOT, MANUAL, ZSL templates
+ return ndk::ScopedAStatus::fromServiceSpecificError(
+ static_cast<int32_t>(Status::ILLEGAL_ARGUMENT));
+ ;
+ default:
+ ALOGE("%s: unknown request template type %d", __FUNCTION__,
+ static_cast<int>(in_type));
+ return ndk::ScopedAStatus::fromServiceSpecificError(
+ static_cast<int32_t>(Status::ILLEGAL_ARGUMENT));
+ ;
+ }
+}
+
+ndk::ScopedAStatus VirtualCameraSession::flush() {
+ ALOGV("%s", __func__);
+ std::lock_guard<std::mutex> lock(mLock);
+ if (mRenderThread != nullptr) {
+ mRenderThread->flush();
+ }
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraSession::getCaptureRequestMetadataQueue(
+ MQDescriptor<int8_t, SynchronizedReadWrite>* _aidl_return) {
+ ALOGV("%s", __func__);
+ *_aidl_return = mRequestMetadataQueue->dupeDesc();
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraSession::getCaptureResultMetadataQueue(
+ MQDescriptor<int8_t, SynchronizedReadWrite>* _aidl_return) {
+ ALOGV("%s", __func__);
+ *_aidl_return = mResultMetadataQueue->dupeDesc();
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraSession::isReconfigurationRequired(
+ const CameraMetadata& in_oldSessionParams,
+ const CameraMetadata& in_newSessionParams, bool* _aidl_return) {
+ ALOGV("%s: oldSessionParams: %s newSessionParams: %s", __func__,
+ in_newSessionParams.toString().c_str(),
+ in_oldSessionParams.toString().c_str());
+
+ if (_aidl_return == nullptr) {
+ return ndk::ScopedAStatus::fromServiceSpecificError(
+ static_cast<int32_t>(Status::ILLEGAL_ARGUMENT));
+ }
+
+ *_aidl_return = true;
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraSession::processCaptureRequest(
+ const std::vector<CaptureRequest>& in_requests,
+ const std::vector<BufferCache>& in_cachesToRemove, int32_t* _aidl_return) {
+ ALOGV("%s", __func__);
+
+ if (!in_cachesToRemove.empty()) {
+ mSessionContext.removeBufferCaches(in_cachesToRemove);
+ }
+
+ for (const auto& captureRequest : in_requests) {
+ auto status = processCaptureRequest(captureRequest);
+ if (!status.isOk()) {
+ return status;
+ }
+ }
+ *_aidl_return = in_requests.size();
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraSession::signalStreamFlush(
+ const std::vector<int32_t>& in_streamIds, int32_t in_streamConfigCounter) {
+ ALOGV("%s", __func__);
+
+ (void)in_streamIds;
+ (void)in_streamConfigCounter;
+ return ndk::ScopedAStatus::ok();
+}
+
+ndk::ScopedAStatus VirtualCameraSession::switchToOffline(
+ const std::vector<int32_t>& in_streamsToKeep,
+ CameraOfflineSessionInfo* out_offlineSessionInfo,
+ std::shared_ptr<ICameraOfflineSession>* _aidl_return) {
+ ALOGV("%s", __func__);
+
+ (void)in_streamsToKeep;
+ (void)out_offlineSessionInfo;
+
+ if (_aidl_return == nullptr) {
+ return ndk::ScopedAStatus::fromServiceSpecificError(
+ static_cast<int32_t>(Status::ILLEGAL_ARGUMENT));
+ }
+
+ *_aidl_return = nullptr;
+ return cameraStatus(Status::OPERATION_NOT_SUPPORTED);
+}
+
+ndk::ScopedAStatus VirtualCameraSession::repeatingRequestEnd(
+ int32_t in_frameNumber, const std::vector<int32_t>& in_streamIds) {
+ ALOGV("%s", __func__);
+ (void)in_frameNumber;
+ (void)in_streamIds;
+ return ndk::ScopedAStatus::ok();
+}
+
+std::set<int> VirtualCameraSession::getStreamIds() const {
+ return mSessionContext.getStreamIds();
+}
+
+ndk::ScopedAStatus VirtualCameraSession::processCaptureRequest(
+ const CaptureRequest& request) {
+ ALOGD("%s: request: %s", __func__, request.toString().c_str());
+
+ if (mFirstRequest.exchange(false) && request.settings.metadata.empty()) {
+ return cameraStatus(Status::ILLEGAL_ARGUMENT);
+ }
+
+ std::shared_ptr<ICameraDeviceCallback> cameraCallback = nullptr;
+ {
+ std::lock_guard<std::mutex> lock(mLock);
+ cameraCallback = mCameraDeviceCallback;
+ }
+
+ if (cameraCallback == nullptr) {
+ ALOGE(
+ "%s: processCaptureRequest called, but there's no camera callback "
+ "configured",
+ __func__);
+ return cameraStatus(Status::INTERNAL_ERROR);
+ }
+
+ if (!mSessionContext.importBuffersFromCaptureRequest(request)) {
+ ALOGE("Failed to import buffers from capture request.");
+ return cameraStatus(Status::INTERNAL_ERROR);
+ }
+
+ std::vector<CaptureRequestBuffer> taskBuffers;
+ taskBuffers.reserve(request.outputBuffers.size());
+ for (const StreamBuffer& streamBuffer : request.outputBuffers) {
+ taskBuffers.emplace_back(streamBuffer.streamId, streamBuffer.bufferId,
+ importFence(streamBuffer.acquireFence));
+ }
+
+ {
+ std::lock_guard<std::mutex> lock(mLock);
+ if (mRenderThread == nullptr) {
+ ALOGE(
+ "%s: processCaptureRequest (frameNumber %d)called before configure "
+ "(render thread not initialized)",
+ __func__, request.frameNumber);
+ return cameraStatus(Status::INTERNAL_ERROR);
+ }
+ mRenderThread->enqueueTask(std::make_unique<ProcessCaptureRequestTask>(
+ request.frameNumber, taskBuffers));
+ }
+
+ if (mVirtualCameraClientCallback != nullptr) {
+ auto status = mVirtualCameraClientCallback->onProcessCaptureRequest(
+ /*streamId=*/0, request.frameNumber);
+ if (!status.isOk()) {
+ ALOGE(
+ "Failed to invoke onProcessCaptureRequest client callback for frame "
+ "%d",
+ request.frameNumber);
+ }
+ }
+
+ return ndk::ScopedAStatus::ok();
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/VirtualCameraSession.h b/services/camera/virtualcamera/VirtualCameraSession.h
new file mode 100644
index 0000000..50962e5
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraSession.h
@@ -0,0 +1,149 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASESSION_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASESSION_H
+
+#include <memory>
+#include <set>
+
+#include "VirtualCameraRenderThread.h"
+#include "VirtualCameraSessionContext.h"
+#include "aidl/android/companion/virtualcamera/IVirtualCameraCallback.h"
+#include "aidl/android/hardware/camera/device/BnCameraDeviceSession.h"
+#include "aidl/android/hardware/camera/device/ICameraDeviceCallback.h"
+#include "utils/Mutex.h"
+
+namespace android {
+
+template <typename T, typename U>
+struct AidlMessageQueue;
+namespace companion {
+namespace virtualcamera {
+
+class VirtualCameraDevice;
+
+// Implementation of ICameraDeviceSession AIDL interface to allow camera
+// framework to read image data from open virtual camera device. This class
+// encapsulates possibly several image streams for the same session.
+class VirtualCameraSession
+ : public ::aidl::android::hardware::camera::device::BnCameraDeviceSession {
+ public:
+ // Construct new virtual camera session.
+ // When virtualCameraClientCallback is null, the input surface will be filled
+ // with test pattern.
+ VirtualCameraSession(
+ VirtualCameraDevice& mCameraDevice,
+ std::shared_ptr<
+ ::aidl::android::hardware::camera::device::ICameraDeviceCallback>
+ cameraDeviceCallback,
+ std::shared_ptr<
+ ::aidl::android::companion::virtualcamera::IVirtualCameraCallback>
+ virtualCameraClientCallback = nullptr);
+
+ virtual ~VirtualCameraSession() override = default;
+
+ ndk::ScopedAStatus close() override EXCLUDES(mLock);
+
+ ndk::ScopedAStatus configureStreams(
+ const ::aidl::android::hardware::camera::device::StreamConfiguration&
+ in_requestedConfiguration,
+ std::vector<::aidl::android::hardware::camera::device::HalStream>*
+ _aidl_return) override EXCLUDES(mLock);
+
+ ndk::ScopedAStatus constructDefaultRequestSettings(
+ ::aidl::android::hardware::camera::device::RequestTemplate in_type,
+ ::aidl::android::hardware::camera::device::CameraMetadata* _aidl_return)
+ override;
+
+ ndk::ScopedAStatus flush() override EXCLUDES(mLock);
+
+ ndk::ScopedAStatus getCaptureRequestMetadataQueue(
+ ::aidl::android::hardware::common::fmq::MQDescriptor<
+ int8_t, ::aidl::android::hardware::common::fmq::SynchronizedReadWrite>*
+ _aidl_return) override;
+
+ ndk::ScopedAStatus getCaptureResultMetadataQueue(
+ ::aidl::android::hardware::common::fmq::MQDescriptor<
+ int8_t, ::aidl::android::hardware::common::fmq::SynchronizedReadWrite>*
+ _aidl_return) override;
+
+ ndk::ScopedAStatus isReconfigurationRequired(
+ const ::aidl::android::hardware::camera::device::CameraMetadata&
+ in_oldSessionParams,
+ const ::aidl::android::hardware::camera::device::CameraMetadata&
+ in_newSessionParams,
+ bool* _aidl_return) override;
+
+ ndk::ScopedAStatus processCaptureRequest(
+ const std::vector<::aidl::android::hardware::camera::device::CaptureRequest>&
+ in_requests,
+ const std::vector<::aidl::android::hardware::camera::device::BufferCache>&
+ in_cachesToRemove,
+ int32_t* _aidl_return) override;
+
+ ndk::ScopedAStatus signalStreamFlush(const std::vector<int32_t>& in_streamIds,
+ int32_t in_streamConfigCounter) override;
+
+ ndk::ScopedAStatus switchToOffline(
+ const std::vector<int32_t>& in_streamsToKeep,
+ ::aidl::android::hardware::camera::device::CameraOfflineSessionInfo*
+ out_offlineSessionInfo,
+ std::shared_ptr<
+ ::aidl::android::hardware::camera::device::ICameraOfflineSession>*
+ _aidl_return) override;
+
+ ndk::ScopedAStatus repeatingRequestEnd(
+ int32_t in_frameNumber, const std::vector<int32_t>& in_streamIds) override;
+
+ std::set<int> getStreamIds() const EXCLUDES(mLock);
+
+ private:
+ ndk::ScopedAStatus processCaptureRequest(
+ const ::aidl::android::hardware::camera::device::CaptureRequest& request)
+ EXCLUDES(mLock);
+
+ VirtualCameraDevice& mCameraDevice;
+
+ mutable std::mutex mLock;
+
+ std::shared_ptr<::aidl::android::hardware::camera::device::ICameraDeviceCallback>
+ mCameraDeviceCallback GUARDED_BY(mLock);
+
+ const std::shared_ptr<
+ ::aidl::android::companion::virtualcamera::IVirtualCameraCallback>
+ mVirtualCameraClientCallback;
+
+ VirtualCameraSessionContext mSessionContext;
+
+ using RequestMetadataQueue = AidlMessageQueue<
+ int8_t, ::aidl::android::hardware::common::fmq::SynchronizedReadWrite>;
+ std::unique_ptr<RequestMetadataQueue> mRequestMetadataQueue;
+
+ using ResultMetadataQueue = AidlMessageQueue<
+ int8_t, ::aidl::android::hardware::common::fmq::SynchronizedReadWrite>;
+ std::shared_ptr<ResultMetadataQueue> mResultMetadataQueue;
+
+ std::atomic_bool mFirstRequest{true};
+
+ std::unique_ptr<VirtualCameraRenderThread> mRenderThread GUARDED_BY(mLock);
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_SERVICES_VIRTUAL_CAMERA_VIRTUALCAMERASESSION_H
diff --git a/services/camera/virtualcamera/VirtualCameraSessionContext.cc b/services/camera/virtualcamera/VirtualCameraSessionContext.cc
new file mode 100644
index 0000000..284ad05
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraSessionContext.cc
@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "VirtualCameraSessionContext.h"
+
+#include <memory>
+#include <mutex>
+#include <unordered_set>
+
+#include "VirtualCameraStream.h"
+#include "aidl/android/hardware/camera/device/StreamConfiguration.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+using ::aidl::android::hardware::camera::device::BufferCache;
+using ::aidl::android::hardware::camera::device::Stream;
+using ::aidl::android::hardware::camera::device::StreamBuffer;
+using ::aidl::android::hardware::camera::device::StreamConfiguration;
+
+bool VirtualCameraSessionContext::initializeStream(
+ const ::aidl::android::hardware::camera::device::Stream& stream) {
+ std::lock_guard<std::mutex> lock(mLock);
+
+ auto s = std::make_unique<VirtualCameraStream>(stream);
+
+ const auto& [_, newlyInserted] = mStreams.emplace(
+ std::piecewise_construct, std::forward_as_tuple(stream.id),
+ std::forward_as_tuple(std::move(s)));
+ return newlyInserted;
+}
+
+void VirtualCameraSessionContext::closeAllStreams() {
+ std::lock_guard<std::mutex> lock(mLock);
+ mStreams.clear();
+}
+
+bool VirtualCameraSessionContext::importBuffersFromCaptureRequest(
+ const ::aidl::android::hardware::camera::device::CaptureRequest&
+ captureRequest) {
+ std::lock_guard<std::mutex> lock(mLock);
+
+ for (const StreamBuffer& buffer : captureRequest.outputBuffers) {
+ auto it = mStreams.find(buffer.streamId);
+ if (it == mStreams.end()) {
+ ALOGE("%s: Cannot import buffer for unknown stream with id %d", __func__,
+ buffer.streamId);
+ return false;
+ }
+ VirtualCameraStream& stream = *it->second;
+ if (stream.getHardwareBuffer(buffer.bufferId) != nullptr) {
+ // This buffer is already imported.
+ continue;
+ }
+
+ if (stream.importBuffer(buffer) == nullptr) {
+ ALOGE("%s: Failed to import buffer %" PRId64 " for streamId %d", __func__,
+ buffer.bufferId, buffer.streamId);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void VirtualCameraSessionContext::removeBufferCaches(
+ const std::vector<BufferCache>& cachesToRemove) {
+ std::lock_guard<std::mutex> lock(mLock);
+ for (const auto& bufferCache : cachesToRemove) {
+ auto it = mStreams.find(bufferCache.streamId);
+ if (it == mStreams.end()) {
+ ALOGE("%s: Ask to remove buffer %" PRId64 " from unknown stream %d",
+ __func__, bufferCache.bufferId, bufferCache.streamId);
+ continue;
+ }
+ if (it->second->removeBuffer(bufferCache.bufferId)) {
+ ALOGD("%s: Successfully removed buffer %" PRId64
+ " from cache of stream %d",
+ __func__, bufferCache.bufferId, bufferCache.streamId);
+ } else {
+ ALOGE("%s: Failed to remove buffer %" PRId64 " from cache of stream %d",
+ __func__, bufferCache.bufferId, bufferCache.streamId);
+ }
+ }
+}
+
+void VirtualCameraSessionContext::removeStreamsNotInStreamConfiguration(
+ const StreamConfiguration& streamConfiguration) {
+ std::unordered_set<int> newConfigurationStreamIds;
+ for (const Stream& stream : streamConfiguration.streams) {
+ newConfigurationStreamIds.insert(stream.id);
+ }
+
+ std::lock_guard<std::mutex> lock(mLock);
+ for (auto it = mStreams.begin(); it != mStreams.end();) {
+ if (newConfigurationStreamIds.find(it->first) ==
+ newConfigurationStreamIds.end()) {
+ ALOGV(
+ "Disposing of stream %d, since it is not referenced by new "
+ "configuration.",
+ it->first);
+ it = mStreams.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+std::optional<Stream> VirtualCameraSessionContext::getStreamConfig(
+ int streamId) const {
+ std::lock_guard<std::mutex> lock(mLock);
+ auto it = mStreams.find(streamId);
+ if (it == mStreams.end()) {
+ ALOGE("%s: StreamBuffer references buffer of unknown streamId %d", __func__,
+ streamId);
+ return std::optional<Stream>();
+ }
+ return {it->second->getStreamConfig()};
+}
+
+std::shared_ptr<AHardwareBuffer> VirtualCameraSessionContext::fetchHardwareBuffer(
+ const int streamId, const int bufferId) const {
+ std::lock_guard<std::mutex> lock(mLock);
+ auto it = mStreams.find(streamId);
+ if (it == mStreams.end()) {
+ ALOGE("%s: StreamBuffer references buffer of unknown streamId %d", __func__,
+ streamId);
+ return nullptr;
+ }
+ return it->second->getHardwareBuffer(bufferId);
+}
+
+std::shared_ptr<EglFrameBuffer>
+VirtualCameraSessionContext::fetchOrCreateEglFramebuffer(
+ const EGLDisplay eglDisplay, const int streamId, const int bufferId) {
+ std::lock_guard<std::mutex> lock(mLock);
+ auto it = mStreams.find(streamId);
+ if (it == mStreams.end()) {
+ ALOGE("%s: StreamBuffer references buffer of unknown streamId %d", __func__,
+ streamId);
+ return nullptr;
+ }
+ return it->second->getEglFrameBuffer(eglDisplay, bufferId);
+}
+
+std::set<int> VirtualCameraSessionContext::getStreamIds() const {
+ std::set<int> result;
+ std::lock_guard<std::mutex> lock(mLock);
+ for (const auto& [streamId, _] : mStreams) {
+ result.insert(streamId);
+ }
+ return result;
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/VirtualCameraSessionContext.h b/services/camera/virtualcamera/VirtualCameraSessionContext.h
new file mode 100644
index 0000000..e5cf5f4
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraSessionContext.h
@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASESSIONCONTEXT_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASESSIONCONTEXT_H
+
+#include <map>
+#include <memory>
+#include <mutex>
+#include <set>
+
+#include "VirtualCameraStream.h"
+#include "aidl/android/hardware/camera/device/BufferCache.h"
+#include "aidl/android/hardware/camera/device/CaptureRequest.h"
+#include "aidl/android/hardware/camera/device/Stream.h"
+#include "aidl/android/hardware/camera/device/StreamConfiguration.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Encapsulates set of streams belonging to the same camera session.
+class VirtualCameraSessionContext {
+ public:
+ // (Re)initialize the stream.
+ //
+ // Returns true if the stream is initialized for the first time.
+ bool initializeStream(
+ const ::aidl::android::hardware::camera::device::Stream& stream)
+ EXCLUDES(mLock);
+
+ // Close all streams and free all asociated buffers.
+ void closeAllStreams() EXCLUDES(mLock);
+
+ // Remove no longer needed buffers.
+ void removeBufferCaches(
+ const std::vector<::aidl::android::hardware::camera::device::BufferCache>&
+ cachesToRemove) EXCLUDES(mLock);
+
+ // Remove all streams not referenced by provided configuration.
+ void removeStreamsNotInStreamConfiguration(
+ const ::aidl::android::hardware::camera::device::StreamConfiguration&
+ streamConfiguration) EXCLUDES(mLock);
+
+ // Importored all not-yet imported buffers referenced by the capture request.
+ bool importBuffersFromCaptureRequest(
+ const ::aidl::android::hardware::camera::device::CaptureRequest&
+ captureRequest) EXCLUDES(mLock);
+
+ // Get stream configuration for provided stream id.
+ // Returns nullopt in case there's no stream with provided stream id.
+ std::optional<::aidl::android::hardware::camera::device::Stream>
+ getStreamConfig(int streamId) const EXCLUDES(mLock);
+
+ // Get hardware buffer for provided streamId & bufferId.
+ // Returns nullptr in case there's no such buffer.
+ std::shared_ptr<AHardwareBuffer> fetchHardwareBuffer(int streamId,
+ int bufferId) const
+ EXCLUDES(mLock);
+
+ // Get EGL framebuffer for provided EGL display, streamId & buffer id.
+ //
+ // This will also lazily create EglFrameBuffer for the provided EGLDisplay
+ // connection and will cache it (subsequent calls for same EGLDisplay and
+ // buffer will return same instance of EglFrameBuffer).
+ //
+ // Returns nullptr in case there's no such buffer or it was not possible
+ // to create EglFrameBuffer.
+ std::shared_ptr<EglFrameBuffer> fetchOrCreateEglFramebuffer(
+ const EGLDisplay eglDisplay, int streamId, int bufferId) EXCLUDES(mLock);
+
+ // Returns set of all stream ids managed by this instance.
+ std::set<int> getStreamIds() const EXCLUDES(mLock);
+
+ private:
+ mutable std::mutex mLock;
+ // streamId -> VirtualCameraStream mapping.
+ std::map<int, std::unique_ptr<VirtualCameraStream>> mStreams GUARDED_BY(mLock);
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASESSIONCONTEXT_H
diff --git a/services/camera/virtualcamera/VirtualCameraStream.cc b/services/camera/virtualcamera/VirtualCameraStream.cc
new file mode 100644
index 0000000..03da171
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraStream.cc
@@ -0,0 +1,189 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "VirtualCameraStream"
+#include "VirtualCameraStream.h"
+
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <tuple>
+#include <utility>
+
+#include "EGL/egl.h"
+#include "aidl/android/hardware/camera/device/Stream.h"
+#include "aidl/android/hardware/camera/device/StreamBuffer.h"
+#include "aidl/android/hardware/graphics/common/PixelFormat.h"
+#include "aidlcommonsupport/NativeHandle.h"
+#include "android/hardware_buffer.h"
+#include "cutils/native_handle.h"
+#include "ui/GraphicBuffer.h"
+#include "ui/GraphicBufferMapper.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+using ::aidl::android::hardware::camera::device::Stream;
+using ::aidl::android::hardware::camera::device::StreamBuffer;
+using ::aidl::android::hardware::common::NativeHandle;
+using ::aidl::android::hardware::graphics::common::PixelFormat;
+
+namespace {
+
+sp<GraphicBuffer> createBlobGraphicBuffer(GraphicBufferMapper& mapper,
+ buffer_handle_t bufferHandle) {
+ uint64_t allocationSize;
+ uint64_t usage;
+ uint64_t layerCount;
+ if (mapper.getAllocationSize(bufferHandle, &allocationSize) != NO_ERROR ||
+ mapper.getUsage(bufferHandle, &usage) != NO_ERROR ||
+ mapper.getLayerCount(bufferHandle, &layerCount) != NO_ERROR) {
+ ALOGE("Error fetching metadata for the imported BLOB buffer handle.");
+ return nullptr;
+ }
+
+ return sp<GraphicBuffer>::make(
+ bufferHandle, GraphicBuffer::HandleWrapMethod::TAKE_HANDLE,
+ allocationSize, /*height=*/1, static_cast<int>(ui::PixelFormat::BLOB),
+ layerCount, usage, 0);
+}
+
+sp<GraphicBuffer> createYCbCr420GraphicBuffer(GraphicBufferMapper& mapper,
+ buffer_handle_t bufferHandle) {
+ uint64_t width;
+ uint64_t height;
+ uint64_t usage;
+ uint64_t layerCount;
+ if (mapper.getWidth(bufferHandle, &width) != NO_ERROR ||
+ mapper.getHeight(bufferHandle, &height) != NO_ERROR ||
+ mapper.getUsage(bufferHandle, &usage) != NO_ERROR ||
+ mapper.getLayerCount(bufferHandle, &layerCount) != NO_ERROR) {
+ ALOGE("Error fetching metadata for the imported YCbCr420 buffer handle.");
+ return nullptr;
+ }
+
+ return sp<GraphicBuffer>::make(
+ bufferHandle, GraphicBuffer::HandleWrapMethod::TAKE_HANDLE, width, height,
+ static_cast<int>(ui::PixelFormat::YCBCR_420_888), /*layers=*/1, usage,
+ width);
+}
+
+std::shared_ptr<AHardwareBuffer> importBufferInternal(
+ const NativeHandle& aidlHandle, const Stream& streamConfig) {
+ if (aidlHandle.fds.empty()) {
+ ALOGE("Empty handle - nothing to import");
+ return nullptr;
+ }
+ std::unique_ptr<native_handle_t, int (*)(native_handle_t*)> nativeHandle(
+ ::android::makeFromAidl(aidlHandle), native_handle_delete);
+
+ GraphicBufferMapper& mapper = GraphicBufferMapper::get();
+
+ buffer_handle_t bufferHandle;
+ // Use importBufferNoValidate to rely on ground-truth metadata passed along
+ // the buffer.
+ int ret = mapper.importBufferNoValidate(nativeHandle.get(), &bufferHandle);
+ if (ret != NO_ERROR) {
+ ALOGE("Failed to import buffer handle: %d", ret);
+ return nullptr;
+ }
+
+ sp<GraphicBuffer> buf =
+ streamConfig.format == PixelFormat::BLOB
+ ? createBlobGraphicBuffer(mapper, bufferHandle)
+ : createYCbCr420GraphicBuffer(mapper, bufferHandle);
+
+ if (buf->initCheck() != NO_ERROR) {
+ ALOGE("Imported graphic buffer is not correcly initialized.");
+ return nullptr;
+ }
+
+ AHardwareBuffer* rawPtr = buf->toAHardwareBuffer();
+ AHardwareBuffer_acquire(rawPtr);
+
+ return std::shared_ptr<AHardwareBuffer>(buf->toAHardwareBuffer(),
+ AHardwareBuffer_release);
+}
+
+} // namespace
+
+VirtualCameraStream::VirtualCameraStream(const Stream& stream)
+ : mStreamConfig(stream) {
+}
+
+std::shared_ptr<AHardwareBuffer> VirtualCameraStream::importBuffer(
+ const ::aidl::android::hardware::camera::device::StreamBuffer& buffer) {
+ auto hwBufferPtr = importBufferInternal(buffer.buffer, mStreamConfig);
+ if (hwBufferPtr != nullptr) {
+ std::lock_guard<std::mutex> lock(mLock);
+ mBuffers.emplace(std::piecewise_construct,
+ std::forward_as_tuple(buffer.bufferId),
+ std::forward_as_tuple(hwBufferPtr));
+ }
+ return hwBufferPtr;
+}
+
+std::shared_ptr<AHardwareBuffer> VirtualCameraStream::getHardwareBuffer(
+ const int bufferId) {
+ std::lock_guard<std::mutex> lock(mLock);
+ return getHardwareBufferLocked(bufferId);
+}
+
+std::shared_ptr<EglFrameBuffer> VirtualCameraStream::getEglFrameBuffer(
+ const EGLDisplay eglDisplay, const int bufferId) {
+ const FramebufferMapKey key(bufferId, eglDisplay);
+
+ std::lock_guard<std::mutex> lock(mLock);
+
+ auto it = mEglFramebuffers.find(key);
+ if (it != mEglFramebuffers.end()) {
+ return it->second;
+ }
+
+ std::shared_ptr<AHardwareBuffer> hwBufferPtr =
+ getHardwareBufferLocked(bufferId);
+ if (hwBufferPtr == nullptr) {
+ return nullptr;
+ }
+ std::shared_ptr<EglFrameBuffer> framebufferPtr =
+ std::make_shared<EglFrameBuffer>(eglDisplay, hwBufferPtr);
+ mEglFramebuffers.emplace(std::piecewise_construct, std::forward_as_tuple(key),
+ std::forward_as_tuple(framebufferPtr));
+
+ return framebufferPtr;
+}
+
+std::shared_ptr<AHardwareBuffer> VirtualCameraStream::getHardwareBufferLocked(
+ const int bufferId) {
+ auto it = mBuffers.find(bufferId);
+ return it != mBuffers.end() ? it->second : nullptr;
+}
+
+bool VirtualCameraStream::removeBuffer(int bufferId) {
+ std::lock_guard<std::mutex> lock(mLock);
+
+ return mBuffers.erase(bufferId) == 1;
+}
+
+Stream VirtualCameraStream::getStreamConfig() const {
+ return mStreamConfig;
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/VirtualCameraStream.h b/services/camera/virtualcamera/VirtualCameraStream.h
new file mode 100644
index 0000000..d76d05c
--- /dev/null
+++ b/services/camera/virtualcamera/VirtualCameraStream.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASTREAM_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASTREAM_H
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <tuple>
+#include <unordered_map>
+
+#include "EGL/egl.h"
+#include "aidl/android/hardware/camera/device/Stream.h"
+#include "aidl/android/hardware/camera/device/StreamBuffer.h"
+#include "android/hardware_buffer.h"
+#include "util/EglFramebuffer.h"
+#include "utils/Mutex.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Encapsulates buffer management for the set of buffers belonging to the single
+// camera stream.
+class VirtualCameraStream {
+ public:
+ VirtualCameraStream(
+ const ::aidl::android::hardware::camera::device::Stream& stream);
+
+ std::shared_ptr<AHardwareBuffer> importBuffer(
+ const ::aidl::android::hardware::camera::device::StreamBuffer& streamBuffer);
+
+ // Get AHardwareBuffer instance corresponding to StreamBuffer from camera AIDL.
+ // In case this is the first occurrence of the buffer, this will perform mapping
+ // and stores hardware buffer in cache for further use.
+ //
+ // Returns nullptr in case buffer cannot be mapped or retrieved from the cache.
+ std::shared_ptr<AHardwareBuffer> getHardwareBuffer(int bufferId)
+ EXCLUDES(mLock);
+
+ std::shared_ptr<EglFrameBuffer> getEglFrameBuffer(const EGLDisplay eglDisplay,
+ int bufferId)
+ EXCLUDES(mLock);
+
+ // Un-maps the previously mapped buffer and removes it from the stream cache.
+ // Returns true if removal is successful, false otherwise.
+ bool removeBuffer(int bufferId) EXCLUDES(mLock);
+
+ // Returns AIDL Stream instance containing configuration of this stream.
+ ::aidl::android::hardware::camera::device::Stream getStreamConfig() const;
+
+ private:
+ std::shared_ptr<AHardwareBuffer> getHardwareBufferLocked(int bufferId)
+ REQUIRES(mLock);
+
+ const ::aidl::android::hardware::camera::device::Stream mStreamConfig;
+ std::mutex mLock;
+
+ // Cache for already mapped buffers, mapping bufferId -> AHardwareBuffer instance.
+ std::unordered_map<int, std::shared_ptr<AHardwareBuffer>> mBuffers
+ GUARDED_BY(mLock);
+
+ using FramebufferMapKey = std::pair<int, EGLDisplay>;
+ struct FramebufferMapKeyHash {
+ std::size_t operator()(const FramebufferMapKey& key) const {
+ return std::hash<int>{}(key.first) ^
+ (std::hash<void*>{}(reinterpret_cast<void*>(key.second)) << 1);
+ }
+ };
+ std::unordered_map<FramebufferMapKey, std::shared_ptr<EglFrameBuffer>,
+ FramebufferMapKeyHash>
+ mEglFramebuffers GUARDED_BY(mLock);
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERASTREAM_H
diff --git a/services/camera/virtualcamera/aidl/Android.bp b/services/camera/virtualcamera/aidl/Android.bp
new file mode 100644
index 0000000..9105b09
--- /dev/null
+++ b/services/camera/virtualcamera/aidl/Android.bp
@@ -0,0 +1,36 @@
+package {
+ // See: http://go/android-license-faq
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+aidl_interface {
+ name: "virtual_camera_service_aidl",
+ unstable: true,
+ srcs: [
+ "android/companion/virtualcamera/Format.aidl",
+ "android/companion/virtualcamera/IVirtualCameraCallback.aidl",
+ "android/companion/virtualcamera/IVirtualCameraService.aidl",
+ "android/companion/virtualcamera/VirtualCameraConfiguration.aidl",
+ "android/companion/virtualcamera/SupportedStreamConfiguration.aidl",
+ ],
+ local_include_dir: ".",
+ include_dirs: [
+ "frameworks/native/aidl/gui",
+ ],
+ backend: {
+ cpp: {
+ enabled: false,
+ },
+ ndk: {
+ enabled: true,
+ additional_shared_libraries: [
+ "libnativewindow",
+ ],
+ min_sdk_version: "34",
+ },
+ java: {
+ enabled: true,
+ platform_apis: true,
+ }
+ },
+}
diff --git a/services/camera/virtualcamera/aidl/android/companion/virtualcamera/Format.aidl b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/Format.aidl
new file mode 100644
index 0000000..d9b90a6
--- /dev/null
+++ b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/Format.aidl
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.companion.virtualcamera;
+
+/**
+ * Pixel format supported by a virtual camera stream.
+ *
+ * @hide
+ */
+@Backing(type="int")
+enum Format {
+ UNKNOWN = 0,
+ YUV_420_888 = 0x23,
+}
diff --git a/services/camera/virtualcamera/aidl/android/companion/virtualcamera/IVirtualCameraCallback.aidl b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/IVirtualCameraCallback.aidl
new file mode 100644
index 0000000..cbe03e9
--- /dev/null
+++ b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/IVirtualCameraCallback.aidl
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.companion.virtualcamera;
+
+import android.companion.virtualcamera.Format;
+import android.view.Surface;
+
+/**
+ * AIDL Interface to receive callbacks from virtual camera instance.
+ * @hide
+ */
+oneway interface IVirtualCameraCallback {
+
+ /**
+ * Called when there's new video stream. This callback is send after clients opens and
+ * configures camera. Implementation should hold onto the surface until corresponding
+ * terminateStream call is received.
+ *
+ * @param streamId - id of the video stream.
+ * @param surface - Surface representing the virtual camera sensor.
+ * @param width - width of the surface.
+ * @param height - height of the surface.
+ * @param pixelFormat - pixel format of the surface.
+ */
+ void onStreamConfigured(int streamId, in Surface surface, int width, int height, in Format pixelFormat);
+
+ /**
+ * Called when framework requests capture. This can be used by the client as a hint
+ * to render another frame into input surface.
+ *
+ * @param streamId - id of the stream corresponding to the Surface for which next
+ * frame is requested.
+ * @param frameId - id of the requested frame.
+ */
+ void onProcessCaptureRequest(int streamId, int frameId);
+
+ /**
+ * Called when the corresponding stream is no longer in use. Implementation should dispose of
+ * corresponding Surface upon receiving this call and no longer interact with it.
+ *
+ * @param streamId - id of the video stream to terminate.
+ */
+ void onStreamClosed(int streamId);
+}
diff --git a/services/camera/virtualcamera/aidl/android/companion/virtualcamera/IVirtualCameraService.aidl b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/IVirtualCameraService.aidl
new file mode 100644
index 0000000..bb74f5c
--- /dev/null
+++ b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/IVirtualCameraService.aidl
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.companion.virtualcamera;
+
+import android.companion.virtualcamera.VirtualCameraConfiguration;
+
+/**
+ * AIDL Interface to communicate with the VirtualCamera HAL
+ * @hide
+ */
+interface IVirtualCameraService {
+
+ /**
+ * Registers a new camera with the virtual camera hal.
+ * @return true if the camera was successfully registered
+ */
+ boolean registerCamera(in IBinder token, in VirtualCameraConfiguration configuration);
+
+ /**
+ * Unregisters the camera from the virtual camera hal. After this call the virtual camera won't
+ * be visible to the camera framework anymore.
+ */
+ void unregisterCamera(in IBinder token);
+
+ /**
+ * Returns the camera id for a given binder token. Note that this id corresponds to the id of
+ * the camera device in the camera framework.
+ */
+ int getCameraId(in IBinder token);
+}
diff --git a/services/camera/virtualcamera/aidl/android/companion/virtualcamera/SupportedStreamConfiguration.aidl b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/SupportedStreamConfiguration.aidl
new file mode 100644
index 0000000..7070cbd
--- /dev/null
+++ b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/SupportedStreamConfiguration.aidl
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.companion.virtualcamera;
+
+import android.companion.virtualcamera.Format;
+
+/**
+ * Configuration supported by virtual camera owner.
+ *
+ * @hide
+ */
+parcelable SupportedStreamConfiguration {
+ int width;
+ int height;
+ Format pixelFormat = Format.UNKNOWN;
+}
diff --git a/services/camera/virtualcamera/aidl/android/companion/virtualcamera/VirtualCameraConfiguration.aidl b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/VirtualCameraConfiguration.aidl
new file mode 100644
index 0000000..c1a2f22
--- /dev/null
+++ b/services/camera/virtualcamera/aidl/android/companion/virtualcamera/VirtualCameraConfiguration.aidl
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.companion.virtualcamera;
+
+import android.companion.virtualcamera.IVirtualCameraCallback;
+import android.companion.virtualcamera.SupportedStreamConfiguration;
+
+/**
+ * Configuration of virtual camera instance.
+ *
+ * @hide
+ */
+parcelable VirtualCameraConfiguration {
+ SupportedStreamConfiguration[] supportedStreamConfigs;
+ IVirtualCameraCallback virtualCameraCallback;
+}
diff --git a/services/camera/virtualcamera/fuzzer/Android.bp b/services/camera/virtualcamera/fuzzer/Android.bp
new file mode 100644
index 0000000..71e8f50
--- /dev/null
+++ b/services/camera/virtualcamera/fuzzer/Android.bp
@@ -0,0 +1,46 @@
+/******************************************************************************
+ *
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *****************************************************************************/
+ package {
+ // See: http://go/android-license-faq
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+cc_fuzz {
+ name: "virtual_camera_fuzzer",
+ defaults: [
+ "libvirtualcamera_defaults",
+ "service_fuzzer_defaults",
+ ],
+ static_libs: [
+ "libvirtualcamera",
+ "libvirtualcamera_utils",
+ ],
+ srcs: [
+ "virtual_camera_fuzzer.cc",
+ ],
+ fuzz_config: {
+ cc: [
+ "if-xr+fuzzer@google.com",
+ ],
+ componentid: 1171888,
+ hotlists: [
+ "5426614",
+ ],
+ description: "The fuzzers target the APIs of virtual_camera binary",
+ },
+}
diff --git a/services/camera/virtualcamera/fuzzer/virtual_camera_fuzzer.cc b/services/camera/virtualcamera/fuzzer/virtual_camera_fuzzer.cc
new file mode 100644
index 0000000..ebd5e73
--- /dev/null
+++ b/services/camera/virtualcamera/fuzzer/virtual_camera_fuzzer.cc
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <android-base/logging.h>
+#include <android/binder_interface_utils.h>
+#include <fuzzbinder/libbinder_ndk_driver.h>
+#include <fuzzer/FuzzedDataProvider.h>
+
+#include "VirtualCameraProvider.h"
+#include "VirtualCameraService.h"
+
+using android::fuzzService;
+using ::android::companion::virtualcamera::VirtualCameraProvider;
+using ::android::companion::virtualcamera::VirtualCameraService;
+using ndk::SharedRefBase;
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+ std::shared_ptr<VirtualCameraProvider> defaultProvider =
+ SharedRefBase::make<VirtualCameraProvider>();
+
+ fuzzService(defaultProvider->asBinder().get(), FuzzedDataProvider(data, size));
+
+ const std::string serviceName =
+ std::string(VirtualCameraProvider::descriptor) + "/virtual/0";
+ auto binder = SharedRefBase::make<VirtualCameraService>(defaultProvider);
+
+ fuzzService(binder->asBinder().get(), FuzzedDataProvider(data, size));
+ return 0;
+}
diff --git a/services/camera/virtualcamera/main.cc b/services/camera/virtualcamera/main.cc
new file mode 100644
index 0000000..b7e9c38
--- /dev/null
+++ b/services/camera/virtualcamera/main.cc
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "VirtualCamera"
+
+#include <android/binder_stability.h>
+
+#include <cstddef>
+
+#include "VirtualCameraProvider.h"
+#include "VirtualCameraService.h"
+#include "android-base/logging.h"
+#include "android/binder_manager.h"
+#include "android/binder_process.h"
+#include "log/log.h"
+
+using ::android::companion::virtualcamera::VirtualCameraProvider;
+using ::android::companion::virtualcamera::VirtualCameraService;
+
+namespace {
+// Default recommended RPC thread count for camera provider implementations
+const int HWBINDER_THREAD_COUNT = 6;
+
+constexpr char kVirtualCameraServiceName[] = "virtual_camera";
+} // namespace
+
+int main() {
+ ALOGI("CameraProvider: virtual webcam service is starting.");
+
+ ABinderProcess_setThreadPoolMaxThreadCount(HWBINDER_THREAD_COUNT);
+
+ std::shared_ptr<VirtualCameraProvider> defaultProvider =
+ ndk::SharedRefBase::make<VirtualCameraProvider>();
+ const std::string serviceName =
+ std::string(VirtualCameraProvider::descriptor) + "/virtual/0";
+
+ auto aidlBinder = defaultProvider->asBinder();
+ AIBinder_forceDowngradeToLocalStability(aidlBinder.get());
+ binder_exception_t ret =
+ AServiceManager_addService(aidlBinder.get(), serviceName.c_str());
+ LOG_ALWAYS_FATAL_IF(
+ ret != EX_NONE,
+ "Error while registering virtual camera provider service: %d", ret);
+
+ std::shared_ptr<VirtualCameraService> virtualCameraService =
+ ndk::SharedRefBase::make<VirtualCameraService>(defaultProvider);
+ ret = AServiceManager_addService(virtualCameraService->asBinder().get(),
+ kVirtualCameraServiceName);
+ LOG_ALWAYS_FATAL_IF(ret != EX_NONE,
+ "Error while registering virtual camera service: %d", ret);
+
+ ABinderProcess_joinThreadPool();
+ return EXIT_FAILURE; // should not reach
+}
diff --git a/services/camera/virtualcamera/tests/Android.bp b/services/camera/virtualcamera/tests/Android.bp
new file mode 100644
index 0000000..bc46ba0
--- /dev/null
+++ b/services/camera/virtualcamera/tests/Android.bp
@@ -0,0 +1,24 @@
+package {
+ // See: http://go/android-license-faq
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+cc_test {
+ name: "virtual_camera_tests",
+ defaults: [
+ "libvirtualcamera_defaults",
+ ],
+ static_libs: [
+ "libvirtualcamera",
+ "libvirtualcamera_utils",
+ "libgtest",
+ "libgmock",
+ ],
+ srcs: ["EglUtilTest.cc",
+ "VirtualCameraDeviceTest.cc",
+ "VirtualCameraProviderTest.cc",
+ "VirtualCameraRenderThreadTest.cc",
+ "VirtualCameraServiceTest.cc",
+ "VirtualCameraSessionTest.cc"],
+ test_suites: ["device-tests"],
+}
diff --git a/services/camera/virtualcamera/tests/EglUtilTest.cc b/services/camera/virtualcamera/tests/EglUtilTest.cc
new file mode 100644
index 0000000..d387ebf
--- /dev/null
+++ b/services/camera/virtualcamera/tests/EglUtilTest.cc
@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdint>
+#include "android/hardware_buffer.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "system/graphics.h"
+#include "ui/GraphicBuffer.h"
+#include "util/EglDisplayContext.h"
+#include "util/EglProgram.h"
+#include "util/EglSurfaceTexture.h"
+#include "util/EglUtil.h"
+#include "utils/Errors.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+namespace {
+
+using ::testing::Eq;
+using ::testing::NotNull;
+
+constexpr int kWidth = 64;
+constexpr int kHeight = 64;
+constexpr char kGlExtYuvTarget[] = "GL_EXT_YUV_target";
+
+uint8_t getY(const android_ycbcr& ycbcr, const int x, const int y) {
+ uint8_t* yPtr = reinterpret_cast<uint8_t*>(ycbcr.y);
+ return *(yPtr + ycbcr.ystride * y + x);
+}
+
+uint8_t getCb(const android_ycbcr& ycbcr, const int x, const int y) {
+ uint8_t* cbPtr = reinterpret_cast<uint8_t*>(ycbcr.cb);
+ return *(cbPtr + ycbcr.cstride * (y / 2) + (x / 2) * ycbcr.chroma_step);
+}
+
+uint8_t getCr(const android_ycbcr& ycbcr, const int x, const int y) {
+ uint8_t* crPtr = reinterpret_cast<uint8_t*>(ycbcr.cr);
+ return *(crPtr + ycbcr.cstride * (y / 2) + (x / 2) * ycbcr.chroma_step);
+}
+
+TEST(EglDisplayContextTest, SuccessfulInitialization) {
+ EglDisplayContext displayContext;
+
+ EXPECT_TRUE(displayContext.isInitialized());
+}
+
+class EglTest : public ::testing::Test {
+public:
+ void SetUp() override {
+ ASSERT_TRUE(mEglDisplayContext.isInitialized());
+ ASSERT_TRUE(mEglDisplayContext.makeCurrent());
+ }
+
+private:
+ EglDisplayContext mEglDisplayContext;
+};
+
+TEST_F(EglTest, EglTestPatternProgramSuccessfulInit) {
+ EglTestPatternProgram eglTestPatternProgram;
+
+ // Verify the shaders compiled and linked successfully.
+ EXPECT_TRUE(eglTestPatternProgram.isInitialized());
+}
+
+TEST_F(EglTest, EglTextureProgramSuccessfulInit) {
+ if (!isGlExtensionSupported(kGlExtYuvTarget)) {
+ GTEST_SKIP() << "Skipping test because of missing required GL extension " << kGlExtYuvTarget;
+ }
+
+ EglTextureProgram eglTextureProgram;
+
+ // Verify the shaders compiled and linked successfully.
+ EXPECT_TRUE(eglTextureProgram.isInitialized());
+}
+
+TEST_F(EglTest, EglSurfaceTextureBlackAfterInit) {
+ if (!isGlExtensionSupported(kGlExtYuvTarget)) {
+ GTEST_SKIP() << "Skipping test because of missing required GL extension " << kGlExtYuvTarget;
+ }
+
+ EglSurfaceTexture surfaceTexture(kWidth, kHeight);
+ surfaceTexture.updateTexture();
+ sp<GraphicBuffer> buffer = surfaceTexture.getCurrentBuffer();
+
+ ASSERT_THAT(buffer, NotNull());
+ const int width = buffer->getWidth();
+ const int height = buffer->getHeight();
+ ASSERT_THAT(width, Eq(kWidth));
+ ASSERT_THAT(height, Eq(kHeight));
+
+ android_ycbcr ycbcr;
+ status_t ret = buffer->lockYCbCr(AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, &ycbcr);
+ ASSERT_THAT(ret, Eq(NO_ERROR));
+ for (int i = 0; i < width; ++i) {
+ for (int j = 0; j < height; ++j) {
+ EXPECT_THAT(getY(ycbcr, i, j), Eq(0x00));
+ EXPECT_THAT(getCb(ycbcr, i, j), Eq(0x7f));
+ EXPECT_THAT(getCr(ycbcr, i, j), Eq(0x7f));
+ }
+ }
+
+ buffer->unlock();
+}
+
+} // namespace
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/tests/VirtualCameraDeviceTest.cc b/services/camera/virtualcamera/tests/VirtualCameraDeviceTest.cc
new file mode 100644
index 0000000..140ae65
--- /dev/null
+++ b/services/camera/virtualcamera/tests/VirtualCameraDeviceTest.cc
@@ -0,0 +1,214 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+
+#include "VirtualCameraDevice.h"
+#include "aidl/android/companion/virtualcamera/Format.h"
+#include "aidl/android/companion/virtualcamera/SupportedStreamConfiguration.h"
+#include "aidl/android/hardware/camera/device/CameraMetadata.h"
+#include "aidl/android/hardware/camera/device/StreamConfiguration.h"
+#include "android/binder_interface_utils.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "log/log_main.h"
+#include "system/camera_metadata.h"
+#include "utils/Errors.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+namespace {
+
+using ::aidl::android::companion::virtualcamera::Format;
+using ::aidl::android::companion::virtualcamera::SupportedStreamConfiguration;
+using ::aidl::android::hardware::camera::device::CameraMetadata;
+using ::aidl::android::hardware::camera::device::Stream;
+using ::aidl::android::hardware::camera::device::StreamConfiguration;
+using ::aidl::android::hardware::camera::device::StreamType;
+using ::aidl::android::hardware::graphics::common::PixelFormat;
+using ::testing::UnorderedElementsAreArray;
+using metadata_stream_t =
+ camera_metadata_enum_android_scaler_available_stream_configurations_t;
+
+constexpr int kCameraId = 42;
+constexpr int kVgaWidth = 640;
+constexpr int kVgaHeight = 480;
+constexpr int kHdWidth = 1280;
+constexpr int kHdHeight = 720;
+
+struct AvailableStreamConfiguration {
+ const int width;
+ const int height;
+ const int pixelFormat;
+ const metadata_stream_t streamConfiguration;
+};
+
+bool operator==(const AvailableStreamConfiguration& a,
+ const AvailableStreamConfiguration& b) {
+ return a.width == b.width && a.height == b.height &&
+ a.pixelFormat == b.pixelFormat &&
+ a.streamConfiguration == b.streamConfiguration;
+}
+
+std::ostream& operator<<(std::ostream& os,
+ const AvailableStreamConfiguration& config) {
+ os << config.width << "x" << config.height << " (pixfmt "
+ << config.pixelFormat << ", streamConfiguration "
+ << config.streamConfiguration << ")";
+ return os;
+}
+
+std::vector<AvailableStreamConfiguration> getAvailableStreamConfigurations(
+ const CameraMetadata& metadata) {
+ const camera_metadata_t* const raw =
+ reinterpret_cast<const camera_metadata_t*>(metadata.metadata.data());
+ camera_metadata_ro_entry_t entry;
+ if (find_camera_metadata_ro_entry(
+ raw, ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, &entry) !=
+ NO_ERROR) {
+ return {};
+ }
+
+ std::vector<AvailableStreamConfiguration> res;
+ for (int i = 0; i < entry.count; i += 4) {
+ res.push_back(AvailableStreamConfiguration{
+ .width = entry.data.i32[i + 1],
+ .height = entry.data.i32[i + 2],
+ .pixelFormat = entry.data.i32[i],
+ .streamConfiguration =
+ static_cast<metadata_stream_t>(entry.data.i32[i + 3])});
+ }
+ return res;
+}
+
+struct VirtualCameraConfigTestParam {
+ std::vector<SupportedStreamConfiguration> inputConfig;
+ std::vector<AvailableStreamConfiguration> expectedAvailableStreamConfigs;
+};
+
+class VirtualCameraDeviceTest
+ : public testing::TestWithParam<VirtualCameraConfigTestParam> {};
+
+TEST_P(VirtualCameraDeviceTest, cameraCharacteristicsForInputFormat) {
+ const VirtualCameraConfigTestParam& param = GetParam();
+ std::shared_ptr<VirtualCameraDevice> camera =
+ ndk::SharedRefBase::make<VirtualCameraDevice>(
+ kCameraId, param.inputConfig, /*virtualCameraClientCallback=*/nullptr);
+
+ CameraMetadata metadata;
+ ASSERT_TRUE(camera->getCameraCharacteristics(&metadata).isOk());
+ EXPECT_THAT(getAvailableStreamConfigurations(metadata),
+ UnorderedElementsAreArray(param.expectedAvailableStreamConfigs));
+
+ // Configuration needs to succeed for every available stream configuration
+ for (const AvailableStreamConfiguration& config :
+ param.expectedAvailableStreamConfigs) {
+ StreamConfiguration configuration{
+ .streams = std::vector<Stream>{Stream{
+ .streamType = StreamType::OUTPUT,
+ .width = config.width,
+ .height = config.height,
+ .format = static_cast<PixelFormat>(config.pixelFormat),
+ }}};
+ bool aidl_ret;
+ ASSERT_TRUE(
+ camera->isStreamCombinationSupported(configuration, &aidl_ret).isOk());
+ EXPECT_TRUE(aidl_ret);
+ }
+}
+
+INSTANTIATE_TEST_SUITE_P(
+ cameraCharacteristicsForInputFormat, VirtualCameraDeviceTest,
+ testing::Values(
+ VirtualCameraConfigTestParam{
+ .inputConfig = {SupportedStreamConfiguration{
+ .width = kVgaWidth,
+ .height = kVgaHeight,
+ .pixelFormat = Format::YUV_420_888}},
+ .expectedAvailableStreamConfigs =
+ {AvailableStreamConfiguration{
+ .width = kVgaWidth,
+ .height = kVgaHeight,
+ .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888,
+ .streamConfiguration =
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
+ AvailableStreamConfiguration{
+ .width = kVgaWidth,
+ .height = kVgaHeight,
+ .pixelFormat =
+ ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED,
+ .streamConfiguration =
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
+ AvailableStreamConfiguration{
+ .width = kVgaWidth,
+ .height = kVgaHeight,
+ .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB,
+ .streamConfiguration =
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT}}},
+ VirtualCameraConfigTestParam{
+ .inputConfig = {SupportedStreamConfiguration{
+ .width = kVgaWidth,
+ .height = kVgaHeight,
+ .pixelFormat = Format::YUV_420_888},
+ SupportedStreamConfiguration{
+ .width = kHdWidth,
+ .height = kHdHeight,
+ .pixelFormat = Format::YUV_420_888}},
+ .expectedAvailableStreamConfigs = {
+ AvailableStreamConfiguration{
+ .width = kVgaWidth,
+ .height = kVgaHeight,
+ .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888,
+ .streamConfiguration =
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
+ AvailableStreamConfiguration{
+ .width = kVgaWidth,
+ .height = kVgaHeight,
+ .pixelFormat =
+ ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED,
+ .streamConfiguration =
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
+ AvailableStreamConfiguration{
+ .width = kVgaWidth,
+ .height = kVgaHeight,
+ .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB,
+ .streamConfiguration =
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
+ AvailableStreamConfiguration{
+ .width = kHdWidth,
+ .height = kHdHeight,
+ .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888,
+ .streamConfiguration =
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
+ AvailableStreamConfiguration{
+ .width = kHdWidth,
+ .height = kHdHeight,
+ .pixelFormat =
+ ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED,
+ .streamConfiguration =
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
+ AvailableStreamConfiguration{
+ .width = kHdWidth,
+ .height = kHdHeight,
+ .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB,
+ .streamConfiguration =
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT}}}));
+
+} // namespace
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/tests/VirtualCameraProviderTest.cc b/services/camera/virtualcamera/tests/VirtualCameraProviderTest.cc
new file mode 100644
index 0000000..615a77c
--- /dev/null
+++ b/services/camera/virtualcamera/tests/VirtualCameraProviderTest.cc
@@ -0,0 +1,174 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+
+#include "VirtualCameraProvider.h"
+#include "aidl/android/hardware/camera/common/CameraDeviceStatus.h"
+#include "aidl/android/hardware/camera/common/Status.h"
+#include "aidl/android/hardware/camera/common/TorchModeStatus.h"
+#include "aidl/android/hardware/camera/provider/BnCameraProviderCallback.h"
+#include "android/binder_auto_utils.h"
+#include "android/binder_interface_utils.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "util/Util.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+namespace {
+
+using ::aidl::android::companion::virtualcamera::Format;
+using ::aidl::android::companion::virtualcamera::SupportedStreamConfiguration;
+using ::aidl::android::hardware::camera::common::CameraDeviceStatus;
+using ::aidl::android::hardware::camera::common::Status;
+using ::aidl::android::hardware::camera::common::TorchModeStatus;
+using ::aidl::android::hardware::camera::provider::BnCameraProviderCallback;
+using ::testing::_;
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::IsEmpty;
+using ::testing::IsNull;
+using ::testing::MatchesRegex;
+using ::testing::Not;
+using ::testing::Return;
+
+constexpr int kVgaWidth = 640;
+constexpr int kVgaHeight = 480;
+constexpr char kVirtualCameraNameRegex[] =
+ "device@[0-9]+\\.[0-9]+/virtual/[0-9]+";
+
+class MockCameraProviderCallback : public BnCameraProviderCallback {
+ public:
+ MOCK_METHOD(ndk::ScopedAStatus, cameraDeviceStatusChange,
+ (const std::string&, CameraDeviceStatus), (override));
+ MOCK_METHOD(ndk::ScopedAStatus, torchModeStatusChange,
+ (const std::string&, TorchModeStatus), (override));
+ MOCK_METHOD(ndk::ScopedAStatus, physicalCameraDeviceStatusChange,
+ (const std::string&, const std::string&, CameraDeviceStatus),
+ (override));
+};
+
+class VirtualCameraProviderTest : public ::testing::Test {
+ public:
+ void SetUp() override {
+ mCameraProvider = ndk::SharedRefBase::make<VirtualCameraProvider>();
+ mMockCameraProviderCallback =
+ ndk::SharedRefBase::make<MockCameraProviderCallback>();
+ ON_CALL(*mMockCameraProviderCallback, cameraDeviceStatusChange)
+ .WillByDefault([](const std::string&, CameraDeviceStatus) {
+ return ndk::ScopedAStatus::ok();
+ });
+ }
+
+ protected:
+ std::shared_ptr<VirtualCameraProvider> mCameraProvider;
+ std::shared_ptr<MockCameraProviderCallback> mMockCameraProviderCallback =
+ ndk::SharedRefBase::make<MockCameraProviderCallback>();
+ std::vector<SupportedStreamConfiguration> mInputConfigs = {
+ SupportedStreamConfiguration{.width = kVgaWidth,
+ .height = kVgaHeight,
+ .pixelFormat = Format::YUV_420_888}};
+};
+
+TEST_F(VirtualCameraProviderTest, SetNullCameraCallbackFails) {
+ // Attempting to set callback to nullptr should fail.
+ EXPECT_FALSE(mCameraProvider->setCallback(nullptr).isOk());
+}
+
+TEST_F(VirtualCameraProviderTest, NoCamerasInitially) {
+ std::vector<std::string> cameras;
+
+ // Initially, the camera provider should return empty list
+ // of cameras.
+ ASSERT_TRUE(mCameraProvider->getCameraIdList(&cameras).isOk());
+ EXPECT_THAT(cameras, IsEmpty());
+}
+
+TEST_F(VirtualCameraProviderTest, CreateCamera) {
+ // When new camera is created, we expect
+ // cameraDeviceStatusChange to be called exactly once with
+ // PRESENT status.
+ EXPECT_CALL(*mMockCameraProviderCallback,
+ cameraDeviceStatusChange(_, CameraDeviceStatus::PRESENT))
+ .WillOnce(Return(ndk::ScopedAStatus::ok()));
+
+ ASSERT_TRUE(mCameraProvider->setCallback(mMockCameraProviderCallback).isOk());
+ std::shared_ptr<VirtualCameraDevice> camera =
+ mCameraProvider->createCamera(mInputConfigs);
+ EXPECT_THAT(camera, Not(IsNull()));
+ EXPECT_THAT(camera->getCameraName(), MatchesRegex(kVirtualCameraNameRegex));
+
+ // Created camera should be in the list of cameras.
+ std::vector<std::string> cameraIds;
+ ASSERT_TRUE(mCameraProvider->getCameraIdList(&cameraIds).isOk());
+ EXPECT_THAT(cameraIds, ElementsAre(camera->getCameraName()));
+}
+
+TEST_F(VirtualCameraProviderTest, CreateCameraBeforeCallbackIsSet) {
+ // We expect cameraDeviceStatusChange to be invoked even when the
+ // setCallback configures the callback after camera is already created.
+ EXPECT_CALL(*mMockCameraProviderCallback,
+ cameraDeviceStatusChange(_, CameraDeviceStatus::PRESENT))
+ .WillOnce(Return(ndk::ScopedAStatus::ok()));
+
+ std::shared_ptr<VirtualCameraDevice> camera =
+ mCameraProvider->createCamera(mInputConfigs);
+ ASSERT_TRUE(mCameraProvider->setCallback(mMockCameraProviderCallback).isOk());
+
+ // Created camera should be in the list of cameras.
+ std::vector<std::string> cameraIds;
+ EXPECT_TRUE(mCameraProvider->getCameraIdList(&cameraIds).isOk());
+ EXPECT_THAT(cameraIds, ElementsAre(camera->getCameraName()));
+}
+
+TEST_F(VirtualCameraProviderTest, RemoveCamera) {
+ ASSERT_TRUE(mCameraProvider->setCallback(mMockCameraProviderCallback).isOk());
+ std::shared_ptr<VirtualCameraDevice> camera =
+ mCameraProvider->createCamera(mInputConfigs);
+
+ EXPECT_CALL(*mMockCameraProviderCallback,
+ cameraDeviceStatusChange(Eq(camera->getCameraName()),
+ CameraDeviceStatus::NOT_PRESENT))
+ .WillOnce(Return(ndk::ScopedAStatus::ok()));
+ EXPECT_TRUE(mCameraProvider->removeCamera(camera->getCameraName()));
+
+ // There are no cameras present after only camera is removed.
+ std::vector<std::string> cameraIds;
+ ASSERT_TRUE(mCameraProvider->getCameraIdList(&cameraIds).isOk());
+ EXPECT_THAT(cameraIds, IsEmpty());
+}
+
+TEST_F(VirtualCameraProviderTest, RemoveNonExistingCamera) {
+ ASSERT_TRUE(mCameraProvider->setCallback(mMockCameraProviderCallback).isOk());
+ std::shared_ptr<VirtualCameraDevice> camera =
+ mCameraProvider->createCamera(mInputConfigs);
+
+ // Removing non-existing camera should fail.
+ const std::string cameraName = "DefinitelyNoTCamera";
+ EXPECT_FALSE(mCameraProvider->removeCamera(cameraName));
+
+ // Camera should be still present in the camera list.
+ std::vector<std::string> cameraIds;
+ ASSERT_TRUE(mCameraProvider->getCameraIdList(&cameraIds).isOk());
+ EXPECT_THAT(cameraIds, ElementsAre(camera->getCameraName()));
+}
+
+} // namespace
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/tests/VirtualCameraRenderThreadTest.cc b/services/camera/virtualcamera/tests/VirtualCameraRenderThreadTest.cc
new file mode 100644
index 0000000..5f899b8
--- /dev/null
+++ b/services/camera/virtualcamera/tests/VirtualCameraRenderThreadTest.cc
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <sys/cdefs.h>
+
+#include <memory>
+
+#include "VirtualCameraRenderThread.h"
+#include "VirtualCameraSessionContext.h"
+#include "aidl/android/hardware/camera/common/CameraDeviceStatus.h"
+#include "aidl/android/hardware/camera/common/TorchModeStatus.h"
+#include "aidl/android/hardware/camera/device/BnCameraDeviceCallback.h"
+#include "aidl/android/hardware/camera/device/BufferRequest.h"
+#include "aidl/android/hardware/camera/device/BufferRequestStatus.h"
+#include "aidl/android/hardware/camera/device/BufferStatus.h"
+#include "aidl/android/hardware/camera/device/CaptureResult.h"
+#include "aidl/android/hardware/camera/device/NotifyMsg.h"
+#include "aidl/android/hardware/camera/device/StreamBuffer.h"
+#include "aidl/android/hardware/camera/device/StreamBufferRet.h"
+#include "android/binder_auto_utils.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+namespace {
+
+using ::aidl::android::hardware::camera::common::CameraDeviceStatus;
+using ::aidl::android::hardware::camera::common::TorchModeStatus;
+using ::aidl::android::hardware::camera::device::BnCameraDeviceCallback;
+using ::aidl::android::hardware::camera::device::BufferRequest;
+using ::aidl::android::hardware::camera::device::BufferRequestStatus;
+using ::aidl::android::hardware::camera::device::BufferStatus;
+using ::aidl::android::hardware::camera::device::CaptureResult;
+using ::aidl::android::hardware::camera::device::ErrorCode;
+using ::aidl::android::hardware::camera::device::ErrorMsg;
+using ::aidl::android::hardware::camera::device::NotifyMsg;
+using ::aidl::android::hardware::camera::device::StreamBuffer;
+using ::aidl::android::hardware::camera::device::StreamBufferRet;
+using ::testing::AllOf;
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::Field;
+using ::testing::Matcher;
+using ::testing::Property;
+using ::testing::Return;
+using ::testing::SizeIs;
+
+constexpr int kInputWidth = 640;
+constexpr int kInputHeight = 480;
+
+Matcher<StreamBuffer> IsStreamBufferWithStatus(const int streamId,
+ const int bufferId,
+ const BufferStatus status) {
+ return AllOf(Field(&StreamBuffer::streamId, Eq(streamId)),
+ Field(&StreamBuffer::bufferId, Eq(bufferId)),
+ Field(&StreamBuffer::status, Eq(status)));
+}
+
+Matcher<NotifyMsg> IsRequestErrorNotifyMsg(const int frameId) {
+ return AllOf(Property(&NotifyMsg::getTag, Eq(NotifyMsg::error)),
+ Property(&NotifyMsg::get<NotifyMsg::error>,
+ Field(&ErrorMsg::frameNumber, Eq(frameId))),
+ Property(&NotifyMsg::get<NotifyMsg::error>,
+ Field(&ErrorMsg::errorStreamId, Eq(-1))),
+ Property(&NotifyMsg::get<NotifyMsg::error>,
+ Field(&ErrorMsg::errorCode, Eq(ErrorCode::ERROR_REQUEST))));
+}
+
+class MockCameraDeviceCallback : public BnCameraDeviceCallback {
+ public:
+ MOCK_METHOD(ndk::ScopedAStatus, notify, (const std::vector<NotifyMsg>&),
+ (override));
+ MOCK_METHOD(ndk::ScopedAStatus, processCaptureResult,
+ (const std::vector<CaptureResult>&), (override));
+ MOCK_METHOD(ndk::ScopedAStatus, requestStreamBuffers,
+ (const std::vector<BufferRequest>&, std::vector<StreamBufferRet>*,
+ BufferRequestStatus*),
+ (override));
+ MOCK_METHOD(ndk::ScopedAStatus, returnStreamBuffers,
+ (const std::vector<StreamBuffer>&), (override));
+};
+
+class VirtualCameraRenderThreadTest : public ::testing::Test {
+ public:
+ void SetUp() override {
+ mSessionContext = std::make_unique<VirtualCameraSessionContext>();
+ mMockCameraDeviceCallback =
+ ndk::SharedRefBase::make<MockCameraDeviceCallback>();
+ mRenderThread = std::make_unique<VirtualCameraRenderThread>(
+ *mSessionContext, kInputWidth, kInputHeight, mMockCameraDeviceCallback);
+ }
+
+ protected:
+ std::unique_ptr<VirtualCameraSessionContext> mSessionContext;
+ std::unique_ptr<VirtualCameraRenderThread> mRenderThread;
+ std::shared_ptr<MockCameraDeviceCallback> mMockCameraDeviceCallback;
+};
+
+TEST_F(VirtualCameraRenderThreadTest, FlushReturnsErrorForInFlightRequests) {
+ const int frameNumber = 42;
+ const int firstStreamId = 1;
+ const int firstStreamBufferId = 1234;
+ const int secondStreamId = 7;
+ const int secondStreamBufferId = 4321;
+
+ // Notify should be called with the error set to corresponding frame.
+ EXPECT_CALL(*mMockCameraDeviceCallback,
+ notify(ElementsAre(IsRequestErrorNotifyMsg(frameNumber))))
+ .WillOnce(Return(ndk::ScopedAStatus::ok()));
+
+ // Process capture result should be called with all buffers in error state.
+ EXPECT_CALL(
+ *mMockCameraDeviceCallback,
+ processCaptureResult(ElementsAre(AllOf(
+ Field(&CaptureResult::frameNumber, frameNumber),
+ Field(&CaptureResult::outputBuffers,
+ testing::UnorderedElementsAre(
+ IsStreamBufferWithStatus(firstStreamId, firstStreamBufferId,
+ BufferStatus::ERROR),
+ IsStreamBufferWithStatus(secondStreamId, secondStreamBufferId,
+ BufferStatus::ERROR)))))))
+ .WillOnce([]() { return ndk::ScopedAStatus::ok(); });
+
+ mRenderThread->enqueueTask(std::make_unique<ProcessCaptureRequestTask>(
+ frameNumber,
+ std::vector<CaptureRequestBuffer>{
+ CaptureRequestBuffer(firstStreamId, firstStreamBufferId),
+ CaptureRequestBuffer(secondStreamId, secondStreamBufferId)}));
+
+ mRenderThread->flush();
+}
+
+} // namespace
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/tests/VirtualCameraServiceTest.cc b/services/camera/virtualcamera/tests/VirtualCameraServiceTest.cc
new file mode 100644
index 0000000..f931cb4
--- /dev/null
+++ b/services/camera/virtualcamera/tests/VirtualCameraServiceTest.cc
@@ -0,0 +1,292 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdio>
+#include <memory>
+
+#include "VirtualCameraService.h"
+#include "aidl/android/companion/virtualcamera/BnVirtualCameraCallback.h"
+#include "aidl/android/companion/virtualcamera/VirtualCameraConfiguration.h"
+#include "aidl/android/hardware/camera/provider/BnCameraProviderCallback.h"
+#include "aidl/android/hardware/graphics/common/PixelFormat.h"
+#include "android/binder_auto_utils.h"
+#include "android/binder_interface_utils.h"
+#include "android/binder_libbinder.h"
+#include "android/binder_status.h"
+#include "binder/Binder.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "utils/Errors.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+namespace {
+
+using ::aidl::android::companion::virtualcamera::BnVirtualCameraCallback;
+using ::aidl::android::companion::virtualcamera::Format;
+using ::aidl::android::companion::virtualcamera::VirtualCameraConfiguration;
+using ::aidl::android::hardware::camera::common::CameraDeviceStatus;
+using ::aidl::android::hardware::camera::common::TorchModeStatus;
+using ::aidl::android::hardware::camera::provider::BnCameraProviderCallback;
+using ::aidl::android::hardware::graphics::common::PixelFormat;
+using ::aidl::android::view::Surface;
+using ::testing::_;
+using ::testing::Eq;
+using ::testing::Ge;
+using ::testing::IsEmpty;
+using ::testing::IsNull;
+using ::testing::Not;
+using ::testing::SizeIs;
+
+constexpr int kVgaWidth = 640;
+constexpr int kVgaHeight = 480;
+
+const VirtualCameraConfiguration kEmptyVirtualCameraConfiguration;
+
+VirtualCameraConfiguration createConfiguration(const int width, const int height,
+ const Format format) {
+ VirtualCameraConfiguration configuration;
+ configuration.supportedStreamConfigs.push_back(
+ {.width = width, .height = height, .pixelFormat = format});
+ return configuration;
+}
+
+class MockCameraProviderCallback : public BnCameraProviderCallback {
+ public:
+ MOCK_METHOD(ndk::ScopedAStatus, cameraDeviceStatusChange,
+ (const std::string&, CameraDeviceStatus), (override));
+ MOCK_METHOD(ndk::ScopedAStatus, torchModeStatusChange,
+ (const std::string&, TorchModeStatus), (override));
+ MOCK_METHOD(ndk::ScopedAStatus, physicalCameraDeviceStatusChange,
+ (const std::string&, const std::string&, CameraDeviceStatus),
+ (override));
+};
+
+class VirtualCameraServiceTest : public ::testing::Test {
+ public:
+ void SetUp() override {
+ mCameraProvider = ndk::SharedRefBase::make<VirtualCameraProvider>();
+ mMockCameraProviderCallback =
+ ndk::SharedRefBase::make<MockCameraProviderCallback>();
+ ON_CALL(*mMockCameraProviderCallback, cameraDeviceStatusChange)
+ .WillByDefault([](const std::string&, CameraDeviceStatus) {
+ return ndk::ScopedAStatus::ok();
+ });
+ mCameraProvider->setCallback(mMockCameraProviderCallback);
+ mCameraService =
+ ndk::SharedRefBase::make<VirtualCameraService>(mCameraProvider);
+
+ mDevNullFd = open("/dev/null", O_RDWR);
+ ASSERT_THAT(mDevNullFd, Ge(0));
+ }
+
+ void createCamera() {
+ mOwnerToken = sp<BBinder>::make();
+ mNdkOwnerToken.set(AIBinder_fromPlatformBinder(mOwnerToken));
+ bool aidlRet;
+
+ ASSERT_TRUE(mCameraService
+ ->registerCamera(mNdkOwnerToken,
+ mVgaYUV420OnlyConfiguration, &aidlRet)
+ .isOk());
+ ASSERT_TRUE(aidlRet);
+ }
+
+ void TearDown() override {
+ close(mDevNullFd);
+ }
+
+ void execute_shell_command(const std::string cmd) {
+ std::array<const char*, 1> args{cmd.data()};
+ ASSERT_THAT(
+ mCameraService->handleShellCommand(mDevNullFd, mDevNullFd, mDevNullFd,
+ args.data(), args.size()),
+ Eq(NO_ERROR));
+ }
+
+ std::vector<std::string> getCameraIds() {
+ std::vector<std::string> cameraIds;
+ EXPECT_TRUE(mCameraProvider->getCameraIdList(&cameraIds).isOk());
+ return cameraIds;
+ }
+
+ protected:
+ std::shared_ptr<VirtualCameraService> mCameraService;
+ std::shared_ptr<VirtualCameraProvider> mCameraProvider;
+ std::shared_ptr<MockCameraProviderCallback> mMockCameraProviderCallback =
+ ndk::SharedRefBase::make<MockCameraProviderCallback>();
+
+ sp<BBinder> mOwnerToken;
+ ndk::SpAIBinder mNdkOwnerToken;
+
+ int mDevNullFd;
+
+ VirtualCameraConfiguration mVgaYUV420OnlyConfiguration =
+ createConfiguration(kVgaWidth, kVgaHeight, Format::YUV_420_888);
+};
+
+TEST_F(VirtualCameraServiceTest, RegisterCameraSucceeds) {
+ sp<BBinder> token = sp<BBinder>::make();
+ ndk::SpAIBinder ndkToken(AIBinder_fromPlatformBinder(token));
+ bool aidlRet;
+
+ ASSERT_TRUE(
+ mCameraService
+ ->registerCamera(ndkToken, mVgaYUV420OnlyConfiguration, &aidlRet)
+ .isOk());
+
+ EXPECT_TRUE(aidlRet);
+ EXPECT_THAT(getCameraIds(), SizeIs(1));
+}
+
+TEST_F(VirtualCameraServiceTest, RegisterCameraTwiceSecondReturnsFalse) {
+ createCamera();
+ bool aidlRet;
+
+ ASSERT_TRUE(mCameraService
+ ->registerCamera(mNdkOwnerToken, mVgaYUV420OnlyConfiguration,
+ &aidlRet)
+ .isOk());
+ EXPECT_FALSE(aidlRet);
+ EXPECT_THAT(getCameraIds(), SizeIs(1));
+}
+
+TEST_F(VirtualCameraServiceTest, EmptyConfigurationFails) {
+ bool aidlRet;
+
+ ASSERT_FALSE(mCameraService
+ ->registerCamera(mNdkOwnerToken,
+ kEmptyVirtualCameraConfiguration, &aidlRet)
+ .isOk());
+ EXPECT_FALSE(aidlRet);
+ EXPECT_THAT(getCameraIds(), IsEmpty());
+}
+
+TEST_F(VirtualCameraServiceTest, ConfigurationWithUnsupportedPixelFormatFails) {
+ bool aidlRet;
+
+ VirtualCameraConfiguration config =
+ createConfiguration(kVgaWidth, kVgaHeight, Format::UNKNOWN);
+
+ ASSERT_FALSE(
+ mCameraService->registerCamera(mNdkOwnerToken, config, &aidlRet).isOk());
+ EXPECT_FALSE(aidlRet);
+ EXPECT_THAT(getCameraIds(), IsEmpty());
+}
+
+TEST_F(VirtualCameraServiceTest, ConfigurationWithTooHighResFails) {
+ bool aidlRet;
+ VirtualCameraConfiguration config =
+ createConfiguration(1000000, 1000000, Format::YUV_420_888);
+
+ ASSERT_FALSE(
+ mCameraService->registerCamera(mNdkOwnerToken, config, &aidlRet).isOk());
+ EXPECT_FALSE(aidlRet);
+ EXPECT_THAT(getCameraIds(), IsEmpty());
+}
+
+TEST_F(VirtualCameraServiceTest, ConfigurationWithUnalignedResolutionFails) {
+ bool aidlRet;
+ VirtualCameraConfiguration config =
+ createConfiguration(641, 481, Format::YUV_420_888);
+
+ ASSERT_FALSE(
+ mCameraService->registerCamera(mNdkOwnerToken, config, &aidlRet).isOk());
+ EXPECT_FALSE(aidlRet);
+ EXPECT_THAT(getCameraIds(), IsEmpty());
+}
+
+TEST_F(VirtualCameraServiceTest, ConfigurationWithNegativeResolutionFails) {
+ bool aidlRet;
+ VirtualCameraConfiguration config =
+ createConfiguration(-1, kVgaHeight, Format::YUV_420_888);
+
+ ASSERT_FALSE(
+ mCameraService->registerCamera(mNdkOwnerToken, config, &aidlRet).isOk());
+ EXPECT_FALSE(aidlRet);
+ EXPECT_THAT(getCameraIds(), IsEmpty());
+}
+
+TEST_F(VirtualCameraServiceTest, GetCamera) {
+ createCamera();
+
+ EXPECT_THAT(mCameraService->getCamera(mNdkOwnerToken), Not(IsNull()));
+
+ sp<BBinder> otherToken = sp<BBinder>::make();
+ EXPECT_THAT(mCameraService->getCamera(
+ ndk::SpAIBinder(AIBinder_fromPlatformBinder(otherToken))),
+ IsNull());
+}
+
+TEST_F(VirtualCameraServiceTest, UnregisterCamera) {
+ createCamera();
+
+ EXPECT_THAT(mCameraService->getCamera(mNdkOwnerToken), Not(IsNull()));
+
+ mCameraService->unregisterCamera(mNdkOwnerToken);
+
+ EXPECT_THAT(mCameraService->getCamera(mNdkOwnerToken), IsNull());
+}
+
+TEST_F(VirtualCameraServiceTest, UnregisterCameraWithUnknownToken) {
+ createCamera();
+
+ EXPECT_THAT(mCameraService->getCamera(mNdkOwnerToken), Not(IsNull()));
+
+ auto otherToken = sp<BBinder>::make();
+ ndk::SpAIBinder ndkOtherToken(AIBinder_fromPlatformBinder(otherToken));
+ mCameraService->unregisterCamera(ndkOtherToken);
+
+ EXPECT_THAT(mCameraService->getCamera(mNdkOwnerToken), Not(IsNull()));
+}
+
+TEST_F(VirtualCameraServiceTest, ShellCmdWithNullArgs) {
+ EXPECT_EQ(mCameraService->handleShellCommand(
+ /*in=*/mDevNullFd, /*out=*/mDevNullFd, /*err=*/mDevNullFd,
+ /*args=*/nullptr, /*numArgs=*/1),
+ STATUS_BAD_VALUE);
+
+ std::array<const char*, 1> args{nullptr};
+ EXPECT_EQ(mCameraService->handleShellCommand(
+ /*in=*/mDevNullFd, /*out=*/mDevNullFd, /*err=*/mDevNullFd,
+ args.data(), /*numArgs=*/1),
+ STATUS_BAD_VALUE);
+}
+
+TEST_F(VirtualCameraServiceTest, ShellCmdWithNoArgs) {
+ EXPECT_EQ(mCameraService->handleShellCommand(
+ /*in=*/mDevNullFd, /*out=*/mDevNullFd, /*err=*/mDevNullFd,
+ /*args=*/nullptr, /*numArgs=*/0),
+ STATUS_OK);
+}
+
+TEST_F(VirtualCameraServiceTest, TestCameraShellCmd) {
+ execute_shell_command("enable_test_camera");
+
+ std::vector<std::string> cameraIdsAfterEnable = getCameraIds();
+ EXPECT_THAT(cameraIdsAfterEnable, SizeIs(1));
+
+ execute_shell_command("disable_test_camera");
+
+ std::vector<std::string> cameraIdsAfterDisable = getCameraIds();
+ EXPECT_THAT(cameraIdsAfterDisable, IsEmpty());
+}
+
+} // namespace
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/tests/VirtualCameraSessionTest.cc b/services/camera/virtualcamera/tests/VirtualCameraSessionTest.cc
new file mode 100644
index 0000000..9edfe77
--- /dev/null
+++ b/services/camera/virtualcamera/tests/VirtualCameraSessionTest.cc
@@ -0,0 +1,229 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdint>
+#include <memory>
+
+#include "VirtualCameraDevice.h"
+#include "VirtualCameraSession.h"
+#include "aidl/android/companion/virtualcamera/BnVirtualCameraCallback.h"
+#include "aidl/android/companion/virtualcamera/IVirtualCameraCallback.h"
+#include "aidl/android/companion/virtualcamera/SupportedStreamConfiguration.h"
+#include "aidl/android/hardware/camera/device/BnCameraDeviceCallback.h"
+#include "aidl/android/hardware/camera/device/StreamConfiguration.h"
+#include "aidl/android/hardware/graphics/common/PixelFormat.h"
+#include "android/binder_auto_utils.h"
+#include "android/binder_interface_utils.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "util/MetadataBuilder.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+namespace {
+
+constexpr int kWidth = 640;
+constexpr int kHeight = 480;
+constexpr int kStreamId = 0;
+constexpr int kCameraId = 42;
+
+using ::aidl::android::companion::virtualcamera::BnVirtualCameraCallback;
+using ::aidl::android::companion::virtualcamera::Format;
+using ::aidl::android::companion::virtualcamera::SupportedStreamConfiguration;
+using ::aidl::android::hardware::camera::device::BnCameraDeviceCallback;
+using ::aidl::android::hardware::camera::device::BufferRequest;
+using ::aidl::android::hardware::camera::device::BufferRequestStatus;
+using ::aidl::android::hardware::camera::device::CaptureRequest;
+using ::aidl::android::hardware::camera::device::CaptureResult;
+using ::aidl::android::hardware::camera::device::HalStream;
+using ::aidl::android::hardware::camera::device::NotifyMsg;
+using ::aidl::android::hardware::camera::device::Stream;
+using ::aidl::android::hardware::camera::device::StreamBuffer;
+using ::aidl::android::hardware::camera::device::StreamBufferRet;
+using ::aidl::android::hardware::camera::device::StreamConfiguration;
+using ::aidl::android::hardware::graphics::common::PixelFormat;
+using ::aidl::android::view::Surface;
+using ::testing::_;
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::Return;
+using ::testing::SizeIs;
+
+Stream createStream(int streamId, int width, int height, PixelFormat format) {
+ Stream s;
+ s.id = streamId;
+ s.width = width;
+ s.height = height;
+ s.format = format;
+ return s;
+}
+
+class MockCameraDeviceCallback : public BnCameraDeviceCallback {
+ public:
+ MOCK_METHOD(ndk::ScopedAStatus, notify, (const std::vector<NotifyMsg>&),
+ (override));
+ MOCK_METHOD(ndk::ScopedAStatus, processCaptureResult,
+ (const std::vector<CaptureResult>&), (override));
+ MOCK_METHOD(ndk::ScopedAStatus, requestStreamBuffers,
+ (const std::vector<BufferRequest>&, std::vector<StreamBufferRet>*,
+ BufferRequestStatus*),
+ (override));
+ MOCK_METHOD(ndk::ScopedAStatus, returnStreamBuffers,
+ (const std::vector<StreamBuffer>&), (override));
+};
+
+class MockVirtualCameraCallback : public BnVirtualCameraCallback {
+ public:
+ MOCK_METHOD(ndk::ScopedAStatus, onStreamConfigured,
+ (int, const Surface&, int32_t, int32_t, Format), (override));
+ MOCK_METHOD(ndk::ScopedAStatus, onProcessCaptureRequest, (int, int),
+ (override));
+ MOCK_METHOD(ndk::ScopedAStatus, onStreamClosed, (int), (override));
+};
+
+class VirtualCameraSessionTest : public ::testing::Test {
+ public:
+ void SetUp() override {
+ mMockCameraDeviceCallback =
+ ndk::SharedRefBase::make<MockCameraDeviceCallback>();
+ mMockVirtualCameraClientCallback =
+ ndk::SharedRefBase::make<MockVirtualCameraCallback>();
+ mVirtualCameraDevice = ndk::SharedRefBase::make<VirtualCameraDevice>(
+ kCameraId,
+ std::vector<SupportedStreamConfiguration>{
+ SupportedStreamConfiguration{.width = kWidth,
+ .height = kHeight,
+ .pixelFormat = Format::YUV_420_888}},
+ mMockVirtualCameraClientCallback);
+ mVirtualCameraSession = ndk::SharedRefBase::make<VirtualCameraSession>(
+ *mVirtualCameraDevice, mMockCameraDeviceCallback,
+ mMockVirtualCameraClientCallback);
+
+ // Explicitly defining default actions below to prevent gmock from
+ // default-constructing ndk::ScopedAStatus, because default-constructed
+ // status wraps nullptr AStatus and causes crash when attempting to print
+ // it in gtest report.
+ ON_CALL(*mMockCameraDeviceCallback, notify)
+ .WillByDefault(ndk::ScopedAStatus::ok);
+ ON_CALL(*mMockCameraDeviceCallback, processCaptureResult)
+ .WillByDefault(ndk::ScopedAStatus::ok);
+ ON_CALL(*mMockCameraDeviceCallback, requestStreamBuffers)
+ .WillByDefault(ndk::ScopedAStatus::ok);
+ ON_CALL(*mMockCameraDeviceCallback, returnStreamBuffers)
+ .WillByDefault(ndk::ScopedAStatus::ok);
+
+ ON_CALL(*mMockVirtualCameraClientCallback, onStreamConfigured)
+ .WillByDefault(ndk::ScopedAStatus::ok);
+ ON_CALL(*mMockVirtualCameraClientCallback, onProcessCaptureRequest)
+ .WillByDefault(ndk::ScopedAStatus::ok);
+ ON_CALL(*mMockVirtualCameraClientCallback, onStreamClosed)
+ .WillByDefault(ndk::ScopedAStatus::ok);
+ }
+
+ protected:
+ std::shared_ptr<MockCameraDeviceCallback> mMockCameraDeviceCallback;
+ std::shared_ptr<MockVirtualCameraCallback> mMockVirtualCameraClientCallback;
+ std::shared_ptr<VirtualCameraDevice> mVirtualCameraDevice;
+ std::shared_ptr<VirtualCameraSession> mVirtualCameraSession;
+};
+
+TEST_F(VirtualCameraSessionTest, ConfigureTriggersClientConfigureCallback) {
+ PixelFormat format = PixelFormat::YCBCR_420_888;
+ StreamConfiguration streamConfiguration;
+ streamConfiguration.streams = {
+ createStream(kStreamId, kWidth, kHeight, format)};
+ std::vector<HalStream> halStreams;
+ EXPECT_CALL(
+ *mMockVirtualCameraClientCallback,
+ onStreamConfigured(kStreamId, _, kWidth, kHeight, Format::YUV_420_888));
+
+ ASSERT_TRUE(
+ mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
+ .isOk());
+
+ EXPECT_THAT(halStreams, SizeIs(streamConfiguration.streams.size()));
+ EXPECT_THAT(mVirtualCameraSession->getStreamIds(), ElementsAre(0));
+}
+
+TEST_F(VirtualCameraSessionTest, SecondConfigureDropsUnreferencedStreams) {
+ PixelFormat format = PixelFormat::YCBCR_420_888;
+ StreamConfiguration streamConfiguration;
+ std::vector<HalStream> halStreams;
+
+ streamConfiguration.streams = {createStream(0, kWidth, kHeight, format),
+ createStream(1, kWidth, kHeight, format),
+ createStream(2, kWidth, kHeight, format)};
+ ASSERT_TRUE(
+ mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
+ .isOk());
+
+ EXPECT_THAT(mVirtualCameraSession->getStreamIds(), ElementsAre(0, 1, 2));
+
+ streamConfiguration.streams = {createStream(0, kWidth, kHeight, format),
+ createStream(2, kWidth, kHeight, format),
+ createStream(3, kWidth, kHeight, format)};
+ ASSERT_TRUE(
+ mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
+ .isOk());
+
+ EXPECT_THAT(mVirtualCameraSession->getStreamIds(), ElementsAre(0, 2, 3));
+}
+
+TEST_F(VirtualCameraSessionTest, CloseTriggersClientTerminateCallback) {
+ EXPECT_CALL(*mMockVirtualCameraClientCallback, onStreamClosed(kStreamId))
+ .WillOnce(Return(ndk::ScopedAStatus::ok()));
+
+ ASSERT_TRUE(mVirtualCameraSession->close().isOk());
+}
+
+TEST_F(VirtualCameraSessionTest, FlushBeforeConfigure) {
+ // Flush request coming before the configure request finished
+ // (so potentially the thread is not yet running) should be
+ // gracefully handled.
+
+ EXPECT_TRUE(mVirtualCameraSession->flush().isOk());
+}
+
+TEST_F(VirtualCameraSessionTest, onProcessCaptureRequestTriggersClientCallback) {
+ StreamConfiguration streamConfiguration;
+ streamConfiguration.streams = {
+ createStream(kStreamId, kWidth, kHeight, PixelFormat::YCBCR_420_888)};
+ std::vector<CaptureRequest> requests(1);
+ requests[0].frameNumber = 42;
+ requests[0].settings = *(
+ MetadataBuilder().setControlAfMode(ANDROID_CONTROL_AF_MODE_AUTO).build());
+
+ std::vector<HalStream> halStreams;
+ ASSERT_TRUE(
+ mVirtualCameraSession->configureStreams(streamConfiguration, &halStreams)
+ .isOk());
+
+ EXPECT_CALL(*mMockVirtualCameraClientCallback,
+ onProcessCaptureRequest(kStreamId, requests[0].frameNumber))
+ .WillOnce(Return(ndk::ScopedAStatus::ok()));
+ int32_t aidlReturn = 0;
+ ASSERT_TRUE(mVirtualCameraSession
+ ->processCaptureRequest(requests, /*in_cachesToRemove=*/{},
+ &aidlReturn)
+ .isOk());
+ EXPECT_THAT(aidlReturn, Eq(requests.size()));
+}
+
+} // namespace
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/util/EglDisplayContext.cc b/services/camera/virtualcamera/util/EglDisplayContext.cc
new file mode 100644
index 0000000..6d343a2
--- /dev/null
+++ b/services/camera/virtualcamera/util/EglDisplayContext.cc
@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "EglDisplayContext"
+#define EGL_EGLEXT_PROTOTYPES
+#define GL_GLEXT_PROTOTYPES
+
+#include "EglDisplayContext.h"
+
+#include "EGL/egl.h"
+#include "EglDisplayContext.h"
+#include "EglFramebuffer.h"
+#include "log/log.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+EglDisplayContext::EglDisplayContext()
+ : mEglDisplay(EGL_NO_DISPLAY),
+ mEglContext(EGL_NO_CONTEXT),
+ mEglConfig(nullptr) {
+ EGLBoolean result;
+
+ mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+ if (mEglDisplay == EGL_NO_DISPLAY) {
+ ALOGE("eglGetDisplay failed: %#x", eglGetError());
+ return;
+ }
+
+ EGLint majorVersion, minorVersion;
+ result = eglInitialize(mEglDisplay, &majorVersion, &minorVersion);
+ if (result != EGL_TRUE) {
+ ALOGE("eglInitialize failed: %#x", eglGetError());
+ return;
+ }
+ ALOGV("Initialized EGL v%d.%d", majorVersion, minorVersion);
+
+ EGLint numConfigs = 0;
+ EGLint configAttribs[] = {
+ EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, EGL_RENDERABLE_TYPE,
+ EGL_OPENGL_ES2_BIT, EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8,
+ // no alpha
+ EGL_NONE};
+
+ result =
+ eglChooseConfig(mEglDisplay, configAttribs, &mEglConfig, 1, &numConfigs);
+ if (result != EGL_TRUE) {
+ ALOGE("eglChooseConfig error: %#x", eglGetError());
+ return;
+ }
+
+ EGLint contextAttribs[] = {EGL_CONTEXT_MAJOR_VERSION_KHR, 3, EGL_NONE};
+ mEglContext =
+ eglCreateContext(mEglDisplay, mEglConfig, EGL_NO_CONTEXT, contextAttribs);
+ if (mEglContext == EGL_NO_CONTEXT) {
+ ALOGE("eglCreateContext error: %#x", eglGetError());
+ return;
+ }
+
+ if (!makeCurrent()) {
+ ALOGE(
+ "Failed to set newly initialized EGLContext and EGLDisplay connection "
+ "as current.");
+ } else {
+ ALOGV("EGL successfully initialized.");
+ }
+}
+
+EglDisplayContext::~EglDisplayContext() {
+ if (mEglDisplay != EGL_NO_DISPLAY) {
+ eglTerminate(mEglDisplay);
+ }
+ if (mEglContext != EGL_NO_CONTEXT) {
+ eglDestroyContext(mEglDisplay, mEglContext);
+ }
+ eglReleaseThread();
+}
+
+EGLDisplay EglDisplayContext::getEglDisplay() const {
+ return mEglDisplay;
+}
+
+bool EglDisplayContext::isInitialized() const {
+ return mEglContext != EGL_NO_CONTEXT && mEglDisplay != EGL_NO_DISPLAY;
+}
+
+bool EglDisplayContext::makeCurrent() {
+ if (!eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, mEglContext)) {
+ ALOGE("eglMakeCurrent failed: %#x", eglGetError());
+ return false;
+ }
+ return true;
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/util/EglDisplayContext.h b/services/camera/virtualcamera/util/EglDisplayContext.h
new file mode 100644
index 0000000..402ca3c
--- /dev/null
+++ b/services/camera/virtualcamera/util/EglDisplayContext.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_EGLDISPLAYCONTEXT_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_EGLDISPLAYCONTEXT_H
+
+#include "EGL/egl.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Encapsulated EGLDisplay & EGLContext.
+//
+// Upon construction, this object will create and configure new
+// EGLDisplay & EGLContext and will destroy them once it goes
+// out of scope.
+class EglDisplayContext {
+ public:
+ EglDisplayContext();
+ ~EglDisplayContext();
+
+ // Sets EGLDisplay & EGLContext for current thread.
+ //
+ // Returns true on success, false otherwise.
+ bool makeCurrent();
+
+ EGLDisplay getEglDisplay() const;
+
+ // Returns true if this instance encapsulates successfully initialized
+ // EGLDisplay & EGLContext.
+ bool isInitialized() const;
+
+ private:
+ EGLDisplay mEglDisplay;
+ EGLContext mEglContext;
+ EGLConfig mEglConfig;
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_EGLDISPLAYCONTEXT_H
diff --git a/services/camera/virtualcamera/util/EglFramebuffer.cc b/services/camera/virtualcamera/util/EglFramebuffer.cc
new file mode 100644
index 0000000..acf0122
--- /dev/null
+++ b/services/camera/virtualcamera/util/EglFramebuffer.cc
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "EglFramebuffer"
+#include "EglFramebuffer.h"
+
+#include "EGL/eglext.h"
+#include "EglUtil.h"
+#include "GLES/gl.h"
+#include "GLES2/gl2.h"
+#include "GLES2/gl2ext.h"
+#include "android/hardware_buffer.h"
+#include "log/log.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+EglFrameBuffer::EglFrameBuffer(EGLDisplay display,
+ std::shared_ptr<AHardwareBuffer> hwBuffer)
+ : mHardwareBuffer(hwBuffer), mEglDisplay(display) {
+ if (hwBuffer == nullptr) {
+ ALOGE("Cannot construct EglFramebuffer from null hwBuffer");
+ return;
+ }
+
+ AHardwareBuffer_Desc hwBufferDesc;
+ AHardwareBuffer_describe(hwBuffer.get(), &hwBufferDesc);
+ mWidth = hwBufferDesc.width;
+ mHeight = hwBufferDesc.height;
+
+ EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID(hwBuffer.get());
+ mEglImageKhr = eglCreateImageKHR(display, EGL_NO_CONTEXT,
+ EGL_NATIVE_BUFFER_ANDROID, clientBuffer, 0);
+ if (checkEglError("eglCreateImageKHR")) {
+ return;
+ }
+
+ // Create texture backed by the hardware buffer.
+ glGenTextures(1, &mTextureId);
+ glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureId);
+ glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES,
+ (GLeglImageOES)mEglImageKhr);
+ if (checkEglError("configure external texture")) {
+ return;
+ }
+
+ // Create framebuffer backed by the texture.
+ glGenFramebuffers(1, &mFramebufferId);
+ glBindFramebuffer(GL_FRAMEBUFFER, mFramebufferId);
+ glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
+ GL_TEXTURE_EXTERNAL_OES, mTextureId, 0);
+ GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
+ if (status != GL_FRAMEBUFFER_COMPLETE) {
+ ALOGE("Failed to configure framebuffer for texture");
+ return; // false;
+ }
+ if (checkEglError("glCheckFramebufferStatus")) {
+ return; // false;
+ }
+}
+
+EglFrameBuffer::~EglFrameBuffer() {
+ if (mFramebufferId != 0) {
+ glDeleteFramebuffers(1, &mFramebufferId);
+ }
+ if (mTextureId != 0) {
+ glDeleteTextures(1, &mTextureId);
+ }
+ if (mEglImageKhr != EGL_NO_IMAGE_KHR) {
+ eglDestroyImageKHR(mEglDisplay, mEglDisplay);
+ }
+}
+
+bool EglFrameBuffer::beforeDraw() {
+ glBindFramebuffer(GL_FRAMEBUFFER, mFramebufferId);
+ if (checkEglError("glBindFramebuffer")) {
+ return false;
+ }
+
+ glViewport(0, 0, mWidth, mHeight);
+
+ return true;
+}
+
+bool EglFrameBuffer::afterDraw() {
+ glFinish();
+ glBindFramebuffer(GL_FRAMEBUFFER, 0);
+ glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0);
+ return true;
+}
+
+int EglFrameBuffer::getWidth() const {
+ return mWidth;
+}
+
+int EglFrameBuffer::getHeight() const {
+ return mHeight;
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/util/EglFramebuffer.h b/services/camera/virtualcamera/util/EglFramebuffer.h
new file mode 100644
index 0000000..35f85e2
--- /dev/null
+++ b/services/camera/virtualcamera/util/EglFramebuffer.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_EGLFRAMEBUFFER_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_EGLFRAMEBUFFER_H
+
+#define EGL_EGLEXT_PROTOTYPES
+#define GL_GLEXT_PROTOTYPES
+
+#include <memory>
+
+#include "EGL/egl.h"
+#include "EGL/eglext.h"
+#include "GLES/gl.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Encapsulates EGL Framebuffer backed by AHardwareBuffer instance.
+//
+// Note that the framebuffer is tied to EGLDisplay connection.
+class EglFrameBuffer {
+ public:
+ EglFrameBuffer(EGLDisplay display, std::shared_ptr<AHardwareBuffer> hwBuffer);
+ virtual ~EglFrameBuffer();
+
+ // Prepare for rendering into the framebuffer.
+ bool beforeDraw();
+
+ // Finishes rendering into the framebuffer.
+ bool afterDraw();
+
+ // Return width of framebuffer (in pixels).
+ int getWidth() const;
+
+ // Return height of framebuffer (in pixels).
+ int getHeight() const;
+
+ private:
+ // Keeping shared_ptr to hardware buffer instance here prevents it from being
+ // freed while tied to EGL framebufer / EGL texture.
+ std::shared_ptr<AHardwareBuffer> mHardwareBuffer;
+ EGLDisplay mEglDisplay;
+ EGLImageKHR mEglImageKhr{EGL_NO_IMAGE_KHR};
+ GLuint mTextureId;
+ GLuint mFramebufferId;
+
+ int mWidth;
+ int mHeight;
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_EGLFRAMEBUFFER_H
diff --git a/services/camera/virtualcamera/util/EglProgram.cc b/services/camera/virtualcamera/util/EglProgram.cc
new file mode 100644
index 0000000..c468d39
--- /dev/null
+++ b/services/camera/virtualcamera/util/EglProgram.cc
@@ -0,0 +1,292 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "EglProgram"
+#include "EglProgram.h"
+
+#include <array>
+#include <complex>
+
+#include "EglUtil.h"
+#include "GLES/gl.h"
+#include "GLES2/gl2.h"
+#include "GLES2/gl2ext.h"
+#include "log/log.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+namespace {
+
+constexpr char kGlExtYuvTarget[] = "GL_EXT_YUV_target";
+
+constexpr char kIdentityVertexShader[] = R"(
+ attribute vec4 vPosition;
+ void main() {
+ gl_Position = vPosition;
+ })";
+
+constexpr char kJuliaFractalFragmentShader[] = R"(
+ precision mediump float;
+ uniform vec2 uResolution;
+ uniform vec2 uC;
+ uniform vec2 uUV;
+ const float kIter = 64.0;
+
+ vec2 imSq(vec2 n){
+ return vec2(pow(n.x,2.0)-pow(n.y,2.0), 2.0*n.x*n.y);
+ }
+
+ float julia(vec2 n, vec2 c) {
+ vec2 z = n;
+ for (float i=0.0;i<kIter; i+=1.0) {
+ z = imSq(z) + c;
+ if (length(z) > 2.0) return i/kIter;
+ }
+ return kIter;
+ }
+
+ void main() {
+ vec2 uv = vec2(gl_FragCoord.x / uResolution.x - 0.5, gl_FragCoord.y / uResolution.y - 0.5);
+ float juliaVal = julia(uv * 4.0, uC);
+ gl_FragColor = vec4( juliaVal,uUV.x,uUV.y,0.0);
+ })";
+
+constexpr char kExternalTextureVertexShader[] = R"(#version 300 es
+ in vec4 aPosition;
+ in vec2 aTextureCoord;
+ out vec2 vTextureCoord;
+ void main() {
+ gl_Position = aPosition;
+ vTextureCoord = aTextureCoord;
+ })";
+
+constexpr char kExternalTextureFragmentShader[] = R"(#version 300 es
+ #extension GL_OES_EGL_image_external_essl3 : require
+ #extension GL_EXT_YUV_target : require
+ precision mediump float;
+ in vec2 vTextureCoord;
+ out vec4 fragColor;
+ uniform __samplerExternal2DY2YEXT uTexture;
+ void main() {
+ fragColor = texture(uTexture, vTextureCoord);
+ })";
+
+constexpr int kCoordsPerVertex = 3;
+constexpr std::array<float, 12> kSquareCoords{-1.f, 1.0f, 0.0f, // top left
+ -1.f, -1.f, 0.0f, // bottom left
+ 1.0f, -1.f, 0.0f, // bottom right
+ 1.0f, 1.0f, 0.0f}; // top right
+
+constexpr std::array<float, 8> kTextureCoords{0.0f, 1.0f, // top left
+ 0.0f, 0.0f, // bottom left
+ 1.0f, 0.0f, // bottom right
+ 1.0f, 1.0f}; // top right
+
+constexpr std::array<uint8_t, 6> kDrawOrder{0, 1, 2, 0, 2, 3};
+
+GLuint compileShader(GLenum shaderType, const char* src) {
+ GLuint shader = glCreateShader(shaderType);
+ if (shader == 0) {
+ ALOGE("glCreateShader(shaderType=%x) error: %#x",
+ static_cast<unsigned int>(shaderType), glGetError());
+ return 0;
+ }
+
+ glShaderSource(shader, 1, &src, NULL);
+ glCompileShader(shader);
+
+ GLint compiled = 0;
+ glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
+ if (!compiled) {
+ ALOGE("Compile of shader type %d failed", shaderType);
+ GLint infoLen = 0;
+ glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen);
+ if (infoLen) {
+ char* buf = new char[infoLen];
+ if (buf) {
+ glGetShaderInfoLog(shader, infoLen, NULL, buf);
+ ALOGE("Compile log: %s", buf);
+ delete[] buf;
+ }
+ }
+ glDeleteShader(shader);
+ return 0;
+ }
+ return shader;
+}
+
+} // namespace
+
+EglProgram::~EglProgram() {
+ if (mProgram) {
+ glDeleteProgram(mProgram);
+ }
+}
+
+bool EglProgram::initialize(const char* vertexShaderSrc,
+ const char* fragmentShaderSrc) {
+ GLuint vertexShaderId = compileShader(GL_VERTEX_SHADER, vertexShaderSrc);
+ if (checkEglError("compileShader(vertex)")) {
+ return false;
+ }
+ GLuint fragmentShaderId = compileShader(GL_FRAGMENT_SHADER, fragmentShaderSrc);
+ if (checkEglError("compileShader(fragment)")) {
+ return false;
+ }
+
+ GLuint programId = glCreateProgram();
+
+ glAttachShader(programId, vertexShaderId);
+ glAttachShader(programId, fragmentShaderId);
+ glLinkProgram(programId);
+
+ GLint linkStatus = GL_FALSE;
+ glGetProgramiv(programId, GL_LINK_STATUS, &linkStatus);
+ if (linkStatus != GL_TRUE) {
+ ALOGE("glLinkProgram failed");
+ GLint bufLength = 0;
+ glGetProgramiv(programId, GL_INFO_LOG_LENGTH, &bufLength);
+ if (bufLength) {
+ char* buf = new char[bufLength];
+ if (buf) {
+ glGetProgramInfoLog(programId, bufLength, NULL, buf);
+ ALOGE("Link log: %s", buf);
+ delete[] buf;
+ }
+ }
+ glDeleteProgram(programId);
+ return false;
+ }
+
+ mProgram = programId;
+
+ mIsInitialized = true;
+ return mIsInitialized;
+}
+
+bool EglProgram::isInitialized() const {
+ return mIsInitialized;
+}
+
+EglTestPatternProgram::EglTestPatternProgram() {
+ if (initialize(kIdentityVertexShader, kJuliaFractalFragmentShader)) {
+ ALOGV("Successfully initialized EGL shaders for test pattern program.");
+ } else {
+ ALOGE("Test pattern EGL shader program initialization failed.");
+ }
+}
+
+bool EglTestPatternProgram::draw(int width, int height, int frameNumber) {
+ glViewport(0, 0, static_cast<GLsizei>(width), static_cast<GLsizei>(height));
+ checkEglError("glViewport");
+
+ // Load compiled shader.
+ glUseProgram(mProgram);
+ checkEglError("glUseProgram");
+
+ // Compute point in complex plane corresponding to fractal for this frame number.
+ float time = float(frameNumber) / 120.0f;
+ const std::complex<float> c(std::sin(time) * 0.78f, std::cos(time) * 0.78f);
+
+ // Pass uniform values to the shader.
+ int resolutionHandle = glGetUniformLocation(mProgram, "uResolution");
+ checkEglError("glGetUniformLocation -> uResolution");
+ glUniform2f(resolutionHandle, static_cast<float>(width),
+ static_cast<float>(height));
+ checkEglError("glUniform2f -> uResolution");
+
+ // Pass "C" constant value determining the Julia set to the shader.
+ int cHandle = glGetUniformLocation(mProgram, "uC");
+ glUniform2f(cHandle, c.imag(), c.real());
+
+ // Pass chroma value to the shader.
+ int uvHandle = glGetUniformLocation(mProgram, "uUV");
+ glUniform2f(uvHandle, (c.imag() + 1.f) / 2.f, (c.real() + 1.f) / 2.f);
+
+ // Pass vertex array to draw.
+ int positionHandle = glGetAttribLocation(mProgram, "vPosition");
+ glEnableVertexAttribArray(positionHandle);
+
+ // Prepare the triangle coordinate data.
+ glVertexAttribPointer(positionHandle, kCoordsPerVertex, GL_FLOAT, false,
+ kSquareCoords.size(), kSquareCoords.data());
+
+ // Draw triangle strip forming a square filling the viewport.
+ glDrawElements(GL_TRIANGLES, kDrawOrder.size(), GL_UNSIGNED_BYTE,
+ kDrawOrder.data());
+ if (checkEglError("glDrawElements")) {
+ return false;
+ }
+
+ return true;
+}
+
+EglTextureProgram::EglTextureProgram() {
+ if (!isGlExtensionSupported(kGlExtYuvTarget)) {
+ ALOGE(
+ "Cannot initialize external texture program due to missing "
+ "GL_EXT_YUV_target extension");
+ return;
+ }
+
+ if (initialize(kExternalTextureVertexShader, kExternalTextureFragmentShader)) {
+ ALOGV("Successfully initialized EGL shaders for external texture program.");
+ } else {
+ ALOGE("External texture EGL shader program initialization failed.");
+ }
+}
+
+bool EglTextureProgram::draw(GLuint textureId) {
+ // Load compiled shader.
+ glUseProgram(mProgram);
+ if (checkEglError("glUseProgram")) {
+ return false;
+ }
+
+ // Pass vertex array to the shader.
+ int positionHandle = glGetAttribLocation(mProgram, "aPosition");
+ glEnableVertexAttribArray(positionHandle);
+ glVertexAttribPointer(positionHandle, kCoordsPerVertex, GL_FLOAT, false,
+ kSquareCoords.size(), kSquareCoords.data());
+
+ // Pass texture coordinates corresponding to vertex array to the shader.
+ int textureCoordHandle = glGetAttribLocation(mProgram, "aTextureCoord");
+ glEnableVertexAttribArray(textureCoordHandle);
+ glVertexAttribPointer(textureCoordHandle, 2, GL_FLOAT, false,
+ kTextureCoords.size(), kTextureCoords.data());
+
+ // Configure texture for the shader.
+ int textureHandle = glGetUniformLocation(mProgram, "uTexture");
+ glActiveTexture(GL_TEXTURE0);
+ glBindTexture(GL_TEXTURE_EXTERNAL_OES, textureId);
+ glUniform1i(textureHandle, 0);
+
+ // Draw triangle strip forming a square filling the viewport.
+ glDrawElements(GL_TRIANGLES, kDrawOrder.size(), GL_UNSIGNED_BYTE,
+ kDrawOrder.data());
+ if (checkEglError("glDrawElements")) {
+ return false;
+ }
+
+ return true;
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/util/EglProgram.h b/services/camera/virtualcamera/util/EglProgram.h
new file mode 100644
index 0000000..8e394e7
--- /dev/null
+++ b/services/camera/virtualcamera/util/EglProgram.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_EGLPROGRAM_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_EGLPROGRAM_H
+
+#include "GLES/gl.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Base class for EGL Shader programs representation.
+class EglProgram {
+ public:
+ virtual ~EglProgram();
+
+ // Returns whether the EGL Program was successfully compiled and linked.
+ bool isInitialized() const;
+
+ protected:
+ // Compile & link program from the vertex & fragment shader source.
+ bool initialize(const char* vertexShaderSrc, const char* fragmentShaderSrc);
+ GLuint mProgram;
+ // Whether the EGL Program was successfully compiled and linked.
+ bool mIsInitialized = false;
+};
+
+// Shader program to draw Julia Set test pattern.
+class EglTestPatternProgram : public EglProgram {
+ public:
+ EglTestPatternProgram();
+
+ bool draw(int width, int height, int frameNumber);
+};
+
+// Shader program to draw texture.
+//
+// Shader stretches the texture over the viewport (if the texture is not same
+// aspect ratio as viewport, it will be deformed).
+//
+// TODO(b/301023410) Add support for translation / cropping.
+class EglTextureProgram : public EglProgram {
+ public:
+ EglTextureProgram();
+
+ bool draw(GLuint textureId);
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_EGLPROGRAM_H
diff --git a/services/camera/virtualcamera/util/EglSurfaceTexture.cc b/services/camera/virtualcamera/util/EglSurfaceTexture.cc
new file mode 100644
index 0000000..266d65a
--- /dev/null
+++ b/services/camera/virtualcamera/util/EglSurfaceTexture.cc
@@ -0,0 +1,109 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "EglSurfaceTexture"
+
+#include <cstdint>
+
+#include "EglSurfaceTexture.h"
+#include "EglUtil.h"
+#include "GLES/gl.h"
+#include "gui/BufferQueue.h"
+#include "gui/GLConsumer.h"
+#include "gui/IGraphicBufferProducer.h"
+#include "hardware/gralloc.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+namespace {
+
+void submitBlackBufferYCbCr420(Surface& surface) {
+ ANativeWindow_Buffer buffer;
+
+ int ret = surface.lock(&buffer, nullptr);
+ if (ret != NO_ERROR) {
+ ALOGE("%s: Cannot lock output surface: %d", __func__, ret);
+ return;
+ }
+ uint8_t* data = reinterpret_cast<uint8_t*>(buffer.bits);
+ const int yPixNr = buffer.width * buffer.height;
+ const int uvPixNr = (buffer.width / 2) * (buffer.height / 2);
+ memset(data, 0x00, yPixNr);
+ memset(data + yPixNr, 0x7f, 2 * uvPixNr);
+ surface.unlockAndPost();
+}
+
+} // namespace
+
+EglSurfaceTexture::EglSurfaceTexture(const uint32_t width, const uint32_t height)
+ : mWidth(width), mHeight(height) {
+ glGenTextures(1, &mTextureId);
+ if (checkEglError("EglSurfaceTexture(): glGenTextures")) {
+ ALOGE("Failed to generate texture");
+ return;
+ }
+ BufferQueue::createBufferQueue(&mBufferProducer, &mBufferConsumer);
+ mGlConsumer = sp<GLConsumer>::make(
+ mBufferConsumer, mTextureId, GLConsumer::TEXTURE_EXTERNAL, false, false);
+ mGlConsumer->setName(String8("VirtualCameraEglSurfaceTexture"));
+ mGlConsumer->setDefaultBufferSize(mWidth, mHeight);
+ mGlConsumer->setConsumerUsageBits(GRALLOC_USAGE_HW_TEXTURE);
+ mGlConsumer->setDefaultBufferFormat(AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420);
+
+ mSurface = sp<Surface>::make(mBufferProducer);
+ // Submit black buffer to the surface to make sure there's input buffer
+ // to process in case capture request comes before client writes something
+ // to the surface.
+ //
+ // Note that if the client does write something before capture request is
+ // processed (& updateTexture is called), this black buffer will be
+ // skipped (and recycled).
+ submitBlackBufferYCbCr420(*mSurface);
+}
+
+EglSurfaceTexture::~EglSurfaceTexture() {
+ if (mTextureId != 0) {
+ glDeleteTextures(1, &mTextureId);
+ }
+}
+
+sp<Surface> EglSurfaceTexture::getSurface() {
+ return mSurface;
+}
+
+sp<GraphicBuffer> EglSurfaceTexture::getCurrentBuffer() {
+ return mGlConsumer->getCurrentBuffer();
+}
+
+GLuint EglSurfaceTexture::updateTexture() {
+ mGlConsumer->updateTexImage();
+ return mTextureId;
+}
+
+uint32_t EglSurfaceTexture::getWidth() const {
+ return mWidth;
+}
+
+uint32_t EglSurfaceTexture::getHeight() const {
+ return mHeight;
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/util/EglSurfaceTexture.h b/services/camera/virtualcamera/util/EglSurfaceTexture.h
new file mode 100644
index 0000000..14dc7d6
--- /dev/null
+++ b/services/camera/virtualcamera/util/EglSurfaceTexture.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_EGLSURFACETEXTURE_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_EGLSURFACETEXTURE_H
+
+#include <cstdint>
+
+#include "GLES/gl.h"
+#include "gui/Surface.h"
+#include "utils/RefBase.h"
+
+namespace android {
+
+class IGraphicBufferProducer;
+class IGraphicBufferConsumer;
+class GLConsumer;
+
+namespace companion {
+namespace virtualcamera {
+
+// Encapsulates GLConsumer & Surface for rendering into EGL texture.
+class EglSurfaceTexture {
+ public:
+ // Create new EGL Texture with specified size.
+ EglSurfaceTexture(uint32_t width, uint32_t height);
+ ~EglSurfaceTexture();
+
+ // Get Surface backing up the texture.
+ sp<Surface> getSurface();
+
+ // Get GraphicBuffer backing the current texture.
+ sp<GraphicBuffer> getCurrentBuffer();
+
+ // Get width of surface / texture.
+ uint32_t getWidth() const;
+
+ // Get height of surface / texture.
+ uint32_t getHeight() const;
+
+ // Update the texture with the most recent submitted buffer.
+ // Most be called on thread with EGL context.
+ //
+ // Returns EGL texture id of the texture.
+ GLuint updateTexture();
+
+ private:
+ sp<IGraphicBufferProducer> mBufferProducer;
+ sp<IGraphicBufferConsumer> mBufferConsumer;
+ sp<GLConsumer> mGlConsumer;
+ sp<Surface> mSurface;
+ GLuint mTextureId;
+ const uint32_t mWidth;
+ const uint32_t mHeight;
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_EGLSURFACETEXTURE_H
diff --git a/services/camera/virtualcamera/util/EglUtil.cc b/services/camera/virtualcamera/util/EglUtil.cc
new file mode 100644
index 0000000..481d8f0
--- /dev/null
+++ b/services/camera/virtualcamera/util/EglUtil.cc
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "EglUtil"
+#include "EglUtil.h"
+
+#include <cstring>
+
+#include "GLES/gl.h"
+#include "log/log.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+bool checkEglError(const char* operation) {
+ GLenum err = glGetError();
+ if (err == GL_NO_ERROR) {
+ return false;
+ }
+ ALOGE("%s failed: %d", operation, err);
+ return true;
+}
+
+bool isGlExtensionSupported(const char* extension) {
+ const char* extensions =
+ reinterpret_cast<const char*>(glGetString(GL_EXTENSIONS));
+ if (extension == nullptr || extensions == nullptr) {
+ return false;
+ }
+ return strstr(extensions, extension) != nullptr;
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/util/EglUtil.h b/services/camera/virtualcamera/util/EglUtil.h
new file mode 100644
index 0000000..71640e3
--- /dev/null
+++ b/services/camera/virtualcamera/util/EglUtil.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_EGLUTIL_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_EGLUTIL_H
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Returns true if the EGL is in an error state and logs the error.
+bool checkEglError(const char* operation = "EGL operation");
+
+// Returns true if the GL extension is supported, false otherwise.
+bool isGlExtensionSupported(const char* extension);
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_EGLUTIL_H
diff --git a/services/camera/virtualcamera/util/JpegUtil.cc b/services/camera/virtualcamera/util/JpegUtil.cc
new file mode 100644
index 0000000..6f10376
--- /dev/null
+++ b/services/camera/virtualcamera/util/JpegUtil.cc
@@ -0,0 +1,228 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// #define LOG_NDEBUG 0
+#define LOG_TAG "JpegUtil"
+#include "JpegUtil.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+
+#include "android/hardware_buffer.h"
+#include "jpeglib.h"
+#include "log/log.h"
+#include "ui/GraphicBuffer.h"
+#include "ui/GraphicBufferMapper.h"
+#include "utils/Errors.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+namespace {
+
+constexpr int kJpegQuality = 80;
+
+class LibJpegContext {
+ public:
+ LibJpegContext(int width, int height, const android_ycbcr& ycbcr,
+ const size_t outBufferSize, void* outBuffer)
+ : mYCbCr(ycbcr),
+ mWidth(width),
+ mHeight(height),
+ mDstBufferSize(outBufferSize),
+ mDstBuffer(outBuffer) {
+ // Initialize error handling for libjpeg.
+ // We call jpeg_std_error to initialize standard error
+ // handling and then override:
+ // * output_message not to print to stderr, but use ALOG instead.
+ // * error_exit not to terminate the process, but failure flag instead.
+ mCompressStruct.err = jpeg_std_error(&mErrorMgr);
+ mCompressStruct.err->output_message = onOutputError;
+ mCompressStruct.err->error_exit = onErrorExit;
+ jpeg_create_compress(&mCompressStruct);
+
+ // Configure input image parameters.
+ mCompressStruct.image_width = width;
+ mCompressStruct.image_height = height;
+ mCompressStruct.input_components = 3;
+ mCompressStruct.in_color_space = JCS_YCbCr;
+ // We pass pointer to this instance as a client data so we can
+ // access this object from the static callbacks invoked by
+ // libjpeg.
+ mCompressStruct.client_data = this;
+
+ // Configure destination manager for libjpeg.
+ mCompressStruct.dest = &mDestinationMgr;
+ mDestinationMgr.init_destination = onInitDestination;
+ mDestinationMgr.empty_output_buffer = onEmptyOutputBuffer;
+ mDestinationMgr.term_destination = onTermDestination;
+ mDestinationMgr.next_output_byte = reinterpret_cast<JOCTET*>(mDstBuffer);
+ mDestinationMgr.free_in_buffer = mDstBufferSize;
+
+ // Configure everything else based on input configuration above.
+ jpeg_set_defaults(&mCompressStruct);
+
+ // Set quality and colorspace.
+ jpeg_set_quality(&mCompressStruct, kJpegQuality, 1);
+ jpeg_set_colorspace(&mCompressStruct, JCS_YCbCr);
+
+ // Configure RAW input mode - this let's libjpeg know we're providing raw,
+ // subsampled YCbCr data.
+ mCompressStruct.raw_data_in = 1;
+ mCompressStruct.dct_method = JDCT_IFAST;
+
+ // Configure sampling factors - this states that every 2 Y
+ // samples share 1 Cb & 1 Cr component vertically & horizontally (YUV420).
+ mCompressStruct.comp_info[0].h_samp_factor = 2;
+ mCompressStruct.comp_info[0].v_samp_factor = 2;
+ mCompressStruct.comp_info[1].h_samp_factor = 1;
+ mCompressStruct.comp_info[1].v_samp_factor = 1;
+ mCompressStruct.comp_info[2].h_samp_factor = 1;
+ mCompressStruct.comp_info[2].v_samp_factor = 1;
+ }
+
+ bool compress() {
+ // Prepare arrays of pointers to scanlines of each plane.
+ std::vector<JSAMPROW> yLines(mHeight);
+ std::vector<JSAMPROW> cbLines(mHeight / 2);
+ std::vector<JSAMPROW> crLines(mHeight / 2);
+
+ uint8_t* y = static_cast<uint8_t*>(mYCbCr.y);
+ uint8_t* cb = static_cast<uint8_t*>(mYCbCr.cb);
+ uint8_t* cr = static_cast<uint8_t*>(mYCbCr.cr);
+
+ // Since UV samples might be interleaved (semiplanar) we need to copy
+ // them to separate planes, since libjpeg doesn't directly
+ // support processing semiplanar YUV.
+ const int c_samples = (mWidth / 2) * (mHeight / 2);
+ std::vector<uint8_t> cb_plane(c_samples);
+ std::vector<uint8_t> cr_plane(c_samples);
+
+ // TODO(b/301023410) - Use libyuv or ARM SIMD for "unzipping" the data.
+ for (int i = 0; i < c_samples; ++i) {
+ cb_plane[i] = *cb;
+ cr_plane[i] = *cr;
+ cb += mYCbCr.chroma_step;
+ cr += mYCbCr.chroma_step;
+ }
+
+ // Collect pointers to individual scanline of each plane.
+ for (int i = 0; i < mHeight; ++i) {
+ yLines[i] = y + i * mYCbCr.ystride;
+ }
+ for (int i = 0; i < (mHeight / 2); ++i) {
+ cbLines[i] = cb_plane.data() + i * (mWidth / 2);
+ crLines[i] = cr_plane.data() + i * (mWidth / 2);
+ }
+
+ // Perform actual compression.
+ jpeg_start_compress(&mCompressStruct, TRUE);
+
+ while (mCompressStruct.next_scanline < mCompressStruct.image_height) {
+ const uint32_t batchSize = DCTSIZE * 2;
+ const uint32_t nl = mCompressStruct.next_scanline;
+ JSAMPARRAY planes[3]{&yLines[nl], &cbLines[nl / 2], &crLines[nl / 2]};
+
+ uint32_t done = jpeg_write_raw_data(&mCompressStruct, planes, batchSize);
+
+ if (done != batchSize) {
+ ALOGE("%s: compressed %u lines, expected %u (total %u/%u)",
+ __FUNCTION__, done, batchSize, mCompressStruct.next_scanline,
+ mCompressStruct.image_height);
+ return false;
+ }
+ }
+ jpeg_finish_compress(&mCompressStruct);
+ return mSuccess;
+ }
+
+ private:
+ void setSuccess(const boolean success) {
+ mSuccess = success;
+ }
+
+ void initDestination() {
+ mDestinationMgr.next_output_byte = reinterpret_cast<JOCTET*>(mDstBuffer);
+ mDestinationMgr.free_in_buffer = mDstBufferSize;
+ ALOGV("%s:%d jpeg start: %p [%zu]", __FUNCTION__, __LINE__, mDstBuffer,
+ mDstBufferSize);
+ }
+
+ void termDestination() {
+ mEncodedSize = mDstBufferSize - mDestinationMgr.free_in_buffer;
+ ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__, mEncodedSize);
+ }
+
+ // === libjpeg callbacks below ===
+
+ static void onOutputError(j_common_ptr cinfo) {
+ char buffer[JMSG_LENGTH_MAX];
+ (*cinfo->err->format_message)(cinfo, buffer);
+ ALOGE("libjpeg error: %s", buffer);
+ };
+
+ static void onErrorExit(j_common_ptr cinfo) {
+ static_cast<LibJpegContext*>(cinfo->client_data)->setSuccess(false);
+ };
+
+ static void onInitDestination(j_compress_ptr cinfo) {
+ static_cast<LibJpegContext*>(cinfo->client_data)->initDestination();
+ }
+
+ static int onEmptyOutputBuffer(j_compress_ptr cinfo __unused) {
+ ALOGV("%s:%d Out of buffer", __FUNCTION__, __LINE__);
+ return 0;
+ }
+
+ static void onTermDestination(j_compress_ptr cinfo) {
+ static_cast<LibJpegContext*>(cinfo->client_data)->termDestination();
+ }
+
+ jpeg_compress_struct mCompressStruct;
+ jpeg_error_mgr mErrorMgr;
+ jpeg_destination_mgr mDestinationMgr;
+
+ // Layout of the input image.
+ android_ycbcr mYCbCr;
+
+ // Dimensions of the input image.
+ int mWidth;
+ int mHeight;
+
+ // Destination buffer and it's capacity.
+ size_t mDstBufferSize;
+ void* mDstBuffer;
+
+ // This will be set to size of encoded data
+ // written to the outputBuffer when encoding finishes.
+ size_t mEncodedSize;
+ // Set to true/false based on whether the encoding
+ // was successful.
+ boolean mSuccess = true;
+};
+
+} // namespace
+
+// Returns true if the EGL is in an error state and logs the error.
+bool compressJpeg(int width, int height, const android_ycbcr& ycbcr,
+ size_t outBufferSize, void* outBuffer) {
+ return LibJpegContext(width, height, ycbcr, outBufferSize, outBuffer)
+ .compress();
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/util/JpegUtil.h b/services/camera/virtualcamera/util/JpegUtil.h
new file mode 100644
index 0000000..8bff008
--- /dev/null
+++ b/services/camera/virtualcamera/util/JpegUtil.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_JPEGUTIL_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_JPEGUTIL_H
+
+#include <memory>
+
+#include "android/hardware_buffer.h"
+#include "system/graphics.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Jpeg-compress image into the output buffer.
+bool compressJpeg(int width, int height, const android_ycbcr& ycbcr,
+ size_t outBufferSize, void* outBuffer);
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_JPEGUTIL_H
diff --git a/services/camera/virtualcamera/util/MetadataBuilder.cc b/services/camera/virtualcamera/util/MetadataBuilder.cc
new file mode 100644
index 0000000..b3b1a26
--- /dev/null
+++ b/services/camera/virtualcamera/util/MetadataBuilder.cc
@@ -0,0 +1,333 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "MetadataBuilder"
+
+#include "MetadataBuilder.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <iterator>
+#include <memory>
+#include <utility>
+#include <variant>
+#include <vector>
+
+#include "CameraMetadata.h"
+#include "aidl/android/hardware/camera/device/CameraMetadata.h"
+#include "log/log.h"
+#include "system/camera_metadata.h"
+#include "utils/Errors.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+namespace {
+
+using ::android::hardware::camera::common::helper::CameraMetadata;
+
+template <typename To, typename From>
+std::vector<To> convertTo(const std::vector<From>& from) {
+ std::vector<To> to;
+ to.reserve(from.size());
+ std::transform(from.begin(), from.end(), std::back_inserter(to),
+ [](const From& x) { return static_cast<To>(x); });
+ return to;
+}
+
+} // namespace
+
+MetadataBuilder& MetadataBuilder::setSupportedHardwareLevel(
+ camera_metadata_enum_android_info_supported_hardware_level_t hwLevel) {
+ mEntryMap[ANDROID_INFO_SUPPORTED_HARDWARE_LEVEL] =
+ std::vector<uint8_t>({static_cast<uint8_t>(hwLevel)});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setFlashAvailable(bool flashAvailable) {
+ const uint8_t metadataVal = flashAvailable
+ ? ANDROID_FLASH_INFO_AVAILABLE_TRUE
+ : ANDROID_FLASH_INFO_AVAILABLE_FALSE;
+ mEntryMap[ANDROID_FLASH_INFO_AVAILABLE] = std::vector<uint8_t>({metadataVal});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setLensFacing(
+ camera_metadata_enum_android_lens_facing lensFacing) {
+ mEntryMap[ANDROID_LENS_FACING] =
+ std::vector<uint8_t>({static_cast<uint8_t>(lensFacing)});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setSensorOrientation(int32_t sensorOrientation) {
+ mEntryMap[ANDROID_SENSOR_ORIENTATION] =
+ std::vector<int32_t>({sensorOrientation});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setSensorTimestamp(
+ std::chrono::nanoseconds timestamp) {
+ mEntryMap[ANDROID_SENSOR_TIMESTAMP] =
+ std::vector<int64_t>({timestamp.count()});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setAvailableFaceDetectModes(
+ const std::vector<camera_metadata_enum_android_statistics_face_detect_mode_t>&
+ faceDetectModes) {
+ mEntryMap[ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES] =
+ convertTo<uint8_t>(faceDetectModes);
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlAvailableModes(
+ const std::vector<camera_metadata_enum_android_control_mode_t>&
+ availableModes) {
+ mEntryMap[ANDROID_CONTROL_AVAILABLE_MODES] =
+ convertTo<uint8_t>(availableModes);
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlAfAvailableModes(
+ const std::vector<camera_metadata_enum_android_control_af_mode_t>&
+ availableModes) {
+ mEntryMap[ANDROID_CONTROL_AF_AVAILABLE_MODES] =
+ convertTo<uint8_t>(availableModes);
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlAfMode(
+ const camera_metadata_enum_android_control_af_mode_t mode) {
+ mEntryMap[ANDROID_CONTROL_AF_MODE] =
+ std::vector<uint8_t>({static_cast<uint8_t>(mode)});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlAeAvailableFpsRange(
+ const int32_t minFps, const int32_t maxFps) {
+ mEntryMap[ANDROID_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES] =
+ std::vector<int32_t>({minFps, maxFps});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlMaxRegions(int32_t maxAeRegions,
+ int32_t maxAwbRegions,
+ int32_t maxAfRegions) {
+ mEntryMap[ANDROID_CONTROL_MAX_REGIONS] =
+ std::vector<int32_t>({maxAeRegions, maxAwbRegions, maxAfRegions});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlAeRegions(
+ const std::vector<ControlRegion>& aeRegions) {
+ std::vector<int32_t> regions;
+ regions.reserve(5 * aeRegions.size());
+ for (const ControlRegion& region : aeRegions) {
+ regions.push_back(region.x0);
+ regions.push_back(region.y0);
+ regions.push_back(region.x1);
+ regions.push_back(region.y1);
+ regions.push_back(region.weight);
+ }
+ mEntryMap[ANDROID_CONTROL_AE_REGIONS] = std::move(regions);
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlAfRegions(
+ const std::vector<ControlRegion>& afRegions) {
+ std::vector<int32_t> regions;
+ regions.reserve(5 * afRegions.size());
+ for (const ControlRegion& region : afRegions) {
+ regions.push_back(region.x0);
+ regions.push_back(region.y0);
+ regions.push_back(region.x1);
+ regions.push_back(region.y1);
+ regions.push_back(region.weight);
+ }
+ mEntryMap[ANDROID_CONTROL_AF_REGIONS] = std::move(regions);
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlAwbRegions(
+ const std::vector<ControlRegion>& awbRegions) {
+ std::vector<int32_t> regions;
+ regions.reserve(5 * awbRegions.size());
+ for (const ControlRegion& region : awbRegions) {
+ regions.push_back(region.x0);
+ regions.push_back(region.y0);
+ regions.push_back(region.x1);
+ regions.push_back(region.y1);
+ regions.push_back(region.weight);
+ }
+ mEntryMap[ANDROID_CONTROL_AWB_REGIONS] = std::move(regions);
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlCaptureIntent(
+ const camera_metadata_enum_android_control_capture_intent_t intent) {
+ mEntryMap[ANDROID_CONTROL_CAPTURE_INTENT] =
+ std::vector<uint8_t>({static_cast<uint8_t>(intent)});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setMaxJpegSize(const int32_t size) {
+ mEntryMap[ANDROID_JPEG_MAX_SIZE] = std::vector<int32_t>({size});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setAvailableOutputStreamConfigurations(
+ const std::vector<StreamConfiguration>& streamConfigurations) {
+ std::vector<int32_t> metadataStreamConfigs;
+ std::vector<int64_t> metadataMinFrameDurations;
+ std::vector<int64_t> metadataStallDurations;
+ metadataStreamConfigs.reserve(streamConfigurations.size());
+ metadataMinFrameDurations.reserve(streamConfigurations.size());
+ metadataStallDurations.reserve(streamConfigurations.size());
+
+ for (const auto& config : streamConfigurations) {
+ metadataStreamConfigs.push_back(config.format);
+ metadataStreamConfigs.push_back(config.width);
+ metadataStreamConfigs.push_back(config.height);
+ metadataStreamConfigs.push_back(
+ ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT);
+
+ metadataMinFrameDurations.push_back(config.format);
+ metadataMinFrameDurations.push_back(config.width);
+ metadataMinFrameDurations.push_back(config.height);
+ metadataMinFrameDurations.push_back(config.minFrameDuration.count());
+
+ metadataStallDurations.push_back(config.format);
+ metadataStallDurations.push_back(config.width);
+ metadataStallDurations.push_back(config.height);
+ metadataStallDurations.push_back(config.minStallDuration.count());
+ }
+
+ mEntryMap[ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS] =
+ metadataStreamConfigs;
+ mEntryMap[ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS] =
+ metadataMinFrameDurations;
+ mEntryMap[ANDROID_SCALER_AVAILABLE_STALL_DURATIONS] =
+ metadataMinFrameDurations;
+
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setAvailableMaxDigitalZoom(const float maxZoom) {
+ mEntryMap[ANDROID_SCALER_AVAILABLE_MAX_DIGITAL_ZOOM] =
+ std::vector<float>(maxZoom);
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setSensorActiveArraySize(int x0, int y0,
+ int x1, int y1) {
+ mEntryMap[ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE] =
+ std::vector<int32_t>({x0, y0, x1, y1});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlAeCompensationRange(int32_t min,
+ int32_t max) {
+ mEntryMap[ANDROID_CONTROL_AE_COMPENSATION_RANGE] =
+ std::vector<int32_t>({min, max});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setControlAeCompensationStep(
+ const camera_metadata_rational step) {
+ mEntryMap[ANDROID_CONTROL_AE_COMPENSATION_STEP] =
+ std::vector<camera_metadata_rational>({step});
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setAvailableRequestKeys(
+ const std::vector<int32_t>& keys) {
+ mEntryMap[ANDROID_REQUEST_AVAILABLE_REQUEST_KEYS] = keys;
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setAvailableResultKeys(
+ const std::vector<int32_t>& keys) {
+ mEntryMap[ANDROID_REQUEST_AVAILABLE_RESULT_KEYS] = keys;
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setAvailableCapabilities(
+ const std::vector<camera_metadata_enum_android_request_available_capabilities_t>&
+ capabilities) {
+ mEntryMap[ANDROID_REQUEST_AVAILABLE_CAPABILITIES] =
+ convertTo<uint8_t>(capabilities);
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setAvailableCharacteristicKeys(
+ const std::vector<camera_metadata_tag_t>& keys) {
+ mEntryMap[ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS] =
+ convertTo<int32_t>(keys);
+ return *this;
+}
+
+MetadataBuilder& MetadataBuilder::setAvailableCharacteristicKeys() {
+ std::vector<camera_metadata_tag_t> availableKeys;
+ availableKeys.reserve(mEntryMap.size());
+ for (const auto& [key, _] : mEntryMap) {
+ if (key != ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS) {
+ availableKeys.push_back(key);
+ }
+ }
+ setAvailableCharacteristicKeys(availableKeys);
+ return *this;
+}
+
+std::unique_ptr<aidl::android::hardware::camera::device::CameraMetadata>
+MetadataBuilder::build() const {
+ CameraMetadata metadataHelper;
+ for (const auto& entry : mEntryMap) {
+ status_t ret = std::visit(
+ [&](auto&& arg) {
+ return metadataHelper.update(entry.first, arg.data(), arg.size());
+ },
+ entry.second);
+ if (ret != NO_ERROR) {
+ ALOGE("Failed to update metadata with key %d - %s: %s", entry.first,
+ get_camera_metadata_tag_name(entry.first),
+ ::android::statusToString(ret).c_str());
+ return nullptr;
+ }
+ }
+
+ const camera_metadata_t* metadata = metadataHelper.getAndLock();
+ if (metadata == nullptr) {
+ ALOGE(
+ "Failure when constructing metadata -> CameraMetadata helper returned "
+ "nullptr");
+ return nullptr;
+ }
+
+ auto aidlMetadata =
+ std::make_unique<aidl::android::hardware::camera::device::CameraMetadata>();
+ const uint8_t* data_ptr = reinterpret_cast<const uint8_t*>(metadata);
+ aidlMetadata->metadata.assign(data_ptr,
+ data_ptr + get_camera_metadata_size(metadata));
+ metadataHelper.unlock(metadata);
+
+ return aidlMetadata;
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/util/MetadataBuilder.h b/services/camera/virtualcamera/util/MetadataBuilder.h
new file mode 100644
index 0000000..2124398
--- /dev/null
+++ b/services/camera/virtualcamera/util/MetadataBuilder.h
@@ -0,0 +1,201 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_METADATABUILDER_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_METADATABUILDER_H
+
+#include <chrono>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <variant>
+#include <vector>
+
+#include "aidl/android/hardware/camera/device/CameraMetadata.h"
+#include "system/camera_metadata.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Convenience builder for the
+// aidl::android::hardware::camera::device::CameraMetadata.
+//
+// Calling the same builder setter multiple will overwrite the value.
+// This class is not thread-safe.
+class MetadataBuilder {
+ public:
+ struct StreamConfiguration {
+ int32_t width = 0;
+ int32_t height = 0;
+ int32_t format = 0;
+ // Minimal frame duration - corresponds to maximal FPS for given format.
+ // See ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS in CameraMetadataTag.aidl.
+ std::chrono::nanoseconds minFrameDuration{std::chrono::seconds(1) / 30};
+ // Minimal stall duration.
+ // See ANDROID_SCALER_AVAILABLE_STALL_DURATIONS in CameraMetadataTag.aidl.
+ std::chrono::nanoseconds minStallDuration{0};
+ };
+
+ struct ControlRegion {
+ int32_t x0 = 0;
+ int32_t y0 = 0;
+ int32_t x1 = 0;
+ int32_t y1 = 0;
+ int32_t weight = 0;
+ };
+
+ MetadataBuilder() = default;
+ ~MetadataBuilder() = default;
+
+ // See ANDROID_INFO_SUPPORTED_HARDWARE_LEVEL in CameraMetadataTag.aidl.
+ MetadataBuilder& setSupportedHardwareLevel(
+ camera_metadata_enum_android_info_supported_hardware_level_t hwLevel);
+
+ // Whether this camera device has a flash unit
+ // See ANDROID_FLASH_INFO_AVAILABLE in CameraMetadataTag.aidl.
+ MetadataBuilder& setFlashAvailable(bool flashAvailable);
+
+ // See ANDROID_LENS_FACING in CameraMetadataTag.aidl.
+ MetadataBuilder& setLensFacing(
+ camera_metadata_enum_android_lens_facing lensFacing);
+
+ // See ANDROID_SENSOR_ORIENTATION in CameraMetadataTag.aidl.
+ MetadataBuilder& setSensorOrientation(int32_t sensorOrientation);
+
+ // Time at start of exposure of first row of the image
+ // sensor active array, in nanoseconds.
+ //
+ // See ANDROID_SENSOR_TIMESTAMP in CameraMetadataTag.aidl.
+ MetadataBuilder& setSensorTimestamp(std::chrono::nanoseconds timestamp);
+
+ // See ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE in CameraMetadataTag.aidl.
+ MetadataBuilder& setSensorActiveArraySize(int x0, int y0, int x1, int y1);
+
+ // See ANDROID_STATISTICS_FACE_DETECT_MODE in CameraMetadataTag.aidl.
+ MetadataBuilder& setAvailableFaceDetectModes(
+ const std::vector<camera_metadata_enum_android_statistics_face_detect_mode_t>&
+ faceDetectMode);
+
+ // Sets available stream configurations along with corresponding minimal frame
+ // durations (corresponding to max fps) and stall durations.
+ //
+ // See ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS,
+ // ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS and
+ // ANDROID_SCALER_AVAILABLE_STALL_DURATIONS in CameraMetadataTag.aidl.
+ MetadataBuilder& setAvailableOutputStreamConfigurations(
+ const std::vector<StreamConfiguration>& streamConfigurations);
+
+ // See ANDROID_CONTROL_AVAILABLE_MODES in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlAvailableModes(
+ const std::vector<camera_metadata_enum_android_control_mode_t>&
+ availableModes);
+
+ // See ANDROID_CONTROL_AE_COMPENSATION_RANGE in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlAeCompensationRange(int32_t min, int32_t max);
+
+ // See ANDROID_CONTROL_AE_COMPENSATION_STEP in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlAeCompensationStep(camera_metadata_rational step);
+
+ // See ANDROID_CONTROL_AF_AVAILABLE_MODES in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlAfAvailableModes(
+ const std::vector<camera_metadata_enum_android_control_af_mode_t>&
+ availableModes);
+
+ // See ANDROID_CONTROL_AF_MODE in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlAfMode(
+ const camera_metadata_enum_android_control_af_mode_t mode);
+
+ // See ANDROID_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlAeAvailableFpsRange(int32_t min, int32_t max);
+
+ // See ANDROID_CONTROL_CAPTURE_INTENT in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlCaptureIntent(
+ camera_metadata_enum_android_control_capture_intent_t intent);
+
+ // See ANDROID_CONTROL_MAX_REGIONS in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlMaxRegions(int32_t maxAeRegions,
+ int32_t maxAwbRegions,
+ int32_t maxAfRegions);
+
+ // See ANDROID_CONTROL_AE_REGIONS in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlAeRegions(
+ const std::vector<ControlRegion>& aeRegions);
+
+ // See ANDROID_CONTROL_AWB_REGIONS in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlAwbRegions(
+ const std::vector<ControlRegion>& awbRegions);
+
+ // See ANDROID_CONTROL_AF_REGIONS in CameraMetadataTag.aidl.
+ MetadataBuilder& setControlAfRegions(
+ const std::vector<ControlRegion>& afRegions);
+
+ // The size of the compressed JPEG image, in bytes.
+ //
+ // See ANDROID_JPEG_SIZE in CameraMetadataTag.aidl.
+ MetadataBuilder& setMaxJpegSize(int32_t size);
+
+ // See ANDROID_SCALER_AVAILABLE_MAX_DIGITAL_ZOOM in CameraMetadataTag.aidl.
+ MetadataBuilder& setAvailableMaxDigitalZoom(const float maxZoom);
+
+ // A list of all keys that the camera device has available to use with
+ // CaptureRequest.
+ //
+ // See ANDROID_REQUEST_AVAILABLE_REQUEST_KEYS in CameraMetadataTag.aidl.
+ MetadataBuilder& setAvailableRequestKeys(const std::vector<int32_t>& keys);
+
+ // A list of all keys that the camera device has available to use with
+ // CaptureResult.
+ //
+ // See ANDROID_RESULT_AVAILABLE_REQUEST_KEYS in CameraMetadataTag.aidl.
+ MetadataBuilder& setAvailableResultKeys(const std::vector<int32_t>& keys);
+
+ // See ANDROID_REQUEST_AVAILABLE_CAPABILITIES in CameraMetadataTag.aidl.
+ MetadataBuilder& setAvailableCapabilities(
+ const std::vector<
+ camera_metadata_enum_android_request_available_capabilities_t>&
+ capabilities);
+
+ // A list of all keys that the camera device has available to use.
+ //
+ // See ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS in CameraMetadataTag.aidl.
+ MetadataBuilder& setAvailableCharacteristicKeys(
+ const std::vector<camera_metadata_tag_t>& keys);
+
+ // Extends metadata with ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS
+ // containing all previously set tags.
+ MetadataBuilder& setAvailableCharacteristicKeys();
+
+ // Build CameraMetadata instance.
+ //
+ // Returns nullptr in case something went wrong.
+ std::unique_ptr<::aidl::android::hardware::camera::device::CameraMetadata>
+ build() const;
+
+ private:
+ // Maps metadata tags to vectors of values for the given tag.
+ std::map<camera_metadata_tag_t,
+ std::variant<std::vector<int64_t>, std::vector<int32_t>,
+ std::vector<uint8_t>, std::vector<float>,
+ std::vector<camera_metadata_rational_t>>>
+ mEntryMap;
+};
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_METADATABUILDER_H
diff --git a/services/camera/virtualcamera/util/TestPatternHelper.cc b/services/camera/virtualcamera/util/TestPatternHelper.cc
new file mode 100644
index 0000000..a00a1b8
--- /dev/null
+++ b/services/camera/virtualcamera/util/TestPatternHelper.cc
@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "TestPatternHelper"
+
+#include "TestPatternHelper.h"
+
+#include <complex>
+#include <cstdint>
+
+#include "log/log.h"
+#include "utils/Errors.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+namespace {
+
+uint8_t julia(const std::complex<float> n, const std::complex<float> c) {
+ std::complex<float> z = n;
+ for (int i = 0; i < 64; i++) {
+ z = z * z + c;
+ if (std::abs(z) > 2.0) return i * 4;
+ }
+ return 0xff;
+}
+
+uint8_t pixelToFractal(const int x, const int y, const std::complex<float> c) {
+ std::complex<float> n(float(x) / 640.0f - 0.5, float(y) / 480.0f - 0.5);
+ return julia(n * 5.f, c);
+}
+
+void renderTestPatternYcbCr420(uint8_t* data_ptr, const int width,
+ const int height, const int frameNumber) {
+ float time = float(frameNumber) / 120.0f;
+ const std::complex<float> c(std::sin(time), std::cos(time));
+
+ uint8_t* y_data = data_ptr;
+ uint8_t* uv_data = static_cast<uint8_t*>(y_data + width * height);
+
+ for (int i = 0; i < width; ++i) {
+ for (int j = 0; j < height; ++j) {
+ y_data[j * width + i] = pixelToFractal(i, j, c * 0.78f);
+ if ((i & 1) && (j & 1)) {
+ uv_data[((j / 2) * (width / 2) + i / 2) * 2] =
+ static_cast<uint8_t>((float(i) / float(width)) * 255.f);
+ uv_data[((j / 2) * (width / 2) + i / 2) * 2 + 1] =
+ static_cast<uint8_t>((float(j) / float(height)) * 255.f);
+ }
+ }
+ }
+}
+
+} // namespace
+
+// This is just to see some meaningfull image in the buffer for testing, only
+// works with YcbCr420.
+void renderTestPatternYCbCr420(const std::shared_ptr<AHardwareBuffer> buffer,
+ const int frameNumber, const int fence) {
+ AHardwareBuffer_Planes planes_info;
+
+ AHardwareBuffer_Desc hwBufferDesc;
+ AHardwareBuffer_describe(buffer.get(), &hwBufferDesc);
+
+ const int width = hwBufferDesc.width;
+ const int height = hwBufferDesc.height;
+
+ int result = AHardwareBuffer_lockPlanes(buffer.get(),
+ AHARDWAREBUFFER_USAGE_CPU_READ_RARELY,
+ fence, nullptr, &planes_info);
+ if (result != OK) {
+ ALOGE("%s: Failed to lock planes: %d", __func__, result);
+ return;
+ }
+
+ renderTestPatternYcbCr420(
+ reinterpret_cast<uint8_t*>(planes_info.planes[0].data), width, height,
+ frameNumber);
+
+ AHardwareBuffer_unlock(buffer.get(), nullptr);
+}
+
+void renderTestPatternYCbCr420(sp<Surface> surface, int frameNumber) {
+ ANativeWindow_Buffer buffer;
+ surface->lock(&buffer, nullptr);
+
+ ALOGV("buffer: %dx%d stride %d, pixfmt %d", buffer.width, buffer.height,
+ buffer.stride, buffer.format);
+
+ renderTestPatternYcbCr420(reinterpret_cast<uint8_t*>(buffer.bits),
+ buffer.width, buffer.height, frameNumber);
+
+ surface->unlockAndPost();
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/util/TestPatternHelper.h b/services/camera/virtualcamera/util/TestPatternHelper.h
new file mode 100644
index 0000000..aca1cdd
--- /dev/null
+++ b/services/camera/virtualcamera/util/TestPatternHelper.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_TESTPATTERNHELPER_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_TESTPATTERNHELPER_H
+
+#include <memory>
+
+#include "android/hardware_buffer.h"
+#include "gui/Surface.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Helper function filling hardware buffer with test pattern for debugging /
+// testing purposes.
+void renderTestPatternYCbCr420(std::shared_ptr<AHardwareBuffer> buffer,
+ int frameNumber, int fence = -1);
+
+// Helper function for rendering test pattern into Surface.
+void renderTestPatternYCbCr420(sp<Surface> surface, int frameNumber);
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_TESTPATTERNHELPER_H
diff --git a/services/camera/virtualcamera/util/Util.cc b/services/camera/virtualcamera/util/Util.cc
new file mode 100644
index 0000000..11dc3a6
--- /dev/null
+++ b/services/camera/virtualcamera/util/Util.cc
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Util.h"
+
+#include <unistd.h>
+
+#include "jpeglib.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Lower bound for maximal supported texture size is at least 2048x2048
+// but on most platforms will be more.
+// TODO(b/301023410) - Query actual max texture size.
+constexpr int kMaxTextureSize = 2048;
+constexpr int kLibJpegDctSize = DCTSIZE;
+
+using ::aidl::android::companion::virtualcamera::Format;
+using ::aidl::android::hardware::common::NativeHandle;
+
+sp<Fence> importFence(const NativeHandle& aidlHandle) {
+ if (aidlHandle.fds.size() != 1) {
+ return sp<Fence>::make();
+ }
+
+ return sp<Fence>::make(::dup(aidlHandle.fds[0].get()));
+}
+
+// Returns true if specified format is supported for virtual camera input.
+bool isFormatSupportedForInput(const int width, const int height,
+ const Format format) {
+ if (format != Format::YUV_420_888) {
+ // For now only YUV_420_888 is supported for input.
+ return false;
+ }
+
+ if (width <= 0 || height <= 0 || width > kMaxTextureSize ||
+ height > kMaxTextureSize) {
+ return false;
+ }
+
+ if (width % kLibJpegDctSize != 0 || height % kLibJpegDctSize != 0) {
+ // Input dimension needs to be multiple of libjpeg DCT size.
+ // TODO(b/301023410) This restriction can be removed once we add support for
+ // unaligned jpeg compression.
+ return false;
+ }
+
+ return true;
+}
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
diff --git a/services/camera/virtualcamera/util/Util.h b/services/camera/virtualcamera/util/Util.h
new file mode 100644
index 0000000..b778321
--- /dev/null
+++ b/services/camera/virtualcamera/util/Util.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_COMPANION_VIRTUALCAMERA_UTIL_H
+#define ANDROID_COMPANION_VIRTUALCAMERA_UTIL_H
+
+#include <cstdint>
+
+#include "aidl/android/companion/virtualcamera/Format.h"
+#include "aidl/android/hardware/camera/common/Status.h"
+#include "aidl/android/hardware/camera/device/StreamBuffer.h"
+#include "android/binder_auto_utils.h"
+#include "ui/Fence.h"
+
+namespace android {
+namespace companion {
+namespace virtualcamera {
+
+// Converts camera AIDL status to ndk::ScopedAStatus
+inline ndk::ScopedAStatus cameraStatus(
+ const ::aidl::android::hardware::camera::common::Status status) {
+ return ndk::ScopedAStatus::fromServiceSpecificError(
+ static_cast<int32_t>(status));
+}
+
+// Import Fence from AIDL NativeHandle.
+//
+// If the handle can't be used to construct Fence (is empty or doesn't contain
+// only single fd) this function will return Fence instance in invalid state.
+sp<Fence> importFence(
+ const ::aidl::android::hardware::common::NativeHandle& handle);
+
+// Returns true if specified format is supported for virtual camera input.
+bool isFormatSupportedForInput(
+ int width, int height,
+ ::aidl::android::companion::virtualcamera::Format format);
+
+} // namespace virtualcamera
+} // namespace companion
+} // namespace android
+
+#endif // ANDROID_COMPANION_VIRTUALCAMERA_UTIL_H
diff --git a/services/camera/virtualcamera/virtual_camera.hal.rc b/services/camera/virtualcamera/virtual_camera.hal.rc
new file mode 100644
index 0000000..fd86965
--- /dev/null
+++ b/services/camera/virtualcamera/virtual_camera.hal.rc
@@ -0,0 +1,7 @@
+service virtual_camera /system/bin/virtual_camera
+ class core
+ user system
+ group system
+ capabilities SYS_NICE
+ rlimit rtprio 10 10
+ task_profiles CameraServiceCapacity CameraServicePerformance
diff --git a/services/mediaresourcemanager/ResourceManagerService.cpp b/services/mediaresourcemanager/ResourceManagerService.cpp
index 1643a83..1953237 100644
--- a/services/mediaresourcemanager/ResourceManagerService.cpp
+++ b/services/mediaresourcemanager/ResourceManagerService.cpp
@@ -24,169 +24,19 @@
#include <binder/IPCThreadState.h>
#include <binder/IServiceManager.h>
#include <cutils/sched_policy.h>
-#include <dirent.h>
#include <media/MediaResourcePolicy.h>
#include <media/stagefright/foundation/ABase.h>
#include <mediautils/BatteryNotifier.h>
#include <mediautils/ProcessInfo.h>
#include <mediautils/SchedulingPolicyService.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <unistd.h>
#include "IMediaResourceMonitor.h"
#include "ResourceManagerMetrics.h"
-#include "ResourceManagerService.h"
-#include "ResourceManagerServiceUtils.h"
#include "ResourceObserverService.h"
#include "ServiceLog.h"
namespace android {
-class DeathNotifier : public std::enable_shared_from_this<DeathNotifier> {
-
- // BinderDiedContext defines the cookie that is passed as DeathRecipient.
- // Since this can maintain more context than a raw pointer, we can
- // validate the scope of DeathNotifier, before deferencing it upon the binder death.
- struct BinderDiedContext {
- std::weak_ptr<DeathNotifier> mDeathNotifier;
- };
-public:
- static std::shared_ptr<DeathNotifier> Create(
- const std::shared_ptr<IResourceManagerClient>& client,
- const std::shared_ptr<ResourceManagerService>& service,
- const ClientInfoParcel& clientInfo,
- bool overrideProcessInfo = false);
-
- DeathNotifier(const std::shared_ptr<IResourceManagerClient>& client,
- const std::shared_ptr<ResourceManagerService>& service,
- const ClientInfoParcel& clientInfo);
-
- virtual ~DeathNotifier() {
- unlink();
- }
-
- // Implement death recipient
- static void BinderDiedCallback(void* cookie);
- static void BinderUnlinkedCallback(void* cookie);
- virtual void binderDied();
-
-private:
- void link() {
- // Create the context that is passed as cookie to the binder death notification.
- // The context gets deleted at BinderUnlinkedCallback.
- mCookie = new BinderDiedContext{.mDeathNotifier = weak_from_this()};
- // Register for the callbacks by linking to death notification.
- AIBinder_linkToDeath(mClient->asBinder().get(), mDeathRecipient.get(), mCookie);
- }
-
- void unlink() {
- if (mClient != nullptr) {
- // Unlink from the death notification.
- AIBinder_unlinkToDeath(mClient->asBinder().get(), mDeathRecipient.get(), mCookie);
- mClient = nullptr;
- }
- }
-
-protected:
- std::shared_ptr<IResourceManagerClient> mClient;
- std::weak_ptr<ResourceManagerService> mService;
- const ClientInfoParcel mClientInfo;
- BinderDiedContext* mCookie;
- ::ndk::ScopedAIBinder_DeathRecipient mDeathRecipient;
-};
-
-DeathNotifier::DeathNotifier(const std::shared_ptr<IResourceManagerClient>& client,
- const std::shared_ptr<ResourceManagerService>& service,
- const ClientInfoParcel& clientInfo)
- : mClient(client), mService(service), mClientInfo(clientInfo),
- mCookie(nullptr),
- mDeathRecipient(::ndk::ScopedAIBinder_DeathRecipient(
- AIBinder_DeathRecipient_new(BinderDiedCallback))) {
- // Setting callback notification when DeathRecipient gets deleted.
- AIBinder_DeathRecipient_setOnUnlinked(mDeathRecipient.get(), BinderUnlinkedCallback);
-}
-
-//static
-void DeathNotifier::BinderUnlinkedCallback(void* cookie) {
- BinderDiedContext* context = reinterpret_cast<BinderDiedContext*>(cookie);
- // Since we don't need the context anymore, we are deleting it now.
- delete context;
-}
-
-//static
-void DeathNotifier::BinderDiedCallback(void* cookie) {
- BinderDiedContext* context = reinterpret_cast<BinderDiedContext*>(cookie);
-
- // Validate the context and check if the DeathNotifier object is still in scope.
- if (context != nullptr) {
- std::shared_ptr<DeathNotifier> thiz = context->mDeathNotifier.lock();
- if (thiz != nullptr) {
- thiz->binderDied();
- } else {
- ALOGI("DeathNotifier is out of scope already");
- }
- }
-}
-
-void DeathNotifier::binderDied() {
- // Don't check for pid validity since we know it's already dead.
- std::shared_ptr<ResourceManagerService> service = mService.lock();
- if (service == nullptr) {
- ALOGW("ResourceManagerService is dead as well.");
- return;
- }
-
- service->overridePid(mClientInfo.pid, -1);
- // thiz is freed in the call below, so it must be last call referring thiz
- service->removeResource(mClientInfo, false /*checkValid*/);
-}
-
-class OverrideProcessInfoDeathNotifier : public DeathNotifier {
-public:
- OverrideProcessInfoDeathNotifier(const std::shared_ptr<IResourceManagerClient>& client,
- const std::shared_ptr<ResourceManagerService>& service,
- const ClientInfoParcel& clientInfo)
- : DeathNotifier(client, service, clientInfo) {}
-
- virtual ~OverrideProcessInfoDeathNotifier() {}
-
- virtual void binderDied();
-};
-
-void OverrideProcessInfoDeathNotifier::binderDied() {
- // Don't check for pid validity since we know it's already dead.
- std::shared_ptr<ResourceManagerService> service = mService.lock();
- if (service == nullptr) {
- ALOGW("ResourceManagerService is dead as well.");
- return;
- }
-
- service->removeProcessInfoOverride(mClientInfo.pid);
-}
-
-std::shared_ptr<DeathNotifier> DeathNotifier::Create(
- const std::shared_ptr<IResourceManagerClient>& client,
- const std::shared_ptr<ResourceManagerService>& service,
- const ClientInfoParcel& clientInfo,
- bool overrideProcessInfo) {
- std::shared_ptr<DeathNotifier> deathNotifier = nullptr;
- if (overrideProcessInfo) {
- deathNotifier = std::make_shared<OverrideProcessInfoDeathNotifier>(
- client, service, clientInfo);
- } else {
- deathNotifier = std::make_shared<DeathNotifier>(client, service, clientInfo);
- }
-
- if (deathNotifier) {
- deathNotifier->link();
- }
-
- return deathNotifier;
-}
-
static void notifyResourceGranted(int pid, const std::vector<MediaResourceParcel>& resources) {
static const char* const kServiceName = "media_resource_monitor";
sp<IBinder> binder = defaultServiceManager()->checkService(String16(kServiceName));
@@ -334,8 +184,7 @@
//static
void ResourceManagerService::instantiate() {
- std::shared_ptr<ResourceManagerService> service =
- ::ndk::SharedRefBase::make<ResourceManagerService>();
+ std::shared_ptr<ResourceManagerService> service = Create();
binder_status_t status =
AServiceManager_addServiceWithFlags(
service->asBinder().get(), getServiceName(),
@@ -356,6 +205,16 @@
//ABinderProcess_startThreadPool();
}
+std::shared_ptr<ResourceManagerService> ResourceManagerService::Create() {
+ return Create(new ProcessInfo(), new SystemCallbackImpl());
+}
+
+std::shared_ptr<ResourceManagerService> ResourceManagerService::Create(
+ const sp<ProcessInfoInterface>& processInfo,
+ const sp<SystemCallbackInterface>& systemResource) {
+ return ::ndk::SharedRefBase::make<ResourceManagerService>(processInfo, systemResource);
+}
+
ResourceManagerService::~ResourceManagerService() {}
void ResourceManagerService::setObserverService(
@@ -380,8 +239,7 @@
return Status::ok();
}
-void ResourceManagerService::onFirstAdded(const MediaResourceParcel& resource,
- const ResourceInfo& clientInfo) {
+void ResourceManagerService::onFirstAdded(const MediaResourceParcel& resource, uid_t uid) {
// first time added
if (resource.type == MediaResource::Type::kCpuBoost
&& resource.subType == MediaResource::SubType::kUnspecifiedSubType) {
@@ -395,12 +253,11 @@
} else if (resource.type == MediaResource::Type::kBattery
&& (resource.subType == MediaResource::SubType::kHwVideoCodec
|| resource.subType == MediaResource::SubType::kSwVideoCodec)) {
- mSystemCB->noteStartVideo(clientInfo.uid);
+ mSystemCB->noteStartVideo(uid);
}
}
-void ResourceManagerService::onLastRemoved(const MediaResourceParcel& resource,
- const ResourceInfo& clientInfo) {
+void ResourceManagerService::onLastRemoved(const MediaResourceParcel& resource, uid_t uid) {
if (resource.type == MediaResource::Type::kCpuBoost
&& resource.subType == MediaResource::SubType::kUnspecifiedSubType
&& mCpuBoostCount > 0) {
@@ -410,24 +267,7 @@
} else if (resource.type == MediaResource::Type::kBattery
&& (resource.subType == MediaResource::SubType::kHwVideoCodec
|| resource.subType == MediaResource::SubType::kSwVideoCodec)) {
- mSystemCB->noteStopVideo(clientInfo.uid);
- }
-}
-
-void ResourceManagerService::mergeResources(MediaResourceParcel& r1,
- const MediaResourceParcel& r2) {
- // The resource entry on record is maintained to be in [0,INT64_MAX].
- // Clamp if merging in the new resource value causes it to go out of bound.
- // Note that the new resource value could be negative, eg.DrmSession, the
- // value goes lower when the session is used more often. During reclaim
- // the session with the highest value (lowest usage) would be closed.
- if (r2.value < INT64_MAX - r1.value) {
- r1.value += r2.value;
- if (r1.value < 0) {
- r1.value = 0;
- }
- } else {
- r1.value = INT64_MAX;
+ mSystemCB->noteStopVideo(uid);
}
}
@@ -469,7 +309,7 @@
ALOGW("Ignoring request to add new resource entry with value <= 0");
continue;
}
- onFirstAdded(res, info);
+ onFirstAdded(res, info.uid);
info.resources[resType] = res;
} else {
mergeResources(info.resources[resType], res);
@@ -540,7 +380,7 @@
if (resource.value > res.value) {
resource.value -= res.value;
} else {
- onLastRemoved(res, info);
+ onLastRemoved(res, info.uid);
actualRemoved.value = resource.value;
info.resources.erase(resType);
}
@@ -595,7 +435,7 @@
const ResourceInfo& info = foundClient->second;
for (auto it = info.resources.begin(); it != info.resources.end(); it++) {
- onLastRemoved(it->second, info);
+ onLastRemoved(it->second, info.uid);
}
// Since this client has been removed, update the metrics collector.
@@ -620,15 +460,13 @@
// Before looking into other processes, check if we have clients marked for
// pending removal in the same process.
- uid_t uid = 0;
- std::shared_ptr<IResourceManagerClient> client;
- if (getBiggestClientPendingRemoval_l(callingPid, res->type, res->subType, uid, &client)) {
- clientsInfo.emplace_back(callingPid, uid, client);
+ ClientInfo clientInfo;
+ if (getBiggestClientPendingRemoval_l(callingPid, res->type, res->subType, clientInfo)) {
+ clientsInfo.emplace_back(clientInfo);
return;
}
// Now find client(s) from a lowest priority process that has needed resources.
- ClientInfo clientInfo;
if (getLowestPriorityBiggestClient_l(resourceRequestInfo, clientInfo)) {
clientsInfo.push_back(clientInfo);
}
@@ -769,49 +607,75 @@
mResourceManagerMetrics->pushReclaimAtom(clientInfo, priorities, targetClients, reclaimed);
}
+std::shared_ptr<IResourceManagerClient> ResourceManagerService::getClient(
+ int pid, const int64_t& clientId) const {
+ std::map<int, ResourceInfos>::const_iterator found = mMap.find(pid);
+ if (found == mMap.end()) {
+ ALOGV("%s: didn't find pid %d for clientId %lld", __func__, pid, (long long) clientId);
+ return nullptr;
+ }
+
+ const ResourceInfos& infos = found->second;
+ ResourceInfos::const_iterator foundClient = infos.find(clientId);
+ if (foundClient == infos.end()) {
+ ALOGV("%s: didn't find clientId %lld", __func__, (long long) clientId);
+ return nullptr;
+ }
+
+ return foundClient->second.client;
+}
+
+bool ResourceManagerService::removeClient(int pid, const int64_t& clientId) {
+ std::map<int, ResourceInfos>::iterator found = mMap.find(pid);
+ if (found == mMap.end()) {
+ ALOGV("%s: didn't find pid %d for clientId %lld", __func__, pid, (long long) clientId);
+ return false;
+ }
+
+ ResourceInfos& infos = found->second;
+ ResourceInfos::iterator foundClient = infos.find(clientId);
+ if (foundClient == infos.end()) {
+ ALOGV("%s: didn't find clientId %lld", __func__, (long long) clientId);
+ return false;
+ }
+
+ infos.erase(foundClient);
+ return true;
+}
+
bool ResourceManagerService::reclaimUnconditionallyFrom(
const std::vector<ClientInfo>& targetClients) {
if (targetClients.size() == 0) {
return false;
}
- std::shared_ptr<IResourceManagerClient> failedClient;
+ int64_t failedClientId = -1;
+ int32_t failedClientPid = -1;
for (const ClientInfo& targetClient : targetClients) {
- if (targetClient.mClient == nullptr) {
+ std::shared_ptr<IResourceManagerClient> client = getClient(
+ targetClient.mPid, targetClient.mClientId);
+ if (client == nullptr) {
// skip already released clients.
continue;
}
- String8 log = String8::format("reclaimResource from client %p", targetClient.mClient.get());
+ String8 log = String8::format("reclaimResource from client %p", client.get());
mServiceLog->add(log);
bool success;
- Status status = targetClient.mClient->reclaimResource(&success);
+ Status status = client->reclaimResource(&success);
if (!status.isOk() || !success) {
- failedClient = targetClient.mClient;
+ failedClientId = targetClient.mClientId;
+ failedClientPid = targetClient.mPid;
break;
}
}
- if (failedClient == NULL) {
+ if (failedClientId == -1) {
return true;
}
- int failedClientPid = -1;
{
std::scoped_lock lock{mLock};
- bool found = false;
- for (auto& [pid, infos] : mMap) {
- for (const auto& [id, info] : infos) {
- if (info.client == failedClient) {
- infos.erase(id);
- found = true;
- break;
- }
- }
- if (found) {
- failedClientPid = pid;
- break;
- }
- }
+ bool found = removeClient(failedClientPid, failedClientId);
if (found) {
ALOGW("Failed to reclaim resources from client with pid %d", failedClientPid);
} else {
@@ -968,21 +832,19 @@
MediaResource::SubType::kSwVideoCodec,
MediaResource::SubType::kHwImageCodec,
MediaResource::SubType::kSwImageCodec}) {
- std::shared_ptr<IResourceManagerClient> client;
- uid_t uid = 0;
- if (getBiggestClientPendingRemoval_l(pid, type, subType, uid, &client)) {
- targetClients.emplace_back(pid, uid, client);
+ ClientInfo clientInfo;
+ if (getBiggestClientPendingRemoval_l(pid, type, subType, clientInfo)) {
+ targetClients.emplace_back(clientInfo);
continue;
}
}
break;
// Non-codec resources are shared by audio, video and image codecs (no subtype).
default:
- std::shared_ptr<IResourceManagerClient> client;
- uid_t uid = 0;
+ ClientInfo clientInfo;
if (getBiggestClientPendingRemoval_l(pid, type,
- MediaResource::SubType::kUnspecifiedSubType, uid, &client)) {
- targetClients.emplace_back(pid, uid, client);
+ MediaResource::SubType::kUnspecifiedSubType, clientInfo)) {
+ targetClients.emplace_back(clientInfo);
}
break;
}
@@ -1024,7 +886,7 @@
clientsInfo.clear();
return false;
}
- clientsInfo.emplace_back(pid, info.uid, info.client);
+ clientsInfo.emplace_back(pid, info.uid, info.clientId);
}
}
}
@@ -1039,15 +901,13 @@
// - Find the bigegst client (with required resources) from that process.
bool ResourceManagerService::getLowestPriorityBiggestClient_l(
const ResourceRequestInfo& resourceRequestInfo,
- ClientInfo& clientsInfo) {
+ ClientInfo& clientInfo) {
int callingPid = resourceRequestInfo.mCallingPid;
MediaResource::Type type = resourceRequestInfo.mResource->type;
MediaResource::SubType subType = resourceRequestInfo.mResource->subType;
int lowestPriorityPid;
int lowestPriority;
int callingPriority;
- uid_t uid = 0;
- std::shared_ptr<IResourceManagerClient> client;
if (!getPriority_l(callingPid, &callingPriority)) {
ALOGE("%s: can't get process priority for pid %d", __func__, callingPid);
@@ -1062,13 +922,10 @@
return false;
}
- if (!getBiggestClient_l(lowestPriorityPid, type, subType, uid, &client)) {
+ if (!getBiggestClient_l(lowestPriorityPid, type, subType, clientInfo)) {
return false;
}
- clientsInfo.mPid = lowestPriorityPid;
- clientsInfo.mUid = uid;
- clientsInfo.mClient = client;
ALOGI("%s: CallingProcess(%d:%d) will reclaim from the lowestPriorityProcess(%d:%d)",
__func__, callingPid, callingPriority, lowestPriorityPid, lowestPriority);
return true;
@@ -1121,15 +978,12 @@
}
bool ResourceManagerService::getBiggestClientPendingRemoval_l(int pid, MediaResource::Type type,
- MediaResource::SubType subType, uid_t& uid,
- std::shared_ptr<IResourceManagerClient> *client) {
- return getBiggestClient_l(pid, type, subType, uid, client, true /* pendingRemovalOnly */);
+ MediaResource::SubType subType, ClientInfo& clientInfo) {
+ return getBiggestClient_l(pid, type, subType, clientInfo, true /* pendingRemovalOnly */);
}
bool ResourceManagerService::getBiggestClient_l(int pid, MediaResource::Type type,
- MediaResource::SubType subType, uid_t& uid,
- std::shared_ptr<IResourceManagerClient> *client,
- bool pendingRemovalOnly) {
+ MediaResource::SubType subType, ClientInfo& clientInfo, bool pendingRemovalOnly) {
PidResourceInfosMap::iterator found = mMap.find(pid);
if (found == mMap.end()) {
ALOGE_IF(!pendingRemovalOnly,
@@ -1137,7 +991,8 @@
return false;
}
- std::shared_ptr<IResourceManagerClient> clientTemp;
+ uid_t uid = -1;
+ int64_t clientId = -1;
uint64_t largestValue = 0;
const ResourceInfos& infos = found->second;
for (const auto& [id, info] : infos) {
@@ -1150,21 +1005,23 @@
if (hasResourceType(type, subType, resource)) {
if (resource.value > largestValue) {
largestValue = resource.value;
- clientTemp = info.client;
+ clientId = info.clientId;
uid = info.uid;
}
}
}
}
- if (clientTemp == NULL) {
+ if (clientId == -1) {
ALOGE_IF(!pendingRemovalOnly,
"getBiggestClient_l: can't find resource type %s and subtype %s for pid %d",
asString(type), asString(subType), pid);
return false;
}
- *client = clientTemp;
+ clientInfo.mPid = pid;
+ clientInfo.mUid = uid;
+ clientInfo.mClientId = clientId;
return true;
}
diff --git a/services/mediaresourcemanager/ResourceManagerService.h b/services/mediaresourcemanager/ResourceManagerService.h
index 27ba004..e22a6b3 100644
--- a/services/mediaresourcemanager/ResourceManagerService.h
+++ b/services/mediaresourcemanager/ResourceManagerService.h
@@ -30,9 +30,10 @@
#include <utils/String8.h>
#include <utils/threads.h>
+#include "ResourceManagerServiceUtils.h"
+
namespace android {
-class DeathNotifier;
class ResourceObserverService;
class ServiceLog;
struct ProcessInfoInterface;
@@ -46,57 +47,6 @@
using ::aidl::android::media::ClientInfoParcel;
using ::aidl::android::media::ClientConfigParcel;
-typedef std::map<std::tuple<
- MediaResource::Type, MediaResource::SubType, std::vector<uint8_t>>,
- MediaResourceParcel> ResourceList;
-
-struct ResourceInfo {
- uid_t uid;
- int64_t clientId;
- std::string name;
- std::shared_ptr<IResourceManagerClient> client;
- std::shared_ptr<DeathNotifier> deathNotifier = nullptr;
- ResourceList resources;
- bool pendingRemoval{false};
-};
-
-/*
- * Resource request info that encapsulates
- * - the calling/requesting process pid.
- * - the resource requesting (to be reclaimed from others)
- */
-struct ResourceRequestInfo {
- // uid of the calling/requesting process.
- int mCallingPid = -1;
- // resources requested.
- const ::aidl::android::media::MediaResourceParcel* mResource;
-};
-
-/*
- * Structure that defines the Client - a possible target to relcaim from.
- * This encapsulates pid, uid of the process and the client.
- * based on the reclaim policy.
- */
-struct ClientInfo {
- // pid of the process.
- pid_t mPid;
- // uid of the process.
- uid_t mUid;
- // Client to relcaim from.
- std::shared_ptr<::aidl::android::media::IResourceManagerClient> mClient;
- ClientInfo(
- pid_t pid = -1,
- uid_t uid = -1,
- const std::shared_ptr<::aidl::android::media::IResourceManagerClient>& client = nullptr)
- : mPid(pid),
- mUid(uid),
- mClient(client) {
- }
-};
-
-typedef std::map<int64_t, ResourceInfo> ResourceInfos;
-typedef std::map<int, ResourceInfos> PidResourceInfosMap;
-
class ResourceManagerService : public BnResourceManagerService {
public:
struct SystemCallbackInterface : public RefBase {
@@ -109,13 +59,20 @@
static char const *getServiceName() { return "media.resource_manager"; }
static void instantiate();
- virtual inline binder_status_t dump(
+ // Static creation methods.
+ static std::shared_ptr<ResourceManagerService> Create();
+ static std::shared_ptr<ResourceManagerService> Create(
+ const sp<ProcessInfoInterface>& processInfo,
+ const sp<SystemCallbackInterface>& systemResource);
+
+ virtual binder_status_t dump(
int /*fd*/, const char** /*args*/, uint32_t /*numArgs*/);
ResourceManagerService();
explicit ResourceManagerService(const sp<ProcessInfoInterface> &processInfo,
const sp<SystemCallbackInterface> &systemResource);
virtual ~ResourceManagerService();
+
void setObserverService(const std::shared_ptr<ResourceObserverService>& observerService);
// IResourceManagerService interface
@@ -158,6 +115,7 @@
private:
friend class ResourceManagerServiceTest;
+ friend class ResourceManagerServiceTestBase;
friend class DeathNotifier;
friend class OverrideProcessInfoDeathNotifier;
@@ -183,12 +141,12 @@
// Returns false with no change to client if there are no clients holding resources of this
// type.
bool getBiggestClient_l(int pid, MediaResource::Type type, MediaResource::SubType subType,
- uid_t& uid, std::shared_ptr<IResourceManagerClient> *client,
+ ClientInfo& clientsInfo,
bool pendingRemovalOnly = false);
// Same method as above, but with pendingRemovalOnly as true.
bool getBiggestClientPendingRemoval_l(int pid, MediaResource::Type type,
- MediaResource::SubType subType, uid_t& uid,
- std::shared_ptr<IResourceManagerClient>* client);
+ MediaResource::SubType subType,
+ ClientInfo& clientsInfo);
// A helper function that returns true if the callingPid has higher priority than pid.
// Returns false otherwise.
@@ -199,11 +157,8 @@
void getClientForResource_l(const ResourceRequestInfo& resourceRequestInfo,
std::vector<ClientInfo>& clientsInfo);
- void onFirstAdded(const MediaResourceParcel& res, const ResourceInfo& clientInfo);
- void onLastRemoved(const MediaResourceParcel& res, const ResourceInfo& clientInfo);
-
- // Merge r2 into r1
- void mergeResources(MediaResourceParcel& r1, const MediaResourceParcel& r2);
+ void onFirstAdded(const MediaResourceParcel& res, uid_t uid);
+ void onLastRemoved(const MediaResourceParcel& res, uid_t uid);
// Get priority from process's pid
bool getPriority_l(int pid, int* priority);
@@ -216,6 +171,12 @@
const std::vector<ClientInfo>& targetClients,
bool reclaimed);
+ // Get the client for given pid and the clientId from the map
+ std::shared_ptr<IResourceManagerClient> getClient(int pid, const int64_t& clientId) const;
+
+ // Remove the client for given pid and the clientId from the map
+ bool removeClient(int pid, const int64_t& clientId);
+
// The following utility functions are used only for testing by ResourceManagerServiceTest
// Gets lowest priority process that has the specified resource type.
// Returns false if failed. The output parameters will remain unchanged if failed.
diff --git a/services/mediaresourcemanager/ResourceManagerServiceUtils.cpp b/services/mediaresourcemanager/ResourceManagerServiceUtils.cpp
index 892b1b3..de682f8 100644
--- a/services/mediaresourcemanager/ResourceManagerServiceUtils.cpp
+++ b/services/mediaresourcemanager/ResourceManagerServiceUtils.cpp
@@ -24,29 +24,32 @@
namespace android {
+// Bunch of utility functions that looks for a specific Resource.
+// Check whether a given resource (of type and subtype) is found in given resource parcel.
bool hasResourceType(MediaResource::Type type, MediaResource::SubType subType,
- const MediaResourceParcel& resource) {
+ const MediaResourceParcel& resource) {
if (type != resource.type) {
return false;
}
switch (type) {
- // Codec subtypes (e.g. video vs. audio) are each considered separate resources, so
- // compare the subtypes as well.
- case MediaResource::Type::kSecureCodec:
- case MediaResource::Type::kNonSecureCodec:
- if (resource.subType == subType) {
- return true;
- }
- break;
- // Non-codec resources are not segregated by the subtype (e.g. video vs. audio).
- default:
+ // Codec subtypes (e.g. video vs. audio and hw vs. sw) are each considered separate resources,
+ // so compare the subtypes as well.
+ case MediaResource::Type::kSecureCodec:
+ case MediaResource::Type::kNonSecureCodec:
+ if (resource.subType == subType) {
return true;
+ }
+ break;
+ // Non-codec resources are not segregated by the subtype (e.g. video vs. audio).
+ default:
+ return true;
}
return false;
}
+// Check whether a given resource (of type and subtype) is found in given resource list.
bool hasResourceType(MediaResource::Type type, MediaResource::SubType subType,
- const ResourceList& resources) {
+ const ResourceList& resources) {
for (auto it = resources.begin(); it != resources.end(); it++) {
if (hasResourceType(type, subType, it->second)) {
return true;
@@ -55,8 +58,9 @@
return false;
}
+// Check whether a given resource (of type and subtype) is found in given resource info list.
bool hasResourceType(MediaResource::Type type, MediaResource::SubType subType,
- const ResourceInfos& infos) {
+ const ResourceInfos& infos) {
for (const auto& [id, info] : infos) {
if (hasResourceType(type, subType, info.resources)) {
return true;
@@ -77,12 +81,17 @@
return found->second;
}
+// Return modifiable ResourceInfo for a given client (look up by client id)
+// from the map of ResourceInfos.
+// If the item is not in the map, create one and add it to the map.
ResourceInfo& getResourceInfoForEdit(const ClientInfoParcel& clientInfo,
- const std::shared_ptr<IResourceManagerClient>& client, ResourceInfos& infos) {
+ const std::shared_ptr<IResourceManagerClient>& client,
+ ResourceInfos& infos) {
ResourceInfos::iterator found = infos.find(clientInfo.id);
if (found == infos.end()) {
- ResourceInfo info{.uid = static_cast<uid_t>(clientInfo.uid),
+ ResourceInfo info{.pid = clientInfo.pid,
+ .uid = static_cast<uid_t>(clientInfo.uid),
.clientId = clientInfo.id,
.name = clientInfo.name.empty()? "<unknown client>" : clientInfo.name,
.client = client,
@@ -95,4 +104,102 @@
return found->second;
}
+// Merge resources from r2 into r1.
+void mergeResources(MediaResourceParcel& r1, const MediaResourceParcel& r2) {
+ // The resource entry on record is maintained to be in [0,INT64_MAX].
+ // Clamp if merging in the new resource value causes it to go out of bound.
+ // Note that the new resource value could be negative, eg.DrmSession, the
+ // value goes lower when the session is used more often. During reclaim
+ // the session with the highest value (lowest usage) would be closed.
+ if (r2.value < INT64_MAX - r1.value) {
+ r1.value += r2.value;
+ if (r1.value < 0) {
+ r1.value = 0;
+ }
+ } else {
+ r1.value = INT64_MAX;
+ }
+}
+
+///////////////////////////////////////////////////////////////////////
+////////////// Death Notifier implementation ////////////////////////
+///////////////////////////////////////////////////////////////////////
+
+DeathNotifier::DeathNotifier(const std::shared_ptr<IResourceManagerClient>& client,
+ const std::weak_ptr<ResourceManagerService>& service,
+ const ClientInfoParcel& clientInfo)
+ : mClient(client), mService(service), mClientInfo(clientInfo),
+ mCookie(nullptr),
+ mDeathRecipient(::ndk::ScopedAIBinder_DeathRecipient(
+ AIBinder_DeathRecipient_new(BinderDiedCallback))) {
+ // Setting callback notification when DeathRecipient gets deleted.
+ AIBinder_DeathRecipient_setOnUnlinked(mDeathRecipient.get(), BinderUnlinkedCallback);
+}
+
+//static
+void DeathNotifier::BinderUnlinkedCallback(void* cookie) {
+ BinderDiedContext* context = reinterpret_cast<BinderDiedContext*>(cookie);
+ // Since we don't need the context anymore, we are deleting it now.
+ delete context;
+}
+
+//static
+void DeathNotifier::BinderDiedCallback(void* cookie) {
+ BinderDiedContext* context = reinterpret_cast<BinderDiedContext*>(cookie);
+
+ // Validate the context and check if the DeathNotifier object is still in scope.
+ if (context != nullptr) {
+ std::shared_ptr<DeathNotifier> thiz = context->mDeathNotifier.lock();
+ if (thiz != nullptr) {
+ thiz->binderDied();
+ } else {
+ ALOGI("DeathNotifier is out of scope already");
+ }
+ }
+}
+
+void DeathNotifier::binderDied() {
+ // Don't check for pid validity since we know it's already dead.
+ std::shared_ptr<ResourceManagerService> service = mService.lock();
+ if (service == nullptr) {
+ ALOGW("ResourceManagerService is dead as well.");
+ return;
+ }
+
+ service->overridePid(mClientInfo.pid, -1);
+ // thiz is freed in the call below, so it must be last call referring thiz
+ service->removeResource(mClientInfo, false /*checkValid*/);
+}
+
+void OverrideProcessInfoDeathNotifier::binderDied() {
+ // Don't check for pid validity since we know it's already dead.
+ std::shared_ptr<ResourceManagerService> service = mService.lock();
+ if (service == nullptr) {
+ ALOGW("ResourceManagerService is dead as well.");
+ return;
+ }
+
+ service->removeProcessInfoOverride(mClientInfo.pid);
+}
+
+std::shared_ptr<DeathNotifier> DeathNotifier::Create(
+ const std::shared_ptr<IResourceManagerClient>& client,
+ const std::weak_ptr<ResourceManagerService>& service,
+ const ClientInfoParcel& clientInfo,
+ bool overrideProcessInfo) {
+ std::shared_ptr<DeathNotifier> deathNotifier = nullptr;
+ if (overrideProcessInfo) {
+ deathNotifier = std::make_shared<OverrideProcessInfoDeathNotifier>(
+ client, service, clientInfo);
+ } else {
+ deathNotifier = std::make_shared<DeathNotifier>(client, service, clientInfo);
+ }
+
+ if (deathNotifier) {
+ deathNotifier->link();
+ }
+
+ return deathNotifier;
+}
+
} // namespace android
diff --git a/services/mediaresourcemanager/ResourceManagerServiceUtils.h b/services/mediaresourcemanager/ResourceManagerServiceUtils.h
index bbc26de..ac1e410 100644
--- a/services/mediaresourcemanager/ResourceManagerServiceUtils.h
+++ b/services/mediaresourcemanager/ResourceManagerServiceUtils.h
@@ -19,10 +19,143 @@
#define ANDROID_MEDIA_RESOURCEMANAGERSERVICEUTILS_H_
#include <vector>
+
+#include <aidl/android/media/BnResourceManagerService.h>
+#include <media/MediaResource.h>
#include <utils/String8.h>
namespace android {
+class ResourceManagerService;
+
+/*
+ * Death Notifier to track IResourceManagerClient's death.
+ */
+class DeathNotifier : public std::enable_shared_from_this<DeathNotifier> {
+
+ // BinderDiedContext defines the cookie that is passed as DeathRecipient.
+ // Since this can maintain more context than a raw pointer, we can
+ // validate the scope of DeathNotifier, before deferencing it upon the binder death.
+ struct BinderDiedContext {
+ std::weak_ptr<DeathNotifier> mDeathNotifier;
+ };
+public:
+ static std::shared_ptr<DeathNotifier> Create(
+ const std::shared_ptr<::aidl::android::media::IResourceManagerClient>& client,
+ const std::weak_ptr<ResourceManagerService>& service,
+ const ::aidl::android::media::ClientInfoParcel& clientInfo,
+ bool overrideProcessInfo = false);
+
+ DeathNotifier(const std::shared_ptr<::aidl::android::media::IResourceManagerClient>& client,
+ const std::weak_ptr<ResourceManagerService>& service,
+ const ::aidl::android::media::ClientInfoParcel& clientInfo);
+
+ virtual ~DeathNotifier() {
+ unlink();
+ }
+
+ // Implement death recipient
+ static void BinderDiedCallback(void* cookie);
+ static void BinderUnlinkedCallback(void* cookie);
+ virtual void binderDied();
+
+private:
+ void link() {
+ // Create the context that is passed as cookie to the binder death notification.
+ // The context gets deleted at BinderUnlinkedCallback.
+ mCookie = new BinderDiedContext{.mDeathNotifier = weak_from_this()};
+ // Register for the callbacks by linking to death notification.
+ AIBinder_linkToDeath(mClient->asBinder().get(), mDeathRecipient.get(), mCookie);
+ }
+
+ void unlink() {
+ if (mClient != nullptr) {
+ // Unlink from the death notification.
+ AIBinder_unlinkToDeath(mClient->asBinder().get(), mDeathRecipient.get(), mCookie);
+ mClient = nullptr;
+ }
+ }
+
+protected:
+ std::shared_ptr<::aidl::android::media::IResourceManagerClient> mClient;
+ std::weak_ptr<ResourceManagerService> mService;
+ const ::aidl::android::media::ClientInfoParcel mClientInfo;
+ BinderDiedContext* mCookie;
+ ::ndk::ScopedAIBinder_DeathRecipient mDeathRecipient;
+};
+
+class OverrideProcessInfoDeathNotifier : public DeathNotifier {
+public:
+ OverrideProcessInfoDeathNotifier(
+ const std::shared_ptr<::aidl::android::media::IResourceManagerClient>& client,
+ const std::weak_ptr<ResourceManagerService>& service,
+ const ::aidl::android::media::ClientInfoParcel& clientInfo)
+ : DeathNotifier(client, service, clientInfo) {}
+
+ virtual ~OverrideProcessInfoDeathNotifier() {}
+
+ virtual void binderDied();
+};
+
+// A map of tuple(type, sub-type, id) and the resource parcel.
+typedef std::map<std::tuple<
+ MediaResource::Type, MediaResource::SubType, std::vector<uint8_t>>,
+ ::aidl::android::media::MediaResourceParcel> ResourceList;
+
+// Encapsulation for Resource Info, that contains
+// - pid of the app
+// - uid of the app
+// - client id
+// - name of the client (specifically for the codec)
+// - the client associted with it
+// - death notifier for the (above) client
+// - list of resources associated with it
+// - A flag that marks whether this resource is pending to be removed.
+struct ResourceInfo {
+ pid_t pid;
+ uid_t uid;
+ int64_t clientId;
+ std::string name;
+ std::shared_ptr<::aidl::android::media::IResourceManagerClient> client;
+ std::shared_ptr<DeathNotifier> deathNotifier = nullptr;
+ ResourceList resources;
+ bool pendingRemoval{false};
+};
+
+/*
+ * Resource request info that encapsulates
+ * - the calling/requesting process pid.
+ * - the resource requesting (to be reclaimed from others)
+ */
+struct ResourceRequestInfo {
+ // pid of the calling/requesting process.
+ int mCallingPid = -1;
+ // resources requested.
+ const ::aidl::android::media::MediaResourceParcel* mResource;
+};
+
+/*
+ * Structure that defines the Client - a possible target to relcaim from.
+ * This encapsulates pid, uid of the process and the client id
+ * based on the reclaim policy.
+ */
+struct ClientInfo {
+ // pid of the process.
+ pid_t mPid = -1;
+ // uid of the process.
+ uid_t mUid = -1;
+ // Client Id.
+ int64_t mClientId = -1;
+ ClientInfo(pid_t pid = -1, uid_t uid = -1, const int64_t& clientId = -1)
+ : mPid(pid), mUid(uid), mClientId(clientId) {}
+};
+
+// Map of Resource information index through the client id.
+typedef std::map<int64_t, ResourceInfo> ResourceInfos;
+
+// Map of Resource information indexed through the process id.
+typedef std::map<int, ResourceInfos> PidResourceInfosMap;
+
// templated function to stringify the given vector of items.
template <typename T>
String8 getString(const std::vector<T>& items) {
@@ -37,15 +170,15 @@
//Check whether a given resource (of type and subtype) is found in given resource parcel.
bool hasResourceType(MediaResource::Type type, MediaResource::SubType subType,
- const MediaResourceParcel& resource);
+ const MediaResourceParcel& resource);
//Check whether a given resource (of type and subtype) is found in given resource list.
bool hasResourceType(MediaResource::Type type, MediaResource::SubType subType,
- const ResourceList& resources);
+ const ResourceList& resources);
//Check whether a given resource (of type and subtype) is found in given resource info list.
bool hasResourceType(MediaResource::Type type, MediaResource::SubType subType,
- const ResourceInfos& infos);
+ const ResourceInfos& infos);
// Return modifiable list of ResourceInfo for a given process (look up by pid)
// from the map of ResourceInfos.
@@ -54,8 +187,13 @@
// Return modifiable ResourceInfo for a given process (look up by pid)
// from the map of ResourceInfos.
// If the item is not in the map, create one and add it to the map.
-ResourceInfo& getResourceInfoForEdit(const ClientInfoParcel& clientInfo,
- const std::shared_ptr<IResourceManagerClient>& client, ResourceInfos& infos);
+ResourceInfo& getResourceInfoForEdit(
+ const aidl::android::media::ClientInfoParcel& clientInfo,
+ const std::shared_ptr<aidl::android::media::IResourceManagerClient>& client,
+ ResourceInfos& infos);
+
+// Merge resources from r2 into r1.
+void mergeResources(MediaResourceParcel& r1, const MediaResourceParcel& r2);
} // namespace android
diff --git a/services/mediaresourcemanager/fuzzer/mediaresourcemanager_fuzzer.cpp b/services/mediaresourcemanager/fuzzer/mediaresourcemanager_fuzzer.cpp
index 6fa9831..643a4e5 100644
--- a/services/mediaresourcemanager/fuzzer/mediaresourcemanager_fuzzer.cpp
+++ b/services/mediaresourcemanager/fuzzer/mediaresourcemanager_fuzzer.cpp
@@ -208,9 +208,9 @@
return nullptr;
}
- shared_ptr<ResourceManagerService> mService =
- ::ndk::SharedRefBase::make<ResourceManagerService>(new TestProcessInfo(),
- new TestSystemCallback());
+ shared_ptr<ResourceManagerService> mService = ResourceManagerService::Create(
+ new TestProcessInfo(),
+ new TestSystemCallback());
FuzzedDataProvider* mFuzzedDataProvider = nullptr;
};
diff --git a/services/mediaresourcemanager/fuzzer/resourcemanager_service_fuzzer.cpp b/services/mediaresourcemanager/fuzzer/resourcemanager_service_fuzzer.cpp
index ca10d20..6253df7 100644
--- a/services/mediaresourcemanager/fuzzer/resourcemanager_service_fuzzer.cpp
+++ b/services/mediaresourcemanager/fuzzer/resourcemanager_service_fuzzer.cpp
@@ -26,7 +26,7 @@
using ndk::SharedRefBase;
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
- auto service = SharedRefBase::make<ResourceManagerService>();
+ std::shared_ptr<ResourceManagerService> service = ResourceManagerService::Create();
fuzzService(service->asBinder().get(), FuzzedDataProvider(data, size));
return 0;
}
diff --git a/services/mediaresourcemanager/test/ResourceManagerServiceTestUtils.h b/services/mediaresourcemanager/test/ResourceManagerServiceTestUtils.h
index 474ff0f..52d82b8 100644
--- a/services/mediaresourcemanager/test/ResourceManagerServiceTestUtils.h
+++ b/services/mediaresourcemanager/test/ResourceManagerServiceTestUtils.h
@@ -207,8 +207,7 @@
// silently ignored.
ABinderProcess_startThreadPool();
mSystemCB = new TestSystemCallback();
- mService = ::ndk::SharedRefBase::make<ResourceManagerService>(
- new TestProcessInfo, mSystemCB);
+ mService = ResourceManagerService::Create(new TestProcessInfo, mSystemCB);
mTestClient1 = ::ndk::SharedRefBase::make<TestClient>(kTestPid1, kTestUid1, mService);
mTestClient2 = ::ndk::SharedRefBase::make<TestClient>(kTestPid2, kTestUid2, mService);
mTestClient3 = ::ndk::SharedRefBase::make<TestClient>(kTestPid2, kTestUid2, mService);
diff --git a/services/mediaresourcemanager/test/ResourceManagerService_test.cpp b/services/mediaresourcemanager/test/ResourceManagerService_test.cpp
index 7452275..8f05b13 100644
--- a/services/mediaresourcemanager/test/ResourceManagerService_test.cpp
+++ b/services/mediaresourcemanager/test/ResourceManagerService_test.cpp
@@ -495,8 +495,8 @@
EXPECT_EQ(2u, targetClients.size());
// (OK to require ordering in clients[], as the pid map is sorted)
- EXPECT_EQ(mTestClient3, targetClients[0].mClient);
- EXPECT_EQ(mTestClient1, targetClients[1].mClient);
+ EXPECT_EQ(getId(mTestClient3), targetClients[0].mClientId);
+ EXPECT_EQ(getId(mTestClient1), targetClients[1].mClientId);
}
void testReclaimResourceSecure() {
@@ -775,7 +775,7 @@
// kTestPid1 is the lowest priority process with MediaResource::Type::kGraphicMemory.
// mTestClient1 has the largest MediaResource::Type::kGraphicMemory within kTestPid1.
- EXPECT_EQ(mTestClient1, clientInfo.mClient);
+ EXPECT_EQ(getId(mTestClient1), clientInfo.mClientId);
}
void testGetLowestPriorityPid() {