Large audio frames in MediaCodec
Bug: 298052174
Change-Id: Ie091c82a102afce908d703278522cbac4cd3bb63
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp
index 6e6d3f7..7b1721e 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.cpp
+++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp
@@ -90,6 +90,28 @@
return v == "true";
}
+// Flags can come with individual BufferInfos
+// when used with large frame audio
+constexpr static std::initializer_list<std::pair<uint32_t, uint32_t>> flagList = {
+ {BUFFER_FLAG_CODEC_CONFIG, C2FrameData::FLAG_CODEC_CONFIG},
+ {BUFFER_FLAG_END_OF_STREAM, C2FrameData::FLAG_END_OF_STREAM},
+ {BUFFER_FLAG_DECODE_ONLY, C2FrameData::FLAG_DROP_FRAME}
+};
+
+static uint32_t convertFlags(uint32_t flags, bool toC2) {
+ return std::transform_reduce(
+ flagList.begin(), flagList.end(),
+ 0u,
+ std::bit_or{},
+ [flags, toC2](const std::pair<uint32_t, uint32_t> &entry) {
+ if (toC2) {
+ return (flags & entry.first) ? entry.second : 0;
+ } else {
+ return (flags & entry.second) ? entry.first : 0;
+ }
+ });
+}
+
} // namespace
CCodecBufferChannel::QueueGuard::QueueGuard(
@@ -245,7 +267,8 @@
if (buffer->meta()->findInt32("decode-only", &tmp) && tmp) {
flags |= C2FrameData::FLAG_DROP_FRAME;
}
- ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
+ ALOGV("[%s] queueInputBuffer: buffer->size() = %zu time: %lld",
+ mName, buffer->size(), (long long)timeUs);
std::list<std::unique_ptr<C2Work>> items;
std::unique_ptr<C2Work> work(new C2Work);
work->input.ordinal.timestamp = timeUs;
@@ -296,6 +319,34 @@
uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
output->rotation[frameIndex] = rotation;
}
+ sp<RefBase> obj;
+ if (buffer->meta()->findObject("accessUnitInfo", &obj)) {
+ ALOGV("Filling C2Info from multiple access units");
+ sp<WrapperObject<std::vector<AccessUnitInfo>>> infos{
+ (decltype(infos.get()))obj.get()};
+ std::vector<AccessUnitInfo> &accessUnitInfoVec = infos->value;
+ std::vector<C2AccessUnitInfosStruct> multipleAccessUnitInfos;
+ uint32_t outFlags = 0;
+ for (int i = 0; i < accessUnitInfoVec.size(); i++) {
+ outFlags = 0;
+ outFlags = convertFlags(accessUnitInfoVec[i].mFlags, true);
+ if (eos && (outFlags & C2FrameData::FLAG_END_OF_STREAM)) {
+ outFlags &= (~C2FrameData::FLAG_END_OF_STREAM);
+ }
+ multipleAccessUnitInfos.emplace_back(
+ outFlags,
+ accessUnitInfoVec[i].mSize,
+ accessUnitInfoVec[i].mTimestamp);
+ ALOGV("%d) flags: %d, size: %d, time: %llu",
+ i, outFlags, accessUnitInfoVec[i].mSize,
+ (long long)accessUnitInfoVec[i].mTimestamp);
+
+ }
+ const std::shared_ptr<C2AccessUnitInfos::input> c2AccessUnitInfos =
+ C2AccessUnitInfos::input::AllocShared(
+ multipleAccessUnitInfos.size(), 0u, multipleAccessUnitInfos);
+ c2buffer->setInfo(c2AccessUnitInfos);
+ }
work->input.buffers.push_back(c2buffer);
if (encryptedBlock) {
work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
@@ -2265,12 +2316,34 @@
case OutputBuffers::DISCARD:
break;
case OutputBuffers::NOTIFY_CLIENT:
+ {
// TRICKY: we want popped buffers reported in order, so sending
// the callback while holding the lock here. This assumes that
// onOutputBufferAvailable() does not block. onOutputBufferAvailable()
// callbacks are always sent with the Output lock held.
+ if (c2Buffer) {
+ std::shared_ptr<const C2AccessUnitInfos::output> bufferMetadata =
+ std::static_pointer_cast<const C2AccessUnitInfos::output>(
+ c2Buffer->getInfo(C2AccessUnitInfos::output::PARAM_TYPE));
+ if (bufferMetadata && bufferMetadata->flexCount() > 0) {
+ uint32_t flag = 0;
+ std::vector<AccessUnitInfo> accessUnitInfos;
+ for (int nMeta = 0; nMeta < bufferMetadata->flexCount(); nMeta++) {
+ const C2AccessUnitInfosStruct &bufferMetadataStruct =
+ bufferMetadata->m.values[nMeta];
+ flag = convertFlags(bufferMetadataStruct.flags, false);
+ accessUnitInfos.emplace_back(flag,
+ static_cast<size_t>(bufferMetadataStruct.size),
+ static_cast<size_t>(bufferMetadataStruct.timestamp));
+ }
+ sp<WrapperObject<std::vector<AccessUnitInfo>>> obj{
+ new WrapperObject<std::vector<AccessUnitInfo>>{accessUnitInfos}};
+ outBuffer->meta()->setObject("accessUnitInfo", obj);
+ }
+ }
mCallback->onOutputBufferAvailable(index, outBuffer);
break;
+ }
case OutputBuffers::REALLOCATE:
if (++reallocTryNum > kMaxReallocTry) {
output.unlock();
diff --git a/media/codec2/sfplugin/CCodecBuffers.cpp b/media/codec2/sfplugin/CCodecBuffers.cpp
index 670923b..8a48777 100644
--- a/media/codec2/sfplugin/CCodecBuffers.cpp
+++ b/media/codec2/sfplugin/CCodecBuffers.cpp
@@ -18,11 +18,14 @@
#define LOG_TAG "CCodecBuffers"
#include <utils/Log.h>
+#include <numeric>
+
#include <C2AllocatorGralloc.h>
#include <C2PlatformSupport.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/MediaDefs.h>
+#include <media/stagefright/CodecBase.h>
#include <media/stagefright/MediaCodecConstants.h>
#include <media/stagefright/SkipCutBuffer.h>
#include <mediadrm/ICrypto.h>
@@ -147,6 +150,165 @@
return copy;
}
+// MultiAccessUnitSkipCutBuffer for buffer and bufferInfos
+
+class MultiAccessUnitSkipCutBuffer : public SkipCutBuffer {
+
+public:
+ explicit MultiAccessUnitSkipCutBuffer(
+ int32_t skip, int32_t cut, size_t num16BitChannels):
+ SkipCutBuffer(skip, cut, num16BitChannels),
+ mFrontPaddingDelay(0), mSize(0) {
+ }
+
+ virtual ~MultiAccessUnitSkipCutBuffer() {
+
+ }
+
+ void submitMultiAccessUnits(
+ const sp<MediaCodecBuffer>& buffer,
+ int32_t sampleRate, size_t num16BitChannels,
+ std::shared_ptr<const C2AccessUnitInfos::output> &infos) {
+ if (infos == nullptr) {
+ // there is nothing to do more.
+ SkipCutBuffer::submit(buffer);
+ return;
+ }
+ typedef WrapperObject<std::vector<AccessUnitInfo>> BufferInfosWrapper;
+ CHECK_EQ(mSize, SkipCutBuffer::size());
+ sp<BufferInfosWrapper> bufferInfos{new BufferInfosWrapper(decltype(bufferInfos->value)())};
+ uint32_t availableSize = buffer->size() + SkipCutBuffer::size();
+ uint32_t frontPadding = mFrontPadding;
+ int32_t lastEmptyAccessUnitIndex = -1;
+ int64_t byteInUs = 0;
+ if (sampleRate > 0 && num16BitChannels > 0) {
+ byteInUs = (1000000u / (sampleRate * num16BitChannels * 2));
+ }
+ if (frontPadding > 0) {
+ mInfos.clear();
+ mSize = 0;
+ }
+ for (int i = 0 ; i < infos->flexCount() && frontPadding > 0; i++) {
+ uint32_t flagsInPadding = 0;
+ int64_t timeInPadding = 0;
+ if (infos->m.values[i].size <= frontPadding) {
+ // we have more front padding so this buffer is not going to be used.
+ int32_t consumed = infos->m.values[i].size;
+ frontPadding -= consumed;
+ mFrontPaddingDelay += byteInUs * (consumed);
+ availableSize -= consumed;
+ flagsInPadding |= toMediaCodecFlags(infos->m.values[i].flags);
+ timeInPadding = infos->m.values[i].timestamp;
+ } else {
+ C2AccessUnitInfosStruct info = infos->m.values[i];
+ mFrontPaddingDelay += byteInUs * (frontPadding);
+ info.size -= frontPadding;
+ info.timestamp -= mFrontPaddingDelay;
+ availableSize -= frontPadding;
+ flagsInPadding |= toMediaCodecFlags(infos->m.values[i].flags);
+ timeInPadding = infos->m.values[i].timestamp;
+ frontPadding = 0;
+ mInfos.push_back(info);
+ mSize += info.size;
+ }
+ if (flagsInPadding != 0) {
+ bufferInfos->value.emplace_back(
+ flagsInPadding, 0, timeInPadding);
+ }
+ lastEmptyAccessUnitIndex = i;
+ }
+ if (frontPadding <= 0) {
+ // process what's already in the buffer first
+ auto it = mInfos.begin();
+ while (it != mInfos.end() && availableSize > mBackPadding) {
+ // we have samples to send out.
+ if ((availableSize - it->size) >= mBackPadding) {
+ // this is totally used here.
+ int32_t consumed = it->size;
+ bufferInfos->value.emplace_back(
+ toMediaCodecFlags(it->flags), consumed, it->timestamp);
+ availableSize -= consumed;
+ mSize -= consumed;
+ it = mInfos.erase(it);
+ } else {
+ int32_t consumed = availableSize - mBackPadding;
+ bufferInfos->value.emplace_back(
+ toMediaCodecFlags(it->flags),
+ consumed,
+ it->timestamp);
+ it->size -= consumed;
+ it->timestamp += consumed * byteInUs;
+ availableSize -= consumed;
+ mSize -= consumed;
+ it++;
+ }
+ }
+ // if buffer has more process all of it and keep the remaining info.
+ for (int i = (lastEmptyAccessUnitIndex + 1) ; i < infos->flexCount() ; i++) {
+ // upddate updatedInfo and mInfos
+ if (availableSize > mBackPadding) {
+ // we have to take data from the new buffer.
+ if (availableSize - infos->m.values[i].size >= mBackPadding) {
+ // we are using this info
+ int32_t consumed = infos->m.values[i].size;
+ bufferInfos->value.emplace_back(
+ toMediaCodecFlags(infos->m.values[i].flags),
+ consumed,
+ infos->m.values[i].timestamp - mFrontPaddingDelay);
+ availableSize -= consumed;
+ } else {
+ // if we need to update the size
+ C2AccessUnitInfosStruct info = infos->m.values[i];
+ int32_t consumed = availableSize - mBackPadding;
+ bufferInfos->value.emplace_back(
+ toMediaCodecFlags(infos->m.values[i].flags),
+ consumed,
+ infos->m.values[i].timestamp - mFrontPaddingDelay);
+ info.size -= consumed;
+ info.timestamp = info.timestamp - mFrontPaddingDelay +
+ consumed * byteInUs;
+ mInfos.push_back(info);
+ availableSize -= consumed;
+ mSize += info.size;
+ }
+ } else {
+ // we have to maintain infos
+ C2AccessUnitInfosStruct info = infos->m.values[i];
+ info.timestamp -= mFrontPaddingDelay;
+ mInfos.push_back(info);
+ mSize += info.size;
+ }
+ }
+ }
+ SkipCutBuffer::submit(buffer);
+ infos = nullptr;
+ if (!bufferInfos->value.empty()) {
+ buffer->meta()->setObject("accessUnitInfo", bufferInfos);
+ }
+ }
+protected:
+ // Flags can come with individual BufferInfos
+ // when used with large frame audio
+ constexpr static std::initializer_list<std::pair<uint32_t, uint32_t>> flagList = {
+ {BUFFER_FLAG_CODEC_CONFIG, C2FrameData::FLAG_CODEC_CONFIG},
+ {BUFFER_FLAG_END_OF_STREAM, C2FrameData::FLAG_END_OF_STREAM},
+ {BUFFER_FLAG_DECODE_ONLY, C2FrameData::FLAG_DROP_FRAME}
+ };
+
+ static uint32_t toMediaCodecFlags(uint32_t flags) {
+ return std::transform_reduce(
+ flagList.begin(), flagList.end(),
+ 0u,
+ std::bit_or{},
+ [flags](const std::pair<uint32_t, uint32_t> &entry) {
+ return (flags & entry.second) ? entry.first : 0;
+ });
+ }
+ std::list<C2AccessUnitInfosStruct> mInfos;
+ int64_t mFrontPaddingDelay;
+ size_t mSize;
+};
+
// OutputBuffers
OutputBuffers::OutputBuffers(const char *componentName, const char *name)
@@ -201,6 +363,15 @@
}
}
+bool OutputBuffers::submit(const sp<MediaCodecBuffer> &buffer, int32_t sampleRate,
+ int32_t channelCount, std::shared_ptr<const C2AccessUnitInfos::output> &infos) {
+ if (mSkipCutBuffer == nullptr) {
+ return false;
+ }
+ mSkipCutBuffer->submitMultiAccessUnits(buffer, sampleRate, channelCount, infos);
+ return true;
+}
+
void OutputBuffers::setSkipCutBuffer(int32_t skip, int32_t cut) {
if (mSkipCutBuffer != nullptr) {
size_t prevSize = mSkipCutBuffer->size();
@@ -208,7 +379,7 @@
ALOGD("[%s] Replacing SkipCutBuffer holding %zu bytes", mName, prevSize);
}
}
- mSkipCutBuffer = new SkipCutBuffer(skip, cut, mChannelCount);
+ mSkipCutBuffer = new MultiAccessUnitSkipCutBuffer(skip, cut, mChannelCount);
}
bool OutputBuffers::convert(
@@ -1160,7 +1331,16 @@
ALOGD("[%s] copy buffer failed", mName);
return WOULD_BLOCK;
}
- submit(c2Buffer);
+ if (buffer && buffer->hasInfo(C2AccessUnitInfos::output::PARAM_TYPE)) {
+ std::shared_ptr<const C2AccessUnitInfos::output> bufferMetadata =
+ std::static_pointer_cast<const C2AccessUnitInfos::output>(
+ buffer->getInfo(C2AccessUnitInfos::output::PARAM_TYPE));
+ if (submit(c2Buffer, mSampleRate, mChannelCount, bufferMetadata)) {
+ buffer->removeInfo(C2AccessUnitInfos::output::PARAM_TYPE);
+ }
+ } else {
+ submit(c2Buffer);
+ }
handleImageData(c2Buffer);
*clientBuffer = c2Buffer;
ALOGV("[%s] grabbed buffer %zu", mName, *index);
diff --git a/media/codec2/sfplugin/CCodecBuffers.h b/media/codec2/sfplugin/CCodecBuffers.h
index cbef644..f0936bc 100644
--- a/media/codec2/sfplugin/CCodecBuffers.h
+++ b/media/codec2/sfplugin/CCodecBuffers.h
@@ -20,6 +20,7 @@
#include <optional>
#include <string>
+#include <vector>
#include <C2Config.h>
#include <DataConverter.h>
@@ -33,6 +34,8 @@
struct ICrypto;
class MemoryDealer;
class SkipCutBuffer;
+class MultiAccessUnitSkipCutBuffer;
+struct AccessUnitInfo;
constexpr size_t kLinearBufferSize = 1048576;
// This can fit an 8K frame.
@@ -382,13 +385,17 @@
sp<MediaCodecBuffer>* outBuffer);
protected:
- sp<SkipCutBuffer> mSkipCutBuffer;
+
+ sp<MultiAccessUnitSkipCutBuffer> mSkipCutBuffer;
/**
* Update the SkipCutBuffer object. No-op if it's never initialized.
*/
void updateSkipCutBuffer(int32_t sampleRate, int32_t channelCount);
+ bool submit(const sp<MediaCodecBuffer> &buffer, int32_t sampleRate,
+ int32_t channelCount, std::shared_ptr<const C2AccessUnitInfos::output> &infos);
+
/**
* Submit buffer to SkipCutBuffer object, if initialized.
*/
diff --git a/media/codec2/sfplugin/CCodecConfig.cpp b/media/codec2/sfplugin/CCodecConfig.cpp
index 6d49fa8..c22deca 100644
--- a/media/codec2/sfplugin/CCodecConfig.cpp
+++ b/media/codec2/sfplugin/CCodecConfig.cpp
@@ -402,10 +402,19 @@
add(ConfigMapper(KEY_MAX_INPUT_SIZE, C2_PARAMKEY_INPUT_MAX_BUFFER_SIZE, "value")
.limitTo(D::INPUT));
+
// remove when codecs switch to PARAMKEY
deprecated(ConfigMapper(KEY_MAX_INPUT_SIZE, "coded.max-frame-size", "value")
.limitTo(D::INPUT));
+ // large frame params
+ add(ConfigMapper(KEY_BUFFER_BATCH_MAX_OUTPUT_SIZE,
+ C2_PARAMKEY_OUTPUT_LARGE_FRAME, "max-size")
+ .limitTo(D::AUDIO & D::OUTPUT));
+ add(ConfigMapper(KEY_BUFFER_BATCH_THRESHOLD_OUTPUT_SIZE,
+ C2_PARAMKEY_OUTPUT_LARGE_FRAME, "threshold-size")
+ .limitTo(D::AUDIO & D::OUTPUT));
+
// Rotation
// Note: SDK rotation is clock-wise, while C2 rotation is counter-clock-wise
add(ConfigMapper(KEY_ROTATION, C2_PARAMKEY_VUI_ROTATION, "value")