blob: 2dc63f3624e034f19d0fd8cb9a52f8f64c74a31c [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodecBufferChannel"
19#include <utils/Log.h>
20
Pawin Vongmasae7bb8612020-06-04 06:15:22 -070021#include <algorithm>
22#include <list>
Pawin Vongmasa36653902018-11-15 00:10:25 -080023#include <numeric>
24
25#include <C2AllocatorGralloc.h>
26#include <C2PlatformSupport.h>
27#include <C2BlockInternal.h>
28#include <C2Config.h>
29#include <C2Debug.h>
30
31#include <android/hardware/cas/native/1.0/IDescrambler.h>
Robert Shih895fba92019-07-16 16:29:44 -070032#include <android/hardware/drm/1.0/types.h>
Josh Hou8eddf4b2021-02-02 16:26:53 +080033#include <android-base/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080034#include <android-base/stringprintf.h>
Wonsik Kimfb7a7672019-12-27 17:13:33 -080035#include <binder/MemoryBase.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080036#include <binder/MemoryDealer.h>
Ray Essick18ea0452019-08-27 16:07:27 -070037#include <cutils/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080038#include <gui/Surface.h>
Robert Shih895fba92019-07-16 16:29:44 -070039#include <hidlmemory/FrameworkUtils.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080040#include <media/openmax/OMX_Core.h>
41#include <media/stagefright/foundation/ABuffer.h>
42#include <media/stagefright/foundation/ALookup.h>
43#include <media/stagefright/foundation/AMessage.h>
44#include <media/stagefright/foundation/AUtils.h>
45#include <media/stagefright/foundation/hexdump.h>
46#include <media/stagefright/MediaCodec.h>
47#include <media/stagefright/MediaCodecConstants.h>
Wonsik Kim155d5cb2019-10-09 12:49:49 -070048#include <media/stagefright/SkipCutBuffer.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080049#include <media/MediaCodecBuffer.h>
Wonsik Kim41d83432020-04-27 16:40:49 -070050#include <mediadrm/ICrypto.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080051#include <system/window.h>
52
53#include "CCodecBufferChannel.h"
54#include "Codec2Buffer.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080055
56namespace android {
57
58using android::base::StringPrintf;
59using hardware::hidl_handle;
60using hardware::hidl_string;
61using hardware::hidl_vec;
Robert Shih895fba92019-07-16 16:29:44 -070062using hardware::fromHeap;
63using hardware::HidlMemory;
64
Pawin Vongmasa36653902018-11-15 00:10:25 -080065using namespace hardware::cas::V1_0;
66using namespace hardware::cas::native::V1_0;
67
68using CasStatus = hardware::cas::V1_0::Status;
Robert Shih895fba92019-07-16 16:29:44 -070069using DrmBufferType = hardware::drm::V1_0::BufferType;
Pawin Vongmasa36653902018-11-15 00:10:25 -080070
Pawin Vongmasa36653902018-11-15 00:10:25 -080071namespace {
72
Wonsik Kim469c8342019-04-11 16:46:09 -070073constexpr size_t kSmoothnessFactor = 4;
74constexpr size_t kRenderingDepth = 3;
Pawin Vongmasa36653902018-11-15 00:10:25 -080075
Sungtak Leeab6f2f32019-02-15 14:43:51 -080076// This is for keeping IGBP's buffer dropping logic in legacy mode other
77// than making it non-blocking. Do not change this value.
78const static size_t kDequeueTimeoutNs = 0;
79
Pawin Vongmasa36653902018-11-15 00:10:25 -080080} // namespace
81
82CCodecBufferChannel::QueueGuard::QueueGuard(
83 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
84 Mutex::Autolock l(mSync.mGuardLock);
85 // At this point it's guaranteed that mSync is not under state transition,
86 // as we are holding its mutex.
87
88 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
89 if (count->value == -1) {
90 mRunning = false;
91 } else {
92 ++count->value;
93 mRunning = true;
94 }
95}
96
97CCodecBufferChannel::QueueGuard::~QueueGuard() {
98 if (mRunning) {
99 // We are not holding mGuardLock at this point so that QueueSync::stop() can
100 // keep holding the lock until mCount reaches zero.
101 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
102 --count->value;
103 count->cond.broadcast();
104 }
105}
106
107void CCodecBufferChannel::QueueSync::start() {
108 Mutex::Autolock l(mGuardLock);
109 // If stopped, it goes to running state; otherwise no-op.
110 Mutexed<Counter>::Locked count(mCount);
111 if (count->value == -1) {
112 count->value = 0;
113 }
114}
115
116void CCodecBufferChannel::QueueSync::stop() {
117 Mutex::Autolock l(mGuardLock);
118 Mutexed<Counter>::Locked count(mCount);
119 if (count->value == -1) {
120 // no-op
121 return;
122 }
123 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
124 // mCount can only decrement. In other words, threads that acquired the lock
125 // are allowed to finish execution but additional threads trying to acquire
126 // the lock at this point will block, and then get QueueGuard at STOPPED
127 // state.
128 while (count->value != 0) {
129 count.waitForCondition(count->cond);
130 }
131 count->value = -1;
132}
133
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700134// Input
135
136CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
137
Pawin Vongmasa36653902018-11-15 00:10:25 -0800138// CCodecBufferChannel
139
140CCodecBufferChannel::CCodecBufferChannel(
141 const std::shared_ptr<CCodecCallback> &callback)
142 : mHeapSeqNum(-1),
143 mCCodecCallback(callback),
144 mFrameIndex(0u),
145 mFirstValidFrameIndex(0u),
146 mMetaMode(MODE_NONE),
Sungtak Lee04b30352020-07-27 13:57:25 -0700147 mInputMetEos(false),
148 mSendEncryptedInfoBuffer(false) {
Sungtak Leed7463d12019-09-04 16:01:00 -0700149 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700150 {
151 Mutexed<Input>::Locked input(mInput);
152 input->buffers.reset(new DummyInputBuffers(""));
153 input->extraBuffers.flush();
154 input->inputDelay = 0u;
155 input->pipelineDelay = 0u;
156 input->numSlots = kSmoothnessFactor;
157 input->numExtraSlots = 0u;
158 }
159 {
160 Mutexed<Output>::Locked output(mOutput);
161 output->outputDelay = 0u;
162 output->numSlots = kSmoothnessFactor;
163 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800164}
165
166CCodecBufferChannel::~CCodecBufferChannel() {
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800167 if (mCrypto != nullptr && mHeapSeqNum >= 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800168 mCrypto->unsetHeap(mHeapSeqNum);
169 }
170}
171
172void CCodecBufferChannel::setComponent(
173 const std::shared_ptr<Codec2Client::Component> &component) {
174 mComponent = component;
175 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
176 mName = mComponentName.c_str();
177}
178
179status_t CCodecBufferChannel::setInputSurface(
180 const std::shared_ptr<InputSurfaceWrapper> &surface) {
181 ALOGV("[%s] setInputSurface", mName);
182 mInputSurface = surface;
183 return mInputSurface->connect(mComponent);
184}
185
186status_t CCodecBufferChannel::signalEndOfInputStream() {
187 if (mInputSurface == nullptr) {
188 return INVALID_OPERATION;
189 }
190 return mInputSurface->signalEndOfInputStream();
191}
192
Sungtak Lee04b30352020-07-27 13:57:25 -0700193status_t CCodecBufferChannel::queueInputBufferInternal(
194 sp<MediaCodecBuffer> buffer,
195 std::shared_ptr<C2LinearBlock> encryptedBlock,
196 size_t blockSize) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800197 int64_t timeUs;
198 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
199
200 if (mInputMetEos) {
201 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
202 return OK;
203 }
204
205 int32_t flags = 0;
206 int32_t tmp = 0;
207 bool eos = false;
Guillaume Chelfi867d4dd2021-07-01 18:38:45 +0200208 bool tunnelFirstFrame = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800209 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
210 eos = true;
211 mInputMetEos = true;
212 ALOGV("[%s] input EOS", mName);
213 }
214 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
215 flags |= C2FrameData::FLAG_CODEC_CONFIG;
216 }
Guillaume Chelfi867d4dd2021-07-01 18:38:45 +0200217 if (buffer->meta()->findInt32("tunnel-first-frame", &tmp) && tmp) {
218 tunnelFirstFrame = true;
219 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800220 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
Wonsik Kime1104ca2020-11-24 15:01:33 -0800221 std::list<std::unique_ptr<C2Work>> items;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800222 std::unique_ptr<C2Work> work(new C2Work);
223 work->input.ordinal.timestamp = timeUs;
224 work->input.ordinal.frameIndex = mFrameIndex++;
225 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
226 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
227 // Keep client timestamp in customOrdinal
228 work->input.ordinal.customOrdinal = timeUs;
229 work->input.buffers.clear();
230
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700231 sp<Codec2Buffer> copy;
Wonsik Kime1104ca2020-11-24 15:01:33 -0800232 bool usesFrameReassembler = false;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800233
Pawin Vongmasa36653902018-11-15 00:10:25 -0800234 if (buffer->size() > 0u) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700235 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800236 std::shared_ptr<C2Buffer> c2buffer;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700237 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800238 return -ENOENT;
239 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700240 // TODO: we want to delay copying buffers.
241 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
242 copy = input->buffers->cloneAndReleaseBuffer(buffer);
243 if (copy != nullptr) {
244 (void)input->extraBuffers.assignSlot(copy);
245 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
246 return UNKNOWN_ERROR;
247 }
248 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
249 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
250 mName, released ? "" : "not ");
251 buffer.clear();
252 } else {
253 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
254 "buffer starvation on component.", mName);
255 }
256 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800257 if (input->frameReassembler) {
258 usesFrameReassembler = true;
259 input->frameReassembler.process(buffer, &items);
260 } else {
Byeongjo Park25c3a3d2020-06-12 17:24:21 +0900261 int32_t cvo = 0;
262 if (buffer->meta()->findInt32("cvo", &cvo)) {
263 int32_t rotation = cvo % 360;
264 // change rotation to counter-clock wise.
265 rotation = ((rotation <= 0) ? 0 : 360) - rotation;
266
267 Mutexed<OutputSurface>::Locked output(mOutputSurface);
268 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
269 output->rotation[frameIndex] = rotation;
270 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800271 work->input.buffers.push_back(c2buffer);
272 if (encryptedBlock) {
273 work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
274 kParamIndexEncryptedBuffer,
275 encryptedBlock->share(0, blockSize, C2Fence())));
276 }
Sungtak Lee04b30352020-07-27 13:57:25 -0700277 }
Wonsik Kimab34ed62019-01-31 15:28:46 -0800278 } else if (eos) {
279 flags |= C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800280 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800281 if (usesFrameReassembler) {
282 if (!items.empty()) {
283 items.front()->input.configUpdate = std::move(mParamsToBeSet);
284 mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
285 }
286 } else {
287 work->input.flags = (C2FrameData::flags_t)flags;
288 // TODO: fill info's
Pawin Vongmasa36653902018-11-15 00:10:25 -0800289
Wonsik Kime1104ca2020-11-24 15:01:33 -0800290 work->input.configUpdate = std::move(mParamsToBeSet);
Guillaume Chelfi867d4dd2021-07-01 18:38:45 +0200291 if (tunnelFirstFrame) {
292 C2StreamTunnelHoldRender::input tunnelHoldRender{
293 0u /* stream */,
294 C2_TRUE /* value */
295 };
296 work->input.configUpdate.push_back(C2Param::Copy(tunnelHoldRender));
297 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800298 work->worklets.clear();
299 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800300
Wonsik Kime1104ca2020-11-24 15:01:33 -0800301 items.push_back(std::move(work));
302
303 eos = eos && buffer->size() > 0u;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800304 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800305 if (eos) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800306 work.reset(new C2Work);
307 work->input.ordinal.timestamp = timeUs;
308 work->input.ordinal.frameIndex = mFrameIndex++;
309 // WORKAROUND: keep client timestamp in customOrdinal
310 work->input.ordinal.customOrdinal = timeUs;
311 work->input.buffers.clear();
312 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800313 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800314 items.push_back(std::move(work));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800315 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800316 c2_status_t err = C2_OK;
317 if (!items.empty()) {
318 {
319 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
320 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
321 for (const std::unique_ptr<C2Work> &work : items) {
322 watcher->onWorkQueued(
323 work->input.ordinal.frameIndex.peeku(),
324 std::vector(work->input.buffers),
325 now);
326 }
327 }
328 err = mComponent->queue(&items);
329 }
330 if (err != C2_OK) {
331 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
332 for (const std::unique_ptr<C2Work> &work : items) {
333 watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
334 }
335 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700336 Mutexed<Input>::Locked input(mInput);
337 bool released = false;
338 if (buffer) {
339 released = input->buffers->releaseBuffer(buffer, nullptr, true);
340 } else if (copy) {
341 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
342 }
343 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
344 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800345 }
346
347 feedInputBufferIfAvailableInternal();
348 return err;
349}
350
351status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> &params) {
352 QueueGuard guard(mSync);
353 if (!guard.isRunning()) {
354 ALOGD("[%s] setParameters is only supported in the running state.", mName);
355 return -ENOSYS;
356 }
357 mParamsToBeSet.insert(mParamsToBeSet.end(),
358 std::make_move_iterator(params.begin()),
359 std::make_move_iterator(params.end()));
360 params.clear();
361 return OK;
362}
363
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800364status_t CCodecBufferChannel::attachBuffer(
365 const std::shared_ptr<C2Buffer> &c2Buffer,
366 const sp<MediaCodecBuffer> &buffer) {
367 if (!buffer->copy(c2Buffer)) {
368 return -ENOSYS;
369 }
370 return OK;
371}
372
373void CCodecBufferChannel::ensureDecryptDestination(size_t size) {
374 if (!mDecryptDestination || mDecryptDestination->size() < size) {
375 sp<IMemoryHeap> heap{new MemoryHeapBase(size * 2)};
376 if (mDecryptDestination && mCrypto && mHeapSeqNum >= 0) {
377 mCrypto->unsetHeap(mHeapSeqNum);
378 }
379 mDecryptDestination = new MemoryBase(heap, 0, size * 2);
380 if (mCrypto) {
381 mHeapSeqNum = mCrypto->setHeap(hardware::fromHeap(heap));
382 }
383 }
384}
385
386int32_t CCodecBufferChannel::getHeapSeqNum(const sp<HidlMemory> &memory) {
387 CHECK(mCrypto);
388 auto it = mHeapSeqNumMap.find(memory);
389 int32_t heapSeqNum = -1;
390 if (it == mHeapSeqNumMap.end()) {
391 heapSeqNum = mCrypto->setHeap(memory);
392 mHeapSeqNumMap.emplace(memory, heapSeqNum);
393 } else {
394 heapSeqNum = it->second;
395 }
396 return heapSeqNum;
397}
398
399status_t CCodecBufferChannel::attachEncryptedBuffer(
400 const sp<hardware::HidlMemory> &memory,
401 bool secure,
402 const uint8_t *key,
403 const uint8_t *iv,
404 CryptoPlugin::Mode mode,
405 CryptoPlugin::Pattern pattern,
406 size_t offset,
407 const CryptoPlugin::SubSample *subSamples,
408 size_t numSubSamples,
409 const sp<MediaCodecBuffer> &buffer) {
410 static const C2MemoryUsage kSecureUsage{C2MemoryUsage::READ_PROTECTED, 0};
411 static const C2MemoryUsage kDefaultReadWriteUsage{
412 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
413
414 size_t size = 0;
415 for (size_t i = 0; i < numSubSamples; ++i) {
416 size += subSamples[i].mNumBytesOfClearData + subSamples[i].mNumBytesOfEncryptedData;
417 }
418 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
419 std::shared_ptr<C2LinearBlock> block;
420 c2_status_t err = pool->fetchLinearBlock(
421 size,
422 secure ? kSecureUsage : kDefaultReadWriteUsage,
423 &block);
424 if (err != C2_OK) {
425 return NO_MEMORY;
426 }
427 if (!secure) {
428 ensureDecryptDestination(size);
429 }
430 ssize_t result = -1;
431 ssize_t codecDataOffset = 0;
432 if (mCrypto) {
433 AString errorDetailMsg;
434 int32_t heapSeqNum = getHeapSeqNum(memory);
435 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
436 hardware::drm::V1_0::DestinationBuffer dst;
437 if (secure) {
438 dst.type = DrmBufferType::NATIVE_HANDLE;
439 dst.secureMemory = hardware::hidl_handle(block->handle());
440 } else {
441 dst.type = DrmBufferType::SHARED_MEMORY;
442 IMemoryToSharedBuffer(
443 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
444 }
445 result = mCrypto->decrypt(
446 key, iv, mode, pattern, src, 0, subSamples, numSubSamples,
447 dst, &errorDetailMsg);
448 if (result < 0) {
449 return result;
450 }
451 if (dst.type == DrmBufferType::SHARED_MEMORY) {
452 C2WriteView view = block->map().get();
453 if (view.error() != C2_OK) {
454 return false;
455 }
456 if (view.size() < result) {
457 return false;
458 }
459 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
460 }
461 } else {
462 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
463 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
464 hidl_vec<SubSample> hidlSubSamples;
465 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
466
467 hardware::cas::native::V1_0::SharedBuffer src{*memory, offset, size};
468 hardware::cas::native::V1_0::DestinationBuffer dst;
469 if (secure) {
470 dst.type = BufferType::NATIVE_HANDLE;
471 dst.secureMemory = hardware::hidl_handle(block->handle());
472 } else {
473 dst.type = BufferType::SHARED_MEMORY;
474 dst.nonsecureMemory = src;
475 }
476
477 CasStatus status = CasStatus::OK;
478 hidl_string detailedError;
479 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
480
481 if (key != nullptr) {
482 sctrl = (ScramblingControl)key[0];
483 // Adjust for the PES offset
484 codecDataOffset = key[2] | (key[3] << 8);
485 }
486
487 auto returnVoid = mDescrambler->descramble(
488 sctrl,
489 hidlSubSamples,
490 src,
491 0,
492 dst,
493 0,
494 [&status, &result, &detailedError] (
495 CasStatus _status, uint32_t _bytesWritten,
496 const hidl_string& _detailedError) {
497 status = _status;
498 result = (ssize_t)_bytesWritten;
499 detailedError = _detailedError;
500 });
501
502 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
503 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
504 mName, returnVoid.description().c_str(), status, result);
505 return UNKNOWN_ERROR;
506 }
507
508 if (result < codecDataOffset) {
509 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
510 return BAD_VALUE;
511 }
512 }
513 if (!secure) {
514 C2WriteView view = block->map().get();
515 if (view.error() != C2_OK) {
516 return UNKNOWN_ERROR;
517 }
518 if (view.size() < result) {
519 return UNKNOWN_ERROR;
520 }
521 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
522 }
523 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
524 block->share(codecDataOffset, result - codecDataOffset, C2Fence{}))};
525 if (!buffer->copy(c2Buffer)) {
526 return -ENOSYS;
527 }
528 return OK;
529}
530
Pawin Vongmasa36653902018-11-15 00:10:25 -0800531status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
532 QueueGuard guard(mSync);
533 if (!guard.isRunning()) {
534 ALOGD("[%s] No more buffers should be queued at current state.", mName);
535 return -ENOSYS;
536 }
537 return queueInputBufferInternal(buffer);
538}
539
540status_t CCodecBufferChannel::queueSecureInputBuffer(
541 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
542 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
543 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
544 AString *errorDetailMsg) {
545 QueueGuard guard(mSync);
546 if (!guard.isRunning()) {
547 ALOGD("[%s] No more buffers should be queued at current state.", mName);
548 return -ENOSYS;
549 }
550
551 if (!hasCryptoOrDescrambler()) {
552 return -ENOSYS;
553 }
554 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
555
Sungtak Lee04b30352020-07-27 13:57:25 -0700556 std::shared_ptr<C2LinearBlock> block;
557 size_t allocSize = buffer->size();
558 size_t bufferSize = 0;
559 c2_status_t blockRes = C2_OK;
560 bool copied = false;
561 if (mSendEncryptedInfoBuffer) {
562 static const C2MemoryUsage kDefaultReadWriteUsage{
563 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
564 constexpr int kAllocGranule0 = 1024 * 64;
565 constexpr int kAllocGranule1 = 1024 * 1024;
566 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
567 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
568 if (allocSize <= kAllocGranule1) {
569 bufferSize = align(allocSize, kAllocGranule0);
570 } else {
571 bufferSize = align(allocSize, kAllocGranule1);
572 }
573 blockRes = pool->fetchLinearBlock(
574 bufferSize, kDefaultReadWriteUsage, &block);
575
576 if (blockRes == C2_OK) {
577 C2WriteView view = block->map().get();
578 if (view.error() == C2_OK && view.size() == bufferSize) {
579 copied = true;
580 // TODO: only copy clear sections
581 memcpy(view.data(), buffer->data(), allocSize);
582 }
583 }
584 }
585
586 if (!copied) {
587 block.reset();
588 }
589
Pawin Vongmasa36653902018-11-15 00:10:25 -0800590 ssize_t result = -1;
591 ssize_t codecDataOffset = 0;
Wonsik Kim557c88c2020-03-13 11:03:52 -0700592 if (numSubSamples == 1
593 && subSamples[0].mNumBytesOfClearData == 0
594 && subSamples[0].mNumBytesOfEncryptedData == 0) {
595 // We don't need to go through crypto or descrambler if the input is empty.
596 result = 0;
597 } else if (mCrypto != nullptr) {
Robert Shih895fba92019-07-16 16:29:44 -0700598 hardware::drm::V1_0::DestinationBuffer destination;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800599 if (secure) {
Robert Shih895fba92019-07-16 16:29:44 -0700600 destination.type = DrmBufferType::NATIVE_HANDLE;
601 destination.secureMemory = hidl_handle(encryptedBuffer->handle());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800602 } else {
Robert Shih895fba92019-07-16 16:29:44 -0700603 destination.type = DrmBufferType::SHARED_MEMORY;
604 IMemoryToSharedBuffer(
605 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800606 }
Robert Shih895fba92019-07-16 16:29:44 -0700607 hardware::drm::V1_0::SharedBuffer source;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800608 encryptedBuffer->fillSourceBuffer(&source);
609 result = mCrypto->decrypt(
610 key, iv, mode, pattern, source, buffer->offset(),
611 subSamples, numSubSamples, destination, errorDetailMsg);
612 if (result < 0) {
Wonsik Kim557c88c2020-03-13 11:03:52 -0700613 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800614 return result;
615 }
Robert Shih895fba92019-07-16 16:29:44 -0700616 if (destination.type == DrmBufferType::SHARED_MEMORY) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800617 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
618 }
619 } else {
620 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
621 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
622 hidl_vec<SubSample> hidlSubSamples;
623 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
624
625 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
626 encryptedBuffer->fillSourceBuffer(&srcBuffer);
627
628 DestinationBuffer dstBuffer;
629 if (secure) {
630 dstBuffer.type = BufferType::NATIVE_HANDLE;
631 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
632 } else {
633 dstBuffer.type = BufferType::SHARED_MEMORY;
634 dstBuffer.nonsecureMemory = srcBuffer;
635 }
636
637 CasStatus status = CasStatus::OK;
638 hidl_string detailedError;
639 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
640
641 if (key != nullptr) {
642 sctrl = (ScramblingControl)key[0];
643 // Adjust for the PES offset
644 codecDataOffset = key[2] | (key[3] << 8);
645 }
646
647 auto returnVoid = mDescrambler->descramble(
648 sctrl,
649 hidlSubSamples,
650 srcBuffer,
651 0,
652 dstBuffer,
653 0,
654 [&status, &result, &detailedError] (
655 CasStatus _status, uint32_t _bytesWritten,
656 const hidl_string& _detailedError) {
657 status = _status;
658 result = (ssize_t)_bytesWritten;
659 detailedError = _detailedError;
660 });
661
662 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
663 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
664 mName, returnVoid.description().c_str(), status, result);
665 return UNKNOWN_ERROR;
666 }
667
668 if (result < codecDataOffset) {
669 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
670 return BAD_VALUE;
671 }
672
673 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
674
675 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
676 encryptedBuffer->copyDecryptedContentFromMemory(result);
677 }
678 }
679
680 buffer->setRange(codecDataOffset, result - codecDataOffset);
Sungtak Lee04b30352020-07-27 13:57:25 -0700681
682 return queueInputBufferInternal(buffer, block, bufferSize);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800683}
684
685void CCodecBufferChannel::feedInputBufferIfAvailable() {
686 QueueGuard guard(mSync);
687 if (!guard.isRunning()) {
688 ALOGV("[%s] We're not running --- no input buffer reported", mName);
689 return;
690 }
691 feedInputBufferIfAvailableInternal();
692}
693
694void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
Taehwan Kimda0517d2020-09-16 17:29:37 +0900695 if (mInputMetEos) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800696 return;
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700697 }
698 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700699 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700700 if (!output->buffers ||
701 output->buffers->hasPending() ||
Wonsik Kim0487b782020-10-28 11:45:50 -0700702 output->buffers->numActiveSlots() >= output->numSlots) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800703 return;
704 }
705 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700706 size_t numActiveSlots = 0;
707 while (!mPipelineWatcher.lock()->pipelineFull()) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800708 sp<MediaCodecBuffer> inBuffer;
709 size_t index;
710 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700711 Mutexed<Input>::Locked input(mInput);
Wonsik Kim0487b782020-10-28 11:45:50 -0700712 numActiveSlots = input->buffers->numActiveSlots();
713 if (numActiveSlots >= input->numSlots) {
714 break;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800715 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700716 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800717 ALOGV("[%s] no new buffer available", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800718 break;
719 }
720 }
721 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
722 mCallback->onInputBufferAvailable(index, inBuffer);
723 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700724 ALOGV("[%s] # active slots after feedInputBufferIfAvailable = %zu", mName, numActiveSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800725}
726
727status_t CCodecBufferChannel::renderOutputBuffer(
728 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800729 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800730 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800731 bool released = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800732 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700733 Mutexed<Output>::Locked output(mOutput);
734 if (output->buffers) {
735 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800736 }
737 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800738 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
739 // set to true.
740 sendOutputBuffers();
741 // input buffer feeding may have been gated by pending output buffers
742 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800743 if (!c2Buffer) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800744 if (released) {
Wonsik Kimf7529dd2019-04-18 17:35:53 -0700745 std::call_once(mRenderWarningFlag, [this] {
746 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
747 "timestamp or render=true with non-video buffers. Apps should "
748 "call releaseOutputBuffer() with render=false for those.",
749 mName);
750 });
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800751 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800752 return INVALID_OPERATION;
753 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800754
755#if 0
756 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
757 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
758 for (const std::shared_ptr<const C2Info> &info : infoParams) {
759 AString res;
760 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
761 if (ix) res.append(", ");
762 res.append(*((int32_t*)info.get() + (ix / 4)));
763 }
764 ALOGV(" [%s]", res.c_str());
765 }
766#endif
767 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
768 std::static_pointer_cast<const C2StreamRotationInfo::output>(
769 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
770 bool flip = rotation && (rotation->flip & 1);
771 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
Byeongjo Park25c3a3d2020-06-12 17:24:21 +0900772
773 {
774 Mutexed<OutputSurface>::Locked output(mOutputSurface);
775 if (output->surface == nullptr) {
776 ALOGI("[%s] cannot render buffer without surface", mName);
777 return OK;
778 }
779 int64_t frameIndex;
780 buffer->meta()->findInt64("frameIndex", &frameIndex);
781 if (output->rotation.count(frameIndex) != 0) {
782 auto it = output->rotation.find(frameIndex);
783 quarters = (it->second / 90) & 3;
784 output->rotation.erase(it);
785 }
786 }
787
Pawin Vongmasa36653902018-11-15 00:10:25 -0800788 uint32_t transform = 0;
789 switch (quarters) {
790 case 0: // no rotation
791 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
792 break;
793 case 1: // 90 degrees counter-clockwise
794 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
795 : HAL_TRANSFORM_ROT_270;
796 break;
797 case 2: // 180 degrees
798 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
799 break;
800 case 3: // 90 degrees clockwise
801 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
802 : HAL_TRANSFORM_ROT_90;
803 break;
804 }
805
806 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
807 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
808 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
809 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
810 if (surfaceScaling) {
811 videoScalingMode = surfaceScaling->value;
812 }
813
814 // Use dataspace from format as it has the default aspects already applied
815 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
816 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
817
818 // HDR static info
819 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
820 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
821 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
822
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800823 // HDR10 plus info
824 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
825 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
826 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
Yichi Chen54be23c2020-06-15 14:30:53 +0800827 if (hdr10PlusInfo && hdr10PlusInfo->flexCount() == 0) {
828 hdr10PlusInfo.reset();
829 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800830
Pawin Vongmasa36653902018-11-15 00:10:25 -0800831 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
832 if (blocks.size() != 1u) {
833 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
834 return UNKNOWN_ERROR;
835 }
836 const C2ConstGraphicBlock &block = blocks.front();
837
838 // TODO: revisit this after C2Fence implementation.
839 android::IGraphicBufferProducer::QueueBufferInput qbi(
840 timestampNs,
841 false, // droppable
842 dataSpace,
843 Rect(blocks.front().crop().left,
844 blocks.front().crop().top,
845 blocks.front().crop().right(),
846 blocks.front().crop().bottom()),
847 videoScalingMode,
848 transform,
849 Fence::NO_FENCE, 0);
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800850 if (hdrStaticInfo || hdr10PlusInfo) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800851 HdrMetadata hdr;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800852 if (hdrStaticInfo) {
wenchangliuf3f92882020-05-14 00:02:01 +0800853 // If mastering max and min luminance fields are 0, do not use them.
854 // It indicates the value may not be present in the stream.
855 if (hdrStaticInfo->mastering.maxLuminance > 0.0f &&
856 hdrStaticInfo->mastering.minLuminance > 0.0f) {
857 struct android_smpte2086_metadata smpte2086_meta = {
858 .displayPrimaryRed = {
859 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
860 },
861 .displayPrimaryGreen = {
862 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
863 },
864 .displayPrimaryBlue = {
865 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
866 },
867 .whitePoint = {
868 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
869 },
870 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
871 .minLuminance = hdrStaticInfo->mastering.minLuminance,
872 };
Yichi Chen54be23c2020-06-15 14:30:53 +0800873 hdr.validTypes |= HdrMetadata::SMPTE2086;
wenchangliuf3f92882020-05-14 00:02:01 +0800874 hdr.smpte2086 = smpte2086_meta;
875 }
Chong Zhang3bb2a7f2020-04-21 10:35:12 -0700876 // If the content light level fields are 0, do not use them, it
877 // indicates the value may not be present in the stream.
878 if (hdrStaticInfo->maxCll > 0.0f && hdrStaticInfo->maxFall > 0.0f) {
879 struct android_cta861_3_metadata cta861_meta = {
880 .maxContentLightLevel = hdrStaticInfo->maxCll,
881 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
882 };
883 hdr.validTypes |= HdrMetadata::CTA861_3;
884 hdr.cta8613 = cta861_meta;
885 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800886 }
887 if (hdr10PlusInfo) {
888 hdr.validTypes |= HdrMetadata::HDR10PLUS;
889 hdr.hdr10plus.assign(
890 hdr10PlusInfo->m.value,
891 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
892 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800893 qbi.setHdrMetadata(hdr);
894 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800895 // we don't have dirty regions
896 qbi.setSurfaceDamage(Region::INVALID_REGION);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800897 android::IGraphicBufferProducer::QueueBufferOutput qbo;
898 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
899 if (result != OK) {
900 ALOGI("[%s] queueBuffer failed: %d", mName, result);
Sungtak Lee47c018a2020-11-07 01:02:49 -0800901 if (result == NO_INIT) {
902 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
903 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800904 return result;
905 }
Josh Hou8eddf4b2021-02-02 16:26:53 +0800906
907 if(android::base::GetBoolProperty("debug.stagefright.fps", false)) {
908 ALOGD("[%s] queue buffer successful", mName);
909 } else {
910 ALOGV("[%s] queue buffer successful", mName);
911 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800912
913 int64_t mediaTimeUs = 0;
914 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
915 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
916
917 return OK;
918}
919
920status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
921 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
922 bool released = false;
923 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700924 Mutexed<Input>::Locked input(mInput);
925 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800926 released = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800927 }
928 }
929 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700930 Mutexed<Output>::Locked output(mOutput);
931 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800932 released = true;
933 }
934 }
935 if (released) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800936 sendOutputBuffers();
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800937 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800938 } else {
939 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
940 }
941 return OK;
942}
943
944void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
945 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700946 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800947
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700948 if (!input->buffers->isArrayMode()) {
949 input->buffers = input->buffers->toArrayMode(input->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800950 }
951
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700952 input->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800953}
954
955void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
956 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700957 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800958
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700959 if (!output->buffers->isArrayMode()) {
960 output->buffers = output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800961 }
962
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700963 output->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800964}
965
966status_t CCodecBufferChannel::start(
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800967 const sp<AMessage> &inputFormat,
968 const sp<AMessage> &outputFormat,
969 bool buffersBoundToCodec) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800970 C2StreamBufferTypeSetting::input iStreamFormat(0u);
971 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kime1104ca2020-11-24 15:01:33 -0800972 C2ComponentKindSetting kind;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800973 C2PortReorderBufferDepthTuning::output reorderDepth;
974 C2PortReorderKeySetting::output reorderKey;
Wonsik Kim078b58e2019-01-09 15:08:06 -0800975 C2PortActualDelayTuning::input inputDelay(0);
976 C2PortActualDelayTuning::output outputDelay(0);
977 C2ActualPipelineDelayTuning pipelineDelay(0);
Sungtak Lee04b30352020-07-27 13:57:25 -0700978 C2SecureModeTuning secureMode(C2Config::SM_UNPROTECTED);
Wonsik Kim078b58e2019-01-09 15:08:06 -0800979
Pawin Vongmasa36653902018-11-15 00:10:25 -0800980 c2_status_t err = mComponent->query(
981 {
982 &iStreamFormat,
983 &oStreamFormat,
Wonsik Kime1104ca2020-11-24 15:01:33 -0800984 &kind,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800985 &reorderDepth,
986 &reorderKey,
Wonsik Kim078b58e2019-01-09 15:08:06 -0800987 &inputDelay,
988 &pipelineDelay,
989 &outputDelay,
Sungtak Lee04b30352020-07-27 13:57:25 -0700990 &secureMode,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800991 },
992 {},
993 C2_DONT_BLOCK,
994 nullptr);
995 if (err == C2_BAD_INDEX) {
Wonsik Kime1104ca2020-11-24 15:01:33 -0800996 if (!iStreamFormat || !oStreamFormat || !kind) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800997 return UNKNOWN_ERROR;
998 }
999 } else if (err != C2_OK) {
1000 return UNKNOWN_ERROR;
1001 }
1002
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001003 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
1004 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
1005 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
1006
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001007 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
1008 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
Wonsik Kim078b58e2019-01-09 15:08:06 -08001009
Pawin Vongmasa36653902018-11-15 00:10:25 -08001010 // TODO: get this from input format
1011 bool secure = mComponent->getName().find(".secure") != std::string::npos;
1012
Sungtak Lee04b30352020-07-27 13:57:25 -07001013 // secure mode is a static parameter (shall not change in the executing state)
1014 mSendEncryptedInfoBuffer = secureMode.value == C2Config::SM_READ_PROTECTED_WITH_ENCRYPTED;
1015
Pawin Vongmasa36653902018-11-15 00:10:25 -08001016 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001017 int poolMask = GetCodec2PoolMask();
1018 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001019
1020 if (inputFormat != nullptr) {
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001021 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
Wonsik Kime1104ca2020-11-24 15:01:33 -08001022 bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001023 C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
1024 API_REFLECTION |
1025 API_VALUES |
1026 API_CURRENT_VALUES |
1027 API_DEPENDENCY |
1028 API_SAME_INPUT_BUFFER);
Wonsik Kime1104ca2020-11-24 15:01:33 -08001029 C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
1030 C2StreamSampleRateInfo::input sampleRate(0u);
1031 C2StreamChannelCountInfo::input channelCount(0u);
1032 C2StreamPcmEncodingInfo::input pcmEncoding(0u);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001033 std::shared_ptr<C2BlockPool> pool;
1034 {
1035 Mutexed<BlockPools>::Locked pools(mBlockPools);
1036
1037 // set default allocator ID.
1038 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001039 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001040
1041 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
1042 // from component, create the input block pool with given ID. Otherwise, use default IDs.
1043 std::vector<std::unique_ptr<C2Param>> params;
Wonsik Kimffb889a2020-05-28 11:32:25 -07001044 C2ApiFeaturesSetting featuresSetting{apiFeatures};
Wonsik Kime1104ca2020-11-24 15:01:33 -08001045 std::vector<C2Param *> stackParams({&featuresSetting});
1046 if (audioEncoder) {
1047 stackParams.push_back(&encoderFrameSize);
1048 stackParams.push_back(&sampleRate);
1049 stackParams.push_back(&channelCount);
1050 stackParams.push_back(&pcmEncoding);
1051 } else {
1052 encoderFrameSize.invalidate();
1053 sampleRate.invalidate();
1054 channelCount.invalidate();
1055 pcmEncoding.invalidate();
1056 }
1057 err = mComponent->query(stackParams,
Pawin Vongmasa36653902018-11-15 00:10:25 -08001058 { C2PortAllocatorsTuning::input::PARAM_TYPE },
1059 C2_DONT_BLOCK,
1060 &params);
1061 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1062 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
1063 mName, params.size(), asString(err), err);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001064 } else if (params.size() == 1) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001065 C2PortAllocatorsTuning::input *inputAllocators =
1066 C2PortAllocatorsTuning::input::From(params[0].get());
1067 if (inputAllocators && inputAllocators->flexCount() > 0) {
1068 std::shared_ptr<C2Allocator> allocator;
1069 // verify allocator IDs and resolve default allocator
1070 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
1071 if (allocator) {
1072 pools->inputAllocatorId = allocator->getId();
1073 } else {
1074 ALOGD("[%s] component requested invalid input allocator ID %u",
1075 mName, inputAllocators->m.values[0]);
1076 }
1077 }
1078 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001079 if (featuresSetting) {
1080 apiFeatures = featuresSetting.value;
1081 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001082
1083 // TODO: use C2Component wrapper to associate this pool with ourselves
1084 if ((poolMask >> pools->inputAllocatorId) & 1) {
1085 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
1086 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
1087 mName, pools->inputAllocatorId,
1088 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1089 asString(err), err);
1090 } else {
1091 err = C2_NOT_FOUND;
1092 }
1093 if (err != C2_OK) {
1094 C2BlockPool::local_id_t inputPoolId =
1095 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1096 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
1097 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
1098 mName, (unsigned long long)inputPoolId,
1099 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1100 asString(err), err);
1101 if (err != C2_OK) {
1102 return NO_MEMORY;
1103 }
1104 }
1105 pools->inputPool = pool;
1106 }
1107
Wonsik Kim51051262018-11-28 13:59:05 -08001108 bool forceArrayMode = false;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001109 Mutexed<Input>::Locked input(mInput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001110 input->inputDelay = inputDelayValue;
1111 input->pipelineDelay = pipelineDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001112 input->numSlots = numInputSlots;
1113 input->extraBuffers.flush();
1114 input->numExtraSlots = 0u;
Wonsik Kime1104ca2020-11-24 15:01:33 -08001115 if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
1116 input->frameReassembler.init(
1117 pool,
1118 {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
1119 encoderFrameSize.value,
1120 sampleRate.value,
1121 channelCount.value,
1122 pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
1123 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001124 bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
1125 // For encrypted content, framework decrypts source buffer (ashmem) into
1126 // C2Buffers. Thus non-conforming codecs can process these.
Wonsik Kime1104ca2020-11-24 15:01:33 -08001127 if (!buffersBoundToCodec
1128 && !input->frameReassembler
1129 && (hasCryptoOrDescrambler() || conforming)) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001130 input->buffers.reset(new SlotInputBuffers(mName));
1131 } else if (graphic) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001132 if (mInputSurface) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001133 input->buffers.reset(new DummyInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001134 } else if (mMetaMode == MODE_ANW) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001135 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
Wonsik Kim1221fd12019-07-12 12:52:05 -07001136 // This is to ensure buffers do not get released prematurely.
1137 // TODO: handle this without going into array mode
1138 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001139 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001140 input->buffers.reset(new GraphicInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001141 }
1142 } else {
1143 if (hasCryptoOrDescrambler()) {
1144 int32_t capacity = kLinearBufferSize;
1145 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
1146 if ((size_t)capacity > kMaxLinearBufferSize) {
1147 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
1148 capacity = kMaxLinearBufferSize;
1149 }
1150 if (mDealer == nullptr) {
1151 mDealer = new MemoryDealer(
1152 align(capacity, MemoryDealer::getAllocationAlignment())
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001153 * (numInputSlots + 1),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001154 "EncryptedLinearInputBuffers");
1155 mDecryptDestination = mDealer->allocate((size_t)capacity);
1156 }
1157 if (mCrypto != nullptr && mHeapSeqNum < 0) {
Robert Shih895fba92019-07-16 16:29:44 -07001158 sp<HidlMemory> heap = fromHeap(mDealer->getMemoryHeap());
1159 mHeapSeqNum = mCrypto->setHeap(heap);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001160 } else {
1161 mHeapSeqNum = -1;
1162 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001163 input->buffers.reset(new EncryptedLinearInputBuffers(
Wonsik Kim078b58e2019-01-09 15:08:06 -08001164 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001165 numInputSlots, mName));
Wonsik Kim51051262018-11-28 13:59:05 -08001166 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001167 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001168 input->buffers.reset(new LinearInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001169 }
1170 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001171 input->buffers->setFormat(inputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001172
1173 if (err == C2_OK) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001174 input->buffers->setPool(pool);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001175 } else {
1176 // TODO: error
1177 }
Wonsik Kim51051262018-11-28 13:59:05 -08001178
1179 if (forceArrayMode) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001180 input->buffers = input->buffers->toArrayMode(numInputSlots);
Wonsik Kim51051262018-11-28 13:59:05 -08001181 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001182 }
1183
1184 if (outputFormat != nullptr) {
1185 sp<IGraphicBufferProducer> outputSurface;
1186 uint32_t outputGeneration;
1187 {
1188 Mutexed<OutputSurface>::Locked output(mOutputSurface);
Sungtak Leed7463d12019-09-04 16:01:00 -07001189 output->maxDequeueBuffers = numOutputSlots +
Sungtak Lee7a7b7422019-07-16 17:40:40 -07001190 reorderDepth.value + kRenderingDepth;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001191 outputSurface = output->surface ?
1192 output->surface->getIGraphicBufferProducer() : nullptr;
Wonsik Kimf5e5c832019-02-21 11:36:05 -08001193 if (outputSurface) {
1194 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1195 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001196 outputGeneration = output->generation;
1197 }
1198
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001199 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001200 C2BlockPool::local_id_t outputPoolId_;
1201
1202 {
1203 Mutexed<BlockPools>::Locked pools(mBlockPools);
1204
1205 // set default allocator ID.
1206 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001207 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001208
1209 // query C2PortAllocatorsTuning::output from component, or use default allocator if
1210 // unsuccessful.
1211 std::vector<std::unique_ptr<C2Param>> params;
1212 err = mComponent->query({ },
1213 { C2PortAllocatorsTuning::output::PARAM_TYPE },
1214 C2_DONT_BLOCK,
1215 &params);
1216 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1217 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
1218 mName, params.size(), asString(err), err);
1219 } else if (err == C2_OK && params.size() == 1) {
1220 C2PortAllocatorsTuning::output *outputAllocators =
1221 C2PortAllocatorsTuning::output::From(params[0].get());
1222 if (outputAllocators && outputAllocators->flexCount() > 0) {
1223 std::shared_ptr<C2Allocator> allocator;
1224 // verify allocator IDs and resolve default allocator
1225 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
1226 if (allocator) {
1227 pools->outputAllocatorId = allocator->getId();
1228 } else {
1229 ALOGD("[%s] component requested invalid output allocator ID %u",
1230 mName, outputAllocators->m.values[0]);
1231 }
1232 }
1233 }
1234
1235 // use bufferqueue if outputting to a surface.
1236 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1237 // if unsuccessful.
1238 if (outputSurface) {
1239 params.clear();
1240 err = mComponent->query({ },
1241 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1242 C2_DONT_BLOCK,
1243 &params);
1244 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1245 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1246 mName, params.size(), asString(err), err);
1247 } else if (err == C2_OK && params.size() == 1) {
1248 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1249 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1250 if (surfaceAllocator) {
1251 std::shared_ptr<C2Allocator> allocator;
1252 // verify allocator IDs and resolve default allocator
1253 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1254 if (allocator) {
1255 pools->outputAllocatorId = allocator->getId();
1256 } else {
1257 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1258 mName, surfaceAllocator->value);
1259 err = C2_BAD_VALUE;
1260 }
1261 }
1262 }
1263 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1264 && err != C2_OK
1265 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1266 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1267 }
1268 }
1269
1270 if ((poolMask >> pools->outputAllocatorId) & 1) {
1271 err = mComponent->createBlockPool(
1272 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1273 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1274 mName, pools->outputAllocatorId,
1275 (unsigned long long)pools->outputPoolId,
1276 asString(err));
1277 } else {
1278 err = C2_NOT_FOUND;
1279 }
1280 if (err != C2_OK) {
1281 // use basic pool instead
1282 pools->outputPoolId =
1283 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1284 }
1285
1286 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1287 // component.
1288 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1289 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1290
1291 std::vector<std::unique_ptr<C2SettingResult>> failures;
1292 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1293 ALOGD("[%s] Configured output block pool ids %llu => %s",
1294 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1295 outputPoolId_ = pools->outputPoolId;
1296 }
1297
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001298 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001299 output->outputDelay = outputDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001300 output->numSlots = numOutputSlots;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001301 if (graphic) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001302 if (outputSurface || !buffersBoundToCodec) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001303 output->buffers.reset(new GraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001304 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001305 output->buffers.reset(new RawGraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001306 }
1307 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001308 output->buffers.reset(new LinearOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001309 }
Wonsik Kime4716c02020-02-28 10:42:21 -08001310 output->buffers->setFormat(outputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001311
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001312 output->buffers->clearStash();
1313 if (reorderDepth) {
1314 output->buffers->setReorderDepth(reorderDepth.value);
1315 }
1316 if (reorderKey) {
1317 output->buffers->setReorderKey(reorderKey.value);
1318 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001319
1320 // Try to set output surface to created block pool if given.
1321 if (outputSurface) {
1322 mComponent->setOutputSurface(
1323 outputPoolId_,
1324 outputSurface,
1325 outputGeneration);
1326 }
1327
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001328 if (oStreamFormat.value == C2BufferData::LINEAR) {
Wonsik Kim58713302020-01-29 22:25:23 -08001329 if (buffersBoundToCodec) {
1330 // WORKAROUND: if we're using early CSD workaround we convert to
1331 // array mode, to appease apps assuming the output
1332 // buffers to be of the same size.
1333 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1334 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001335
1336 int32_t channelCount;
1337 int32_t sampleRate;
1338 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1339 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1340 int32_t delay = 0;
1341 int32_t padding = 0;;
1342 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1343 delay = 0;
1344 }
1345 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1346 padding = 0;
1347 }
1348 if (delay || padding) {
1349 // We need write access to the buffers, and we're already in
1350 // array mode.
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001351 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001352 }
1353 }
1354 }
1355 }
1356
1357 // Set up pipeline control. This has to be done after mInputBuffers and
1358 // mOutputBuffers are initialized to make sure that lingering callbacks
1359 // about buffers from the previous generation do not interfere with the
1360 // newly initialized pipeline capacity.
1361
Wonsik Kim62545252021-01-20 11:25:41 -08001362 if (inputFormat || outputFormat) {
Wonsik Kimab34ed62019-01-31 15:28:46 -08001363 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001364 watcher->inputDelay(inputDelayValue)
1365 .pipelineDelay(pipelineDelayValue)
1366 .outputDelay(outputDelayValue)
Wonsik Kimab34ed62019-01-31 15:28:46 -08001367 .smoothnessFactor(kSmoothnessFactor);
1368 watcher->flush();
1369 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001370
1371 mInputMetEos = false;
1372 mSync.start();
1373 return OK;
1374}
1375
1376status_t CCodecBufferChannel::requestInitialInputBuffers() {
1377 if (mInputSurface) {
1378 return OK;
1379 }
1380
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001381 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001382 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
1383 c2_status_t err = mComponent->query({ &oStreamFormat, &prepend }, {}, C2_DONT_BLOCK, nullptr);
1384 if (err != C2_OK && err != C2_BAD_INDEX) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001385 return UNKNOWN_ERROR;
1386 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001387 size_t numInputSlots = mInput.lock()->numSlots;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001388
1389 struct ClientInputBuffer {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001390 size_t index;
1391 sp<MediaCodecBuffer> buffer;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001392 size_t capacity;
1393 };
1394 std::list<ClientInputBuffer> clientInputBuffers;
1395
1396 {
1397 Mutexed<Input>::Locked input(mInput);
1398 while (clientInputBuffers.size() < numInputSlots) {
1399 ClientInputBuffer clientInputBuffer;
1400 if (!input->buffers->requestNewBuffer(&clientInputBuffer.index,
1401 &clientInputBuffer.buffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001402 break;
1403 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001404 clientInputBuffer.capacity = clientInputBuffer.buffer->capacity();
1405 clientInputBuffers.emplace_back(std::move(clientInputBuffer));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001406 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001407 }
1408 if (clientInputBuffers.empty()) {
1409 ALOGW("[%s] start: cannot allocate memory at all", mName);
1410 return NO_MEMORY;
1411 } else if (clientInputBuffers.size() < numInputSlots) {
1412 ALOGD("[%s] start: cannot allocate memory for all slots, "
1413 "only %zu buffers allocated",
1414 mName, clientInputBuffers.size());
1415 } else {
1416 ALOGV("[%s] %zu initial input buffers available",
1417 mName, clientInputBuffers.size());
1418 }
1419 // Sort input buffers by their capacities in increasing order.
1420 clientInputBuffers.sort(
1421 [](const ClientInputBuffer& a, const ClientInputBuffer& b) {
1422 return a.capacity < b.capacity;
1423 });
1424
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001425 std::list<std::unique_ptr<C2Work>> flushedConfigs;
1426 mFlushedConfigs.lock()->swap(flushedConfigs);
1427 if (!flushedConfigs.empty()) {
1428 err = mComponent->queue(&flushedConfigs);
1429 if (err != C2_OK) {
1430 ALOGW("[%s] Error while queueing a flushed config", mName);
1431 return UNKNOWN_ERROR;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001432 }
1433 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001434 if (oStreamFormat.value == C2BufferData::LINEAR &&
1435 (!prepend || prepend.value == PREPEND_HEADER_TO_NONE)) {
1436 sp<MediaCodecBuffer> buffer = clientInputBuffers.front().buffer;
1437 // WORKAROUND: Some apps expect CSD available without queueing
1438 // any input. Queue an empty buffer to get the CSD.
1439 buffer->setRange(0, 0);
1440 buffer->meta()->clear();
1441 buffer->meta()->setInt64("timeUs", 0);
1442 if (queueInputBufferInternal(buffer) != OK) {
1443 ALOGW("[%s] Error while queueing an empty buffer to get CSD",
1444 mName);
1445 return UNKNOWN_ERROR;
1446 }
1447 clientInputBuffers.pop_front();
1448 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001449
1450 for (const ClientInputBuffer& clientInputBuffer: clientInputBuffers) {
1451 mCallback->onInputBufferAvailable(
1452 clientInputBuffer.index,
1453 clientInputBuffer.buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001454 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001455
Pawin Vongmasa36653902018-11-15 00:10:25 -08001456 return OK;
1457}
1458
1459void CCodecBufferChannel::stop() {
1460 mSync.stop();
1461 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001462}
1463
Wonsik Kim936a89c2020-05-08 16:07:50 -07001464void CCodecBufferChannel::reset() {
1465 stop();
Wonsik Kim62545252021-01-20 11:25:41 -08001466 if (mInputSurface != nullptr) {
1467 mInputSurface.reset();
1468 }
1469 mPipelineWatcher.lock()->flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001470 {
1471 Mutexed<Input>::Locked input(mInput);
1472 input->buffers.reset(new DummyInputBuffers(""));
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001473 input->extraBuffers.flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001474 }
1475 {
1476 Mutexed<Output>::Locked output(mOutput);
1477 output->buffers.reset();
1478 }
1479}
1480
1481void CCodecBufferChannel::release() {
1482 mComponent.reset();
1483 mInputAllocator.reset();
1484 mOutputSurface.lock()->surface.clear();
1485 {
1486 Mutexed<BlockPools>::Locked blockPools{mBlockPools};
1487 blockPools->inputPool.reset();
1488 blockPools->outputPoolIntf.reset();
1489 }
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001490 setCrypto(nullptr);
1491 setDescrambler(nullptr);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001492}
1493
1494
Pawin Vongmasa36653902018-11-15 00:10:25 -08001495void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1496 ALOGV("[%s] flush", mName);
Wonsik Kim62545252021-01-20 11:25:41 -08001497 std::vector<uint64_t> indices;
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001498 std::list<std::unique_ptr<C2Work>> configs;
1499 for (const std::unique_ptr<C2Work> &work : flushedWork) {
Wonsik Kim62545252021-01-20 11:25:41 -08001500 indices.push_back(work->input.ordinal.frameIndex.peeku());
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001501 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1502 continue;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001503 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001504 if (work->input.buffers.empty()
1505 || work->input.buffers.front() == nullptr
1506 || work->input.buffers.front()->data().linearBlocks().empty()) {
1507 ALOGD("[%s] no linear codec config data found", mName);
1508 continue;
1509 }
1510 std::unique_ptr<C2Work> copy(new C2Work);
1511 copy->input.flags = C2FrameData::flags_t(work->input.flags | C2FrameData::FLAG_DROP_FRAME);
1512 copy->input.ordinal = work->input.ordinal;
Wonsik Kim62545252021-01-20 11:25:41 -08001513 copy->input.ordinal.frameIndex = mFrameIndex++;
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001514 copy->input.buffers.insert(
1515 copy->input.buffers.begin(),
1516 work->input.buffers.begin(),
1517 work->input.buffers.end());
1518 for (const std::unique_ptr<C2Param> &param : work->input.configUpdate) {
1519 copy->input.configUpdate.push_back(C2Param::Copy(*param));
1520 }
1521 copy->input.infoBuffers.insert(
1522 copy->input.infoBuffers.begin(),
1523 work->input.infoBuffers.begin(),
1524 work->input.infoBuffers.end());
1525 copy->worklets.emplace_back(new C2Worklet);
1526 configs.push_back(std::move(copy));
1527 ALOGV("[%s] stashed flushed codec config data", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001528 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001529 mFlushedConfigs.lock()->swap(configs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001530 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001531 Mutexed<Input>::Locked input(mInput);
1532 input->buffers->flush();
1533 input->extraBuffers.flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001534 }
1535 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001536 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001537 if (output->buffers) {
1538 output->buffers->flush(flushedWork);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001539 output->buffers->flushStash();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001540 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001541 }
Wonsik Kim62545252021-01-20 11:25:41 -08001542 {
1543 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1544 for (uint64_t index : indices) {
1545 watcher->onWorkDone(index);
1546 }
1547 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001548}
1549
1550void CCodecBufferChannel::onWorkDone(
1551 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
Wonsik Kimab34ed62019-01-31 15:28:46 -08001552 const C2StreamInitDataInfo::output *initData) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001553 if (handleWork(std::move(work), outputFormat, initData)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001554 feedInputBufferIfAvailable();
1555 }
1556}
1557
1558void CCodecBufferChannel::onInputBufferDone(
Wonsik Kimab34ed62019-01-31 15:28:46 -08001559 uint64_t frameIndex, size_t arrayIndex) {
Pawin Vongmasa8e2cfb52019-05-15 05:20:52 -07001560 if (mInputSurface) {
1561 return;
1562 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001563 std::shared_ptr<C2Buffer> buffer =
1564 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001565 bool newInputSlotAvailable;
1566 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001567 Mutexed<Input>::Locked input(mInput);
1568 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1569 if (!newInputSlotAvailable) {
1570 (void)input->extraBuffers.expireComponentBuffer(buffer);
1571 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001572 }
1573 if (newInputSlotAvailable) {
1574 feedInputBufferIfAvailable();
1575 }
1576}
1577
1578bool CCodecBufferChannel::handleWork(
1579 std::unique_ptr<C2Work> work,
1580 const sp<AMessage> &outputFormat,
1581 const C2StreamInitDataInfo::output *initData) {
Wonsik Kim936a89c2020-05-08 16:07:50 -07001582 {
Wonsik Kima4e049d2020-04-28 19:42:23 +00001583 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001584 if (!output->buffers) {
1585 return false;
1586 }
Wonsik Kime75a5da2020-02-14 17:29:03 -08001587 }
1588
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001589 // Whether the output buffer should be reported to the client or not.
1590 bool notifyClient = false;
1591
1592 if (work->result == C2_OK){
1593 notifyClient = true;
1594 } else if (work->result == C2_NOT_FOUND) {
1595 ALOGD("[%s] flushed work; ignored.", mName);
1596 } else {
1597 // C2_OK and C2_NOT_FOUND are the only results that we accept for processing
1598 // the config update.
1599 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1600 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1601 return false;
1602 }
1603
1604 if ((work->input.ordinal.frameIndex -
1605 mFirstValidFrameIndex.load()).peek() < 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001606 // Discard frames from previous generation.
1607 ALOGD("[%s] Discard frames from previous generation.", mName);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001608 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001609 }
1610
Wonsik Kim524b0582019-03-12 11:28:57 -07001611 if (mInputSurface == nullptr && (work->worklets.size() != 1u
Pawin Vongmasa36653902018-11-15 00:10:25 -08001612 || !work->worklets.front()
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001613 || !(work->worklets.front()->output.flags &
1614 C2FrameData::FLAG_INCOMPLETE))) {
1615 mPipelineWatcher.lock()->onWorkDone(
1616 work->input.ordinal.frameIndex.peeku());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001617 }
1618
1619 // NOTE: MediaCodec usage supposedly have only one worklet
1620 if (work->worklets.size() != 1u) {
1621 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1622 mName, work->worklets.size());
1623 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1624 return false;
1625 }
1626
1627 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1628
1629 std::shared_ptr<C2Buffer> buffer;
1630 // NOTE: MediaCodec usage supposedly have only one output stream.
1631 if (worklet->output.buffers.size() > 1u) {
1632 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1633 mName, worklet->output.buffers.size());
1634 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1635 return false;
1636 } else if (worklet->output.buffers.size() == 1u) {
1637 buffer = worklet->output.buffers[0];
1638 if (!buffer) {
1639 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1640 }
1641 }
1642
Wonsik Kim3dedf682021-05-03 10:57:09 -07001643 std::optional<uint32_t> newInputDelay, newPipelineDelay, newOutputDelay, newReorderDepth;
1644 std::optional<C2Config::ordinal_key_t> newReorderKey;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001645 bool needMaxDequeueBufferCountUpdate = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001646 while (!worklet->output.configUpdate.empty()) {
1647 std::unique_ptr<C2Param> param;
1648 worklet->output.configUpdate.back().swap(param);
1649 worklet->output.configUpdate.pop_back();
1650 switch (param->coreIndex().coreIndex()) {
1651 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1652 C2PortReorderBufferDepthTuning::output reorderDepth;
1653 if (reorderDepth.updateFrom(*param)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001654 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1655 mName, reorderDepth.value);
Wonsik Kim3dedf682021-05-03 10:57:09 -07001656 newReorderDepth = reorderDepth.value;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001657 needMaxDequeueBufferCountUpdate = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001658 } else {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001659 ALOGD("[%s] onWorkDone: failed to read reorder depth",
1660 mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001661 }
1662 break;
1663 }
1664 case C2PortReorderKeySetting::CORE_INDEX: {
1665 C2PortReorderKeySetting::output reorderKey;
1666 if (reorderKey.updateFrom(*param)) {
Wonsik Kim3dedf682021-05-03 10:57:09 -07001667 newReorderKey = reorderKey.value;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001668 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1669 mName, reorderKey.value);
1670 } else {
1671 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1672 }
1673 break;
1674 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001675 case C2PortActualDelayTuning::CORE_INDEX: {
1676 if (param->isGlobal()) {
1677 C2ActualPipelineDelayTuning pipelineDelay;
1678 if (pipelineDelay.updateFrom(*param)) {
1679 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1680 mName, pipelineDelay.value);
1681 newPipelineDelay = pipelineDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001682 (void)mPipelineWatcher.lock()->pipelineDelay(
1683 pipelineDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001684 }
1685 }
1686 if (param->forInput()) {
1687 C2PortActualDelayTuning::input inputDelay;
1688 if (inputDelay.updateFrom(*param)) {
1689 ALOGV("[%s] onWorkDone: updating input delay %u",
1690 mName, inputDelay.value);
1691 newInputDelay = inputDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001692 (void)mPipelineWatcher.lock()->inputDelay(
1693 inputDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001694 }
1695 }
1696 if (param->forOutput()) {
1697 C2PortActualDelayTuning::output outputDelay;
1698 if (outputDelay.updateFrom(*param)) {
1699 ALOGV("[%s] onWorkDone: updating output delay %u",
1700 mName, outputDelay.value);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001701 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
Wonsik Kim3dedf682021-05-03 10:57:09 -07001702 newOutputDelay = outputDelay.value;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001703 needMaxDequeueBufferCountUpdate = true;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001704
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001705 }
1706 }
1707 break;
1708 }
ted.sunb1fbfdb2020-06-23 14:03:41 +08001709 case C2PortTunnelSystemTime::CORE_INDEX: {
1710 C2PortTunnelSystemTime::output frameRenderTime;
1711 if (frameRenderTime.updateFrom(*param)) {
1712 ALOGV("[%s] onWorkDone: frame rendered (sys:%lld ns, media:%lld us)",
1713 mName, (long long)frameRenderTime.value,
1714 (long long)worklet->output.ordinal.timestamp.peekll());
1715 mCCodecCallback->onOutputFramesRendered(
1716 worklet->output.ordinal.timestamp.peek(), frameRenderTime.value);
1717 }
1718 break;
1719 }
Guillaume Chelfi867d4dd2021-07-01 18:38:45 +02001720 case C2StreamTunnelHoldRender::CORE_INDEX: {
1721 C2StreamTunnelHoldRender::output firstTunnelFrameHoldRender;
1722 if (!(worklet->output.flags & C2FrameData::FLAG_INCOMPLETE)) break;
1723 if (!firstTunnelFrameHoldRender.updateFrom(*param)) break;
1724 if (firstTunnelFrameHoldRender.value != C2_TRUE) break;
1725 ALOGV("[%s] onWorkDone: first tunnel frame ready", mName);
1726 mCCodecCallback->onFirstTunnelFrameReady();
1727 break;
1728 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001729 default:
1730 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
1731 mName, param->index());
1732 break;
1733 }
1734 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001735 if (newInputDelay || newPipelineDelay) {
1736 Mutexed<Input>::Locked input(mInput);
1737 size_t newNumSlots =
1738 newInputDelay.value_or(input->inputDelay) +
1739 newPipelineDelay.value_or(input->pipelineDelay) +
1740 kSmoothnessFactor;
1741 if (input->buffers->isArrayMode()) {
1742 if (input->numSlots >= newNumSlots) {
1743 input->numExtraSlots = 0;
1744 } else {
1745 input->numExtraSlots = newNumSlots - input->numSlots;
1746 }
1747 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
1748 mName, input->numExtraSlots);
1749 } else {
1750 input->numSlots = newNumSlots;
1751 }
1752 }
Wonsik Kim3dedf682021-05-03 10:57:09 -07001753 size_t numOutputSlots = 0;
1754 uint32_t reorderDepth = 0;
1755 bool outputBuffersChanged = false;
1756 if (newReorderKey || newReorderDepth || needMaxDequeueBufferCountUpdate) {
1757 Mutexed<Output>::Locked output(mOutput);
1758 if (!output->buffers) {
1759 return false;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001760 }
Wonsik Kim3dedf682021-05-03 10:57:09 -07001761 numOutputSlots = output->numSlots;
1762 if (newReorderKey) {
1763 output->buffers->setReorderKey(newReorderKey.value());
1764 }
1765 if (newReorderDepth) {
1766 output->buffers->setReorderDepth(newReorderDepth.value());
1767 }
1768 reorderDepth = output->buffers->getReorderDepth();
1769 if (newOutputDelay) {
1770 output->outputDelay = newOutputDelay.value();
1771 numOutputSlots = newOutputDelay.value() + kSmoothnessFactor;
1772 if (output->numSlots < numOutputSlots) {
1773 output->numSlots = numOutputSlots;
1774 if (output->buffers->isArrayMode()) {
1775 OutputBuffersArray *array =
1776 (OutputBuffersArray *)output->buffers.get();
1777 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
1778 mName, numOutputSlots);
1779 array->grow(numOutputSlots);
1780 outputBuffersChanged = true;
1781 }
1782 }
1783 }
1784 numOutputSlots = output->numSlots;
1785 }
1786 if (outputBuffersChanged) {
1787 mCCodecCallback->onOutputBuffersChanged();
1788 }
1789 if (needMaxDequeueBufferCountUpdate) {
Wonsik Kim315e40a2020-09-09 14:11:50 -07001790 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1791 output->maxDequeueBuffers = numOutputSlots + reorderDepth + kRenderingDepth;
1792 if (output->surface) {
1793 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1794 }
1795 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001796
Pawin Vongmasa36653902018-11-15 00:10:25 -08001797 int32_t flags = 0;
1798 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
1799 flags |= MediaCodec::BUFFER_FLAG_EOS;
1800 ALOGV("[%s] onWorkDone: output EOS", mName);
1801 }
1802
Pawin Vongmasa36653902018-11-15 00:10:25 -08001803 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
1804 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
1805 // the codec input timestamp, but client output timestamp should (reported in timeUs)
1806 // shall correspond to the client input timesamp (in customOrdinal). By using the
1807 // delta between the two, this allows for some timestamp deviation - e.g. if one input
1808 // produces multiple output.
1809 c2_cntr64_t timestamp =
1810 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
1811 - work->input.ordinal.timestamp;
Wonsik Kim95ba0162019-03-19 15:51:54 -07001812 if (mInputSurface != nullptr) {
1813 // When using input surface we need to restore the original input timestamp.
1814 timestamp = work->input.ordinal.customOrdinal;
1815 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001816 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
1817 mName,
1818 work->input.ordinal.customOrdinal.peekll(),
1819 work->input.ordinal.timestamp.peekll(),
1820 worklet->output.ordinal.timestamp.peekll(),
1821 timestamp.peekll());
1822
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001823 // csd cannot be re-ordered and will always arrive first.
Pawin Vongmasa36653902018-11-15 00:10:25 -08001824 if (initData != nullptr) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001825 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001826 if (output->buffers && outputFormat) {
1827 output->buffers->updateSkipCutBuffer(outputFormat);
1828 output->buffers->setFormat(outputFormat);
1829 }
1830 if (!notifyClient) {
1831 return false;
1832 }
1833 size_t index;
1834 sp<MediaCodecBuffer> outBuffer;
Wonsik Kim936a89c2020-05-08 16:07:50 -07001835 if (output->buffers && output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001836 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
1837 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
1838 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
1839
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001840 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001841 mCallback->onOutputBufferAvailable(index, outBuffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001842 } else {
1843 ALOGD("[%s] onWorkDone: unable to register csd", mName);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001844 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001845 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001846 return false;
1847 }
1848 }
1849
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001850 if (notifyClient && !buffer && !flags) {
Wonsik Kim35bf5732020-05-14 17:40:29 +00001851 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
Pawin Vongmasa36653902018-11-15 00:10:25 -08001852 mName, work->input.ordinal.frameIndex.peekull());
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001853 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001854 }
1855
1856 if (buffer) {
1857 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
1858 // TODO: properly translate these to metadata
1859 switch (info->coreIndex().coreIndex()) {
1860 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001861 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001862 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
1863 }
1864 break;
1865 default:
1866 break;
1867 }
1868 }
1869 }
1870
1871 {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001872 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimc23cc402020-05-28 14:53:40 -07001873 if (!output->buffers) {
1874 return false;
1875 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001876 output->buffers->pushToStash(
1877 buffer,
1878 notifyClient,
1879 timestamp.peek(),
1880 flags,
1881 outputFormat,
1882 worklet->output.ordinal);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001883 }
1884 sendOutputBuffers();
1885 return true;
1886}
1887
1888void CCodecBufferChannel::sendOutputBuffers() {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001889 OutputBuffers::BufferAction action;
Wonsik Kima4e049d2020-04-28 19:42:23 +00001890 size_t index;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001891 sp<MediaCodecBuffer> outBuffer;
1892 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001893
1894 while (true) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001895 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001896 if (!output->buffers) {
1897 return;
1898 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001899 action = output->buffers->popFromStashAndRegister(
1900 &c2Buffer, &index, &outBuffer);
1901 switch (action) {
1902 case OutputBuffers::SKIP:
1903 return;
1904 case OutputBuffers::DISCARD:
1905 break;
1906 case OutputBuffers::NOTIFY_CLIENT:
Wonsik Kima4e049d2020-04-28 19:42:23 +00001907 output.unlock();
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001908 mCallback->onOutputBufferAvailable(index, outBuffer);
1909 break;
1910 case OutputBuffers::REALLOCATE:
1911 if (!output->buffers->isArrayMode()) {
1912 output->buffers =
1913 output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001914 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001915 static_cast<OutputBuffersArray*>(output->buffers.get())->
1916 realloc(c2Buffer);
1917 output.unlock();
1918 mCCodecCallback->onOutputBuffersChanged();
Wonsik Kim4ada73d2020-05-26 14:58:07 -07001919 break;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001920 case OutputBuffers::RETRY:
1921 ALOGV("[%s] sendOutputBuffers: unable to register output buffer",
1922 mName);
1923 return;
1924 default:
1925 LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: "
1926 "corrupted BufferAction value (%d) "
1927 "returned from popFromStashAndRegister.",
1928 mName, int(action));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001929 return;
1930 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001931 }
1932}
1933
1934status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
1935 static std::atomic_uint32_t surfaceGeneration{0};
1936 uint32_t generation = (getpid() << 10) |
1937 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
1938 & ((1 << 10) - 1));
1939
1940 sp<IGraphicBufferProducer> producer;
1941 if (newSurface) {
1942 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
Sungtak Leeab6f2f32019-02-15 14:43:51 -08001943 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
Sungtak Lee08515812019-06-05 11:16:32 -07001944 newSurface->setMaxDequeuedBufferCount(mOutputSurface.lock()->maxDequeueBuffers);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001945 producer = newSurface->getIGraphicBufferProducer();
1946 producer->setGenerationNumber(generation);
1947 } else {
1948 ALOGE("[%s] setting output surface to null", mName);
1949 return INVALID_OPERATION;
1950 }
1951
1952 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
1953 C2BlockPool::local_id_t outputPoolId;
1954 {
1955 Mutexed<BlockPools>::Locked pools(mBlockPools);
1956 outputPoolId = pools->outputPoolId;
1957 outputPoolIntf = pools->outputPoolIntf;
1958 }
1959
1960 if (outputPoolIntf) {
1961 if (mComponent->setOutputSurface(
1962 outputPoolId,
1963 producer,
1964 generation) != C2_OK) {
1965 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
1966 return INVALID_OPERATION;
1967 }
1968 }
1969
1970 {
1971 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1972 output->surface = newSurface;
1973 output->generation = generation;
1974 }
1975
1976 return OK;
1977}
1978
Wonsik Kimab34ed62019-01-31 15:28:46 -08001979PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001980 // When client pushed EOS, we want all the work to be done quickly.
1981 // Otherwise, component may have stalled work due to input starvation up to
1982 // the sum of the delay in the pipeline.
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001983 size_t n = 0;
1984 if (!mInputMetEos) {
1985 size_t outputDelay = mOutput.lock()->outputDelay;
1986 Mutexed<Input>::Locked input(mInput);
1987 n = input->inputDelay + input->pipelineDelay + outputDelay;
1988 }
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001989 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
Wonsik Kimab34ed62019-01-31 15:28:46 -08001990}
1991
Pawin Vongmasa36653902018-11-15 00:10:25 -08001992void CCodecBufferChannel::setMetaMode(MetaMode mode) {
1993 mMetaMode = mode;
1994}
1995
Wonsik Kim596187e2019-10-25 12:44:10 -07001996void CCodecBufferChannel::setCrypto(const sp<ICrypto> &crypto) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001997 if (mCrypto != nullptr) {
1998 for (std::pair<wp<HidlMemory>, int32_t> entry : mHeapSeqNumMap) {
1999 mCrypto->unsetHeap(entry.second);
2000 }
2001 mHeapSeqNumMap.clear();
2002 if (mHeapSeqNum >= 0) {
2003 mCrypto->unsetHeap(mHeapSeqNum);
2004 mHeapSeqNum = -1;
2005 }
2006 }
Wonsik Kim596187e2019-10-25 12:44:10 -07002007 mCrypto = crypto;
2008}
2009
2010void CCodecBufferChannel::setDescrambler(const sp<IDescrambler> &descrambler) {
2011 mDescrambler = descrambler;
2012}
2013
Pawin Vongmasa36653902018-11-15 00:10:25 -08002014status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
2015 // C2_OK is always translated to OK.
2016 if (c2s == C2_OK) {
2017 return OK;
2018 }
2019
2020 // Operation-dependent translation
2021 // TODO: Add as necessary
2022 switch (c2op) {
2023 case C2_OPERATION_Component_start:
2024 switch (c2s) {
2025 case C2_NO_MEMORY:
2026 return NO_MEMORY;
2027 default:
2028 return UNKNOWN_ERROR;
2029 }
2030 default:
2031 break;
2032 }
2033
2034 // Backup operation-agnostic translation
2035 switch (c2s) {
2036 case C2_BAD_INDEX:
2037 return BAD_INDEX;
2038 case C2_BAD_VALUE:
2039 return BAD_VALUE;
2040 case C2_BLOCKING:
2041 return WOULD_BLOCK;
2042 case C2_DUPLICATE:
2043 return ALREADY_EXISTS;
2044 case C2_NO_INIT:
2045 return NO_INIT;
2046 case C2_NO_MEMORY:
2047 return NO_MEMORY;
2048 case C2_NOT_FOUND:
2049 return NAME_NOT_FOUND;
2050 case C2_TIMED_OUT:
2051 return TIMED_OUT;
2052 case C2_BAD_STATE:
2053 case C2_CANCELED:
2054 case C2_CANNOT_DO:
2055 case C2_CORRUPTED:
2056 case C2_OMITTED:
2057 case C2_REFUSED:
2058 return UNKNOWN_ERROR;
2059 default:
2060 return -static_cast<status_t>(c2s);
2061 }
2062}
2063
2064} // namespace android