blob: 74480b862b287bd703d67db5b4a553c459e84a8e [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodecBufferChannel"
19#include <utils/Log.h>
20
Pawin Vongmasae7bb8612020-06-04 06:15:22 -070021#include <algorithm>
22#include <list>
Pawin Vongmasa36653902018-11-15 00:10:25 -080023#include <numeric>
24
25#include <C2AllocatorGralloc.h>
26#include <C2PlatformSupport.h>
27#include <C2BlockInternal.h>
28#include <C2Config.h>
29#include <C2Debug.h>
30
31#include <android/hardware/cas/native/1.0/IDescrambler.h>
Robert Shih895fba92019-07-16 16:29:44 -070032#include <android/hardware/drm/1.0/types.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080033#include <android-base/stringprintf.h>
Wonsik Kimfb7a7672019-12-27 17:13:33 -080034#include <binder/MemoryBase.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080035#include <binder/MemoryDealer.h>
Ray Essick18ea0452019-08-27 16:07:27 -070036#include <cutils/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080037#include <gui/Surface.h>
Robert Shih895fba92019-07-16 16:29:44 -070038#include <hidlmemory/FrameworkUtils.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080039#include <media/openmax/OMX_Core.h>
40#include <media/stagefright/foundation/ABuffer.h>
41#include <media/stagefright/foundation/ALookup.h>
42#include <media/stagefright/foundation/AMessage.h>
43#include <media/stagefright/foundation/AUtils.h>
44#include <media/stagefright/foundation/hexdump.h>
45#include <media/stagefright/MediaCodec.h>
46#include <media/stagefright/MediaCodecConstants.h>
Wonsik Kim155d5cb2019-10-09 12:49:49 -070047#include <media/stagefright/SkipCutBuffer.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080048#include <media/MediaCodecBuffer.h>
Wonsik Kim41d83432020-04-27 16:40:49 -070049#include <mediadrm/ICrypto.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080050#include <system/window.h>
51
52#include "CCodecBufferChannel.h"
53#include "Codec2Buffer.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080054
55namespace android {
56
57using android::base::StringPrintf;
58using hardware::hidl_handle;
59using hardware::hidl_string;
60using hardware::hidl_vec;
Robert Shih895fba92019-07-16 16:29:44 -070061using hardware::fromHeap;
62using hardware::HidlMemory;
63
Pawin Vongmasa36653902018-11-15 00:10:25 -080064using namespace hardware::cas::V1_0;
65using namespace hardware::cas::native::V1_0;
66
67using CasStatus = hardware::cas::V1_0::Status;
Robert Shih895fba92019-07-16 16:29:44 -070068using DrmBufferType = hardware::drm::V1_0::BufferType;
Pawin Vongmasa36653902018-11-15 00:10:25 -080069
Pawin Vongmasa36653902018-11-15 00:10:25 -080070namespace {
71
Wonsik Kim469c8342019-04-11 16:46:09 -070072constexpr size_t kSmoothnessFactor = 4;
73constexpr size_t kRenderingDepth = 3;
Pawin Vongmasa36653902018-11-15 00:10:25 -080074
Sungtak Leeab6f2f32019-02-15 14:43:51 -080075// This is for keeping IGBP's buffer dropping logic in legacy mode other
76// than making it non-blocking. Do not change this value.
77const static size_t kDequeueTimeoutNs = 0;
78
Pawin Vongmasa36653902018-11-15 00:10:25 -080079} // namespace
80
81CCodecBufferChannel::QueueGuard::QueueGuard(
82 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
83 Mutex::Autolock l(mSync.mGuardLock);
84 // At this point it's guaranteed that mSync is not under state transition,
85 // as we are holding its mutex.
86
87 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
88 if (count->value == -1) {
89 mRunning = false;
90 } else {
91 ++count->value;
92 mRunning = true;
93 }
94}
95
96CCodecBufferChannel::QueueGuard::~QueueGuard() {
97 if (mRunning) {
98 // We are not holding mGuardLock at this point so that QueueSync::stop() can
99 // keep holding the lock until mCount reaches zero.
100 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
101 --count->value;
102 count->cond.broadcast();
103 }
104}
105
106void CCodecBufferChannel::QueueSync::start() {
107 Mutex::Autolock l(mGuardLock);
108 // If stopped, it goes to running state; otherwise no-op.
109 Mutexed<Counter>::Locked count(mCount);
110 if (count->value == -1) {
111 count->value = 0;
112 }
113}
114
115void CCodecBufferChannel::QueueSync::stop() {
116 Mutex::Autolock l(mGuardLock);
117 Mutexed<Counter>::Locked count(mCount);
118 if (count->value == -1) {
119 // no-op
120 return;
121 }
122 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
123 // mCount can only decrement. In other words, threads that acquired the lock
124 // are allowed to finish execution but additional threads trying to acquire
125 // the lock at this point will block, and then get QueueGuard at STOPPED
126 // state.
127 while (count->value != 0) {
128 count.waitForCondition(count->cond);
129 }
130 count->value = -1;
131}
132
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700133// Input
134
135CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
136
Pawin Vongmasa36653902018-11-15 00:10:25 -0800137// CCodecBufferChannel
138
139CCodecBufferChannel::CCodecBufferChannel(
140 const std::shared_ptr<CCodecCallback> &callback)
141 : mHeapSeqNum(-1),
142 mCCodecCallback(callback),
143 mFrameIndex(0u),
144 mFirstValidFrameIndex(0u),
145 mMetaMode(MODE_NONE),
Sungtak Lee04b30352020-07-27 13:57:25 -0700146 mInputMetEos(false),
147 mSendEncryptedInfoBuffer(false) {
Sungtak Leed7463d12019-09-04 16:01:00 -0700148 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700149 {
150 Mutexed<Input>::Locked input(mInput);
151 input->buffers.reset(new DummyInputBuffers(""));
152 input->extraBuffers.flush();
153 input->inputDelay = 0u;
154 input->pipelineDelay = 0u;
155 input->numSlots = kSmoothnessFactor;
156 input->numExtraSlots = 0u;
157 }
158 {
159 Mutexed<Output>::Locked output(mOutput);
160 output->outputDelay = 0u;
161 output->numSlots = kSmoothnessFactor;
162 }
David Stevensc3fbb282021-01-18 18:11:20 +0900163 {
164 Mutexed<BlockPools>::Locked pools(mBlockPools);
165 pools->outputPoolId = C2BlockPool::BASIC_LINEAR;
166 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800167}
168
169CCodecBufferChannel::~CCodecBufferChannel() {
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800170 if (mCrypto != nullptr && mHeapSeqNum >= 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800171 mCrypto->unsetHeap(mHeapSeqNum);
172 }
173}
174
175void CCodecBufferChannel::setComponent(
176 const std::shared_ptr<Codec2Client::Component> &component) {
177 mComponent = component;
178 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
179 mName = mComponentName.c_str();
180}
181
182status_t CCodecBufferChannel::setInputSurface(
183 const std::shared_ptr<InputSurfaceWrapper> &surface) {
184 ALOGV("[%s] setInputSurface", mName);
185 mInputSurface = surface;
186 return mInputSurface->connect(mComponent);
187}
188
189status_t CCodecBufferChannel::signalEndOfInputStream() {
190 if (mInputSurface == nullptr) {
191 return INVALID_OPERATION;
192 }
193 return mInputSurface->signalEndOfInputStream();
194}
195
Sungtak Lee04b30352020-07-27 13:57:25 -0700196status_t CCodecBufferChannel::queueInputBufferInternal(
197 sp<MediaCodecBuffer> buffer,
198 std::shared_ptr<C2LinearBlock> encryptedBlock,
199 size_t blockSize) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800200 int64_t timeUs;
201 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
202
203 if (mInputMetEos) {
204 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
205 return OK;
206 }
207
208 int32_t flags = 0;
209 int32_t tmp = 0;
210 bool eos = false;
211 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
212 eos = true;
213 mInputMetEos = true;
214 ALOGV("[%s] input EOS", mName);
215 }
216 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
217 flags |= C2FrameData::FLAG_CODEC_CONFIG;
218 }
219 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
Wonsik Kim0379ae82020-11-24 15:01:33 -0800220 std::list<std::unique_ptr<C2Work>> items;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800221 std::unique_ptr<C2Work> work(new C2Work);
222 work->input.ordinal.timestamp = timeUs;
223 work->input.ordinal.frameIndex = mFrameIndex++;
224 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
225 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
226 // Keep client timestamp in customOrdinal
227 work->input.ordinal.customOrdinal = timeUs;
228 work->input.buffers.clear();
229
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700230 sp<Codec2Buffer> copy;
Wonsik Kim0379ae82020-11-24 15:01:33 -0800231 bool usesFrameReassembler = false;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800232
Pawin Vongmasa36653902018-11-15 00:10:25 -0800233 if (buffer->size() > 0u) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700234 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800235 std::shared_ptr<C2Buffer> c2buffer;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700236 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800237 return -ENOENT;
238 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700239 // TODO: we want to delay copying buffers.
240 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
241 copy = input->buffers->cloneAndReleaseBuffer(buffer);
242 if (copy != nullptr) {
243 (void)input->extraBuffers.assignSlot(copy);
244 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
245 return UNKNOWN_ERROR;
246 }
247 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
248 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
249 mName, released ? "" : "not ");
250 buffer.clear();
251 } else {
252 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
253 "buffer starvation on component.", mName);
254 }
255 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800256 if (input->frameReassembler) {
257 usesFrameReassembler = true;
258 input->frameReassembler.process(buffer, &items);
259 } else {
260 int32_t cvo = 0;
261 if (buffer->meta()->findInt32("cvo", &cvo)) {
262 int32_t rotation = cvo % 360;
263 // change rotation to counter-clock wise.
264 rotation = ((rotation <= 0) ? 0 : 360) - rotation;
265
266 Mutexed<OutputSurface>::Locked output(mOutputSurface);
267 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
268 output->rotation[frameIndex] = rotation;
269 }
270 work->input.buffers.push_back(c2buffer);
271 if (encryptedBlock) {
272 work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
273 kParamIndexEncryptedBuffer,
274 encryptedBlock->share(0, blockSize, C2Fence())));
275 }
Byeongjo Park2eef13e2020-06-12 17:24:21 +0900276 }
Wonsik Kimab34ed62019-01-31 15:28:46 -0800277 } else if (eos) {
278 flags |= C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800279 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800280 if (usesFrameReassembler) {
281 if (!items.empty()) {
282 items.front()->input.configUpdate = std::move(mParamsToBeSet);
283 mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
284 }
285 } else {
286 work->input.flags = (C2FrameData::flags_t)flags;
287 // TODO: fill info's
Pawin Vongmasa36653902018-11-15 00:10:25 -0800288
Wonsik Kim0379ae82020-11-24 15:01:33 -0800289 work->input.configUpdate = std::move(mParamsToBeSet);
290 work->worklets.clear();
291 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800292
Wonsik Kim0379ae82020-11-24 15:01:33 -0800293 items.push_back(std::move(work));
294
295 eos = eos && buffer->size() > 0u;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800296 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800297 if (eos) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800298 work.reset(new C2Work);
299 work->input.ordinal.timestamp = timeUs;
300 work->input.ordinal.frameIndex = mFrameIndex++;
301 // WORKAROUND: keep client timestamp in customOrdinal
302 work->input.ordinal.customOrdinal = timeUs;
303 work->input.buffers.clear();
304 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800305 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800306 items.push_back(std::move(work));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800307 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800308 c2_status_t err = C2_OK;
309 if (!items.empty()) {
310 {
311 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
312 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
313 for (const std::unique_ptr<C2Work> &work : items) {
314 watcher->onWorkQueued(
315 work->input.ordinal.frameIndex.peeku(),
316 std::vector(work->input.buffers),
317 now);
318 }
319 }
320 err = mComponent->queue(&items);
321 }
322 if (err != C2_OK) {
323 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
324 for (const std::unique_ptr<C2Work> &work : items) {
325 watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
326 }
327 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700328 Mutexed<Input>::Locked input(mInput);
329 bool released = false;
330 if (buffer) {
331 released = input->buffers->releaseBuffer(buffer, nullptr, true);
332 } else if (copy) {
333 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
334 }
335 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
336 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800337 }
338
339 feedInputBufferIfAvailableInternal();
340 return err;
341}
342
343status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> &params) {
344 QueueGuard guard(mSync);
345 if (!guard.isRunning()) {
346 ALOGD("[%s] setParameters is only supported in the running state.", mName);
347 return -ENOSYS;
348 }
349 mParamsToBeSet.insert(mParamsToBeSet.end(),
350 std::make_move_iterator(params.begin()),
351 std::make_move_iterator(params.end()));
352 params.clear();
353 return OK;
354}
355
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800356status_t CCodecBufferChannel::attachBuffer(
357 const std::shared_ptr<C2Buffer> &c2Buffer,
358 const sp<MediaCodecBuffer> &buffer) {
359 if (!buffer->copy(c2Buffer)) {
360 return -ENOSYS;
361 }
362 return OK;
363}
364
365void CCodecBufferChannel::ensureDecryptDestination(size_t size) {
366 if (!mDecryptDestination || mDecryptDestination->size() < size) {
367 sp<IMemoryHeap> heap{new MemoryHeapBase(size * 2)};
368 if (mDecryptDestination && mCrypto && mHeapSeqNum >= 0) {
369 mCrypto->unsetHeap(mHeapSeqNum);
370 }
371 mDecryptDestination = new MemoryBase(heap, 0, size * 2);
372 if (mCrypto) {
373 mHeapSeqNum = mCrypto->setHeap(hardware::fromHeap(heap));
374 }
375 }
376}
377
378int32_t CCodecBufferChannel::getHeapSeqNum(const sp<HidlMemory> &memory) {
379 CHECK(mCrypto);
380 auto it = mHeapSeqNumMap.find(memory);
381 int32_t heapSeqNum = -1;
382 if (it == mHeapSeqNumMap.end()) {
383 heapSeqNum = mCrypto->setHeap(memory);
384 mHeapSeqNumMap.emplace(memory, heapSeqNum);
385 } else {
386 heapSeqNum = it->second;
387 }
388 return heapSeqNum;
389}
390
391status_t CCodecBufferChannel::attachEncryptedBuffer(
392 const sp<hardware::HidlMemory> &memory,
393 bool secure,
394 const uint8_t *key,
395 const uint8_t *iv,
396 CryptoPlugin::Mode mode,
397 CryptoPlugin::Pattern pattern,
398 size_t offset,
399 const CryptoPlugin::SubSample *subSamples,
400 size_t numSubSamples,
401 const sp<MediaCodecBuffer> &buffer) {
402 static const C2MemoryUsage kSecureUsage{C2MemoryUsage::READ_PROTECTED, 0};
403 static const C2MemoryUsage kDefaultReadWriteUsage{
404 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
405
406 size_t size = 0;
407 for (size_t i = 0; i < numSubSamples; ++i) {
408 size += subSamples[i].mNumBytesOfClearData + subSamples[i].mNumBytesOfEncryptedData;
409 }
410 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
411 std::shared_ptr<C2LinearBlock> block;
412 c2_status_t err = pool->fetchLinearBlock(
413 size,
414 secure ? kSecureUsage : kDefaultReadWriteUsage,
415 &block);
416 if (err != C2_OK) {
417 return NO_MEMORY;
418 }
419 if (!secure) {
420 ensureDecryptDestination(size);
421 }
422 ssize_t result = -1;
423 ssize_t codecDataOffset = 0;
424 if (mCrypto) {
425 AString errorDetailMsg;
426 int32_t heapSeqNum = getHeapSeqNum(memory);
427 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
428 hardware::drm::V1_0::DestinationBuffer dst;
429 if (secure) {
430 dst.type = DrmBufferType::NATIVE_HANDLE;
431 dst.secureMemory = hardware::hidl_handle(block->handle());
432 } else {
433 dst.type = DrmBufferType::SHARED_MEMORY;
434 IMemoryToSharedBuffer(
435 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
436 }
437 result = mCrypto->decrypt(
438 key, iv, mode, pattern, src, 0, subSamples, numSubSamples,
439 dst, &errorDetailMsg);
440 if (result < 0) {
441 return result;
442 }
443 if (dst.type == DrmBufferType::SHARED_MEMORY) {
444 C2WriteView view = block->map().get();
445 if (view.error() != C2_OK) {
446 return false;
447 }
448 if (view.size() < result) {
449 return false;
450 }
451 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
452 }
453 } else {
454 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
455 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
456 hidl_vec<SubSample> hidlSubSamples;
457 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
458
459 hardware::cas::native::V1_0::SharedBuffer src{*memory, offset, size};
460 hardware::cas::native::V1_0::DestinationBuffer dst;
461 if (secure) {
462 dst.type = BufferType::NATIVE_HANDLE;
463 dst.secureMemory = hardware::hidl_handle(block->handle());
464 } else {
465 dst.type = BufferType::SHARED_MEMORY;
466 dst.nonsecureMemory = src;
467 }
468
469 CasStatus status = CasStatus::OK;
470 hidl_string detailedError;
471 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
472
473 if (key != nullptr) {
474 sctrl = (ScramblingControl)key[0];
475 // Adjust for the PES offset
476 codecDataOffset = key[2] | (key[3] << 8);
477 }
478
479 auto returnVoid = mDescrambler->descramble(
480 sctrl,
481 hidlSubSamples,
482 src,
483 0,
484 dst,
485 0,
486 [&status, &result, &detailedError] (
487 CasStatus _status, uint32_t _bytesWritten,
488 const hidl_string& _detailedError) {
489 status = _status;
490 result = (ssize_t)_bytesWritten;
491 detailedError = _detailedError;
492 });
493
494 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
495 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
496 mName, returnVoid.description().c_str(), status, result);
497 return UNKNOWN_ERROR;
498 }
499
500 if (result < codecDataOffset) {
501 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
502 return BAD_VALUE;
503 }
504 }
505 if (!secure) {
506 C2WriteView view = block->map().get();
507 if (view.error() != C2_OK) {
508 return UNKNOWN_ERROR;
509 }
510 if (view.size() < result) {
511 return UNKNOWN_ERROR;
512 }
513 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
514 }
515 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
516 block->share(codecDataOffset, result - codecDataOffset, C2Fence{}))};
517 if (!buffer->copy(c2Buffer)) {
518 return -ENOSYS;
519 }
520 return OK;
521}
522
Pawin Vongmasa36653902018-11-15 00:10:25 -0800523status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
524 QueueGuard guard(mSync);
525 if (!guard.isRunning()) {
526 ALOGD("[%s] No more buffers should be queued at current state.", mName);
527 return -ENOSYS;
528 }
529 return queueInputBufferInternal(buffer);
530}
531
532status_t CCodecBufferChannel::queueSecureInputBuffer(
533 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
534 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
535 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
536 AString *errorDetailMsg) {
537 QueueGuard guard(mSync);
538 if (!guard.isRunning()) {
539 ALOGD("[%s] No more buffers should be queued at current state.", mName);
540 return -ENOSYS;
541 }
542
543 if (!hasCryptoOrDescrambler()) {
544 return -ENOSYS;
545 }
546 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
547
Sungtak Lee04b30352020-07-27 13:57:25 -0700548 std::shared_ptr<C2LinearBlock> block;
549 size_t allocSize = buffer->size();
550 size_t bufferSize = 0;
551 c2_status_t blockRes = C2_OK;
552 bool copied = false;
553 if (mSendEncryptedInfoBuffer) {
554 static const C2MemoryUsage kDefaultReadWriteUsage{
555 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
556 constexpr int kAllocGranule0 = 1024 * 64;
557 constexpr int kAllocGranule1 = 1024 * 1024;
558 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
559 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
560 if (allocSize <= kAllocGranule1) {
561 bufferSize = align(allocSize, kAllocGranule0);
562 } else {
563 bufferSize = align(allocSize, kAllocGranule1);
564 }
565 blockRes = pool->fetchLinearBlock(
566 bufferSize, kDefaultReadWriteUsage, &block);
567
568 if (blockRes == C2_OK) {
569 C2WriteView view = block->map().get();
570 if (view.error() == C2_OK && view.size() == bufferSize) {
571 copied = true;
572 // TODO: only copy clear sections
573 memcpy(view.data(), buffer->data(), allocSize);
574 }
575 }
576 }
577
578 if (!copied) {
579 block.reset();
580 }
581
Pawin Vongmasa36653902018-11-15 00:10:25 -0800582 ssize_t result = -1;
583 ssize_t codecDataOffset = 0;
Wonsik Kim557c88c2020-03-13 11:03:52 -0700584 if (numSubSamples == 1
585 && subSamples[0].mNumBytesOfClearData == 0
586 && subSamples[0].mNumBytesOfEncryptedData == 0) {
587 // We don't need to go through crypto or descrambler if the input is empty.
588 result = 0;
589 } else if (mCrypto != nullptr) {
Robert Shih895fba92019-07-16 16:29:44 -0700590 hardware::drm::V1_0::DestinationBuffer destination;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800591 if (secure) {
Robert Shih895fba92019-07-16 16:29:44 -0700592 destination.type = DrmBufferType::NATIVE_HANDLE;
593 destination.secureMemory = hidl_handle(encryptedBuffer->handle());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800594 } else {
Robert Shih895fba92019-07-16 16:29:44 -0700595 destination.type = DrmBufferType::SHARED_MEMORY;
596 IMemoryToSharedBuffer(
597 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800598 }
Robert Shih895fba92019-07-16 16:29:44 -0700599 hardware::drm::V1_0::SharedBuffer source;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800600 encryptedBuffer->fillSourceBuffer(&source);
601 result = mCrypto->decrypt(
602 key, iv, mode, pattern, source, buffer->offset(),
603 subSamples, numSubSamples, destination, errorDetailMsg);
604 if (result < 0) {
Wonsik Kim557c88c2020-03-13 11:03:52 -0700605 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800606 return result;
607 }
Robert Shih895fba92019-07-16 16:29:44 -0700608 if (destination.type == DrmBufferType::SHARED_MEMORY) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800609 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
610 }
611 } else {
612 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
613 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
614 hidl_vec<SubSample> hidlSubSamples;
615 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
616
617 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
618 encryptedBuffer->fillSourceBuffer(&srcBuffer);
619
620 DestinationBuffer dstBuffer;
621 if (secure) {
622 dstBuffer.type = BufferType::NATIVE_HANDLE;
623 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
624 } else {
625 dstBuffer.type = BufferType::SHARED_MEMORY;
626 dstBuffer.nonsecureMemory = srcBuffer;
627 }
628
629 CasStatus status = CasStatus::OK;
630 hidl_string detailedError;
631 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
632
633 if (key != nullptr) {
634 sctrl = (ScramblingControl)key[0];
635 // Adjust for the PES offset
636 codecDataOffset = key[2] | (key[3] << 8);
637 }
638
639 auto returnVoid = mDescrambler->descramble(
640 sctrl,
641 hidlSubSamples,
642 srcBuffer,
643 0,
644 dstBuffer,
645 0,
646 [&status, &result, &detailedError] (
647 CasStatus _status, uint32_t _bytesWritten,
648 const hidl_string& _detailedError) {
649 status = _status;
650 result = (ssize_t)_bytesWritten;
651 detailedError = _detailedError;
652 });
653
654 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
655 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
656 mName, returnVoid.description().c_str(), status, result);
657 return UNKNOWN_ERROR;
658 }
659
660 if (result < codecDataOffset) {
661 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
662 return BAD_VALUE;
663 }
664
665 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
666
667 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
668 encryptedBuffer->copyDecryptedContentFromMemory(result);
669 }
670 }
671
672 buffer->setRange(codecDataOffset, result - codecDataOffset);
Sungtak Lee04b30352020-07-27 13:57:25 -0700673
674 return queueInputBufferInternal(buffer, block, bufferSize);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800675}
676
677void CCodecBufferChannel::feedInputBufferIfAvailable() {
678 QueueGuard guard(mSync);
679 if (!guard.isRunning()) {
680 ALOGV("[%s] We're not running --- no input buffer reported", mName);
681 return;
682 }
683 feedInputBufferIfAvailableInternal();
684}
685
686void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
Taehwan Kimda0517d2020-09-16 17:29:37 +0900687 if (mInputMetEos) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800688 return;
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700689 }
690 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700691 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700692 if (!output->buffers ||
693 output->buffers->hasPending() ||
Wonsik Kim0487b782020-10-28 11:45:50 -0700694 output->buffers->numActiveSlots() >= output->numSlots) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800695 return;
696 }
697 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700698 size_t numActiveSlots = 0;
699 while (!mPipelineWatcher.lock()->pipelineFull()) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800700 sp<MediaCodecBuffer> inBuffer;
701 size_t index;
702 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700703 Mutexed<Input>::Locked input(mInput);
Wonsik Kim0487b782020-10-28 11:45:50 -0700704 numActiveSlots = input->buffers->numActiveSlots();
705 if (numActiveSlots >= input->numSlots) {
706 break;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800707 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700708 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800709 ALOGV("[%s] no new buffer available", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800710 break;
711 }
712 }
713 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
714 mCallback->onInputBufferAvailable(index, inBuffer);
715 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700716 ALOGV("[%s] # active slots after feedInputBufferIfAvailable = %zu", mName, numActiveSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800717}
718
719status_t CCodecBufferChannel::renderOutputBuffer(
720 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800721 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800722 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800723 bool released = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800724 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700725 Mutexed<Output>::Locked output(mOutput);
726 if (output->buffers) {
727 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800728 }
729 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800730 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
731 // set to true.
732 sendOutputBuffers();
733 // input buffer feeding may have been gated by pending output buffers
734 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800735 if (!c2Buffer) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800736 if (released) {
Wonsik Kimf7529dd2019-04-18 17:35:53 -0700737 std::call_once(mRenderWarningFlag, [this] {
738 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
739 "timestamp or render=true with non-video buffers. Apps should "
740 "call releaseOutputBuffer() with render=false for those.",
741 mName);
742 });
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800743 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800744 return INVALID_OPERATION;
745 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800746
747#if 0
748 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
749 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
750 for (const std::shared_ptr<const C2Info> &info : infoParams) {
751 AString res;
752 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
753 if (ix) res.append(", ");
754 res.append(*((int32_t*)info.get() + (ix / 4)));
755 }
756 ALOGV(" [%s]", res.c_str());
757 }
758#endif
759 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
760 std::static_pointer_cast<const C2StreamRotationInfo::output>(
761 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
762 bool flip = rotation && (rotation->flip & 1);
763 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
Byeongjo Park2eef13e2020-06-12 17:24:21 +0900764
765 {
766 Mutexed<OutputSurface>::Locked output(mOutputSurface);
767 if (output->surface == nullptr) {
768 ALOGI("[%s] cannot render buffer without surface", mName);
769 return OK;
770 }
771 int64_t frameIndex;
772 buffer->meta()->findInt64("frameIndex", &frameIndex);
773 if (output->rotation.count(frameIndex) != 0) {
774 auto it = output->rotation.find(frameIndex);
775 quarters = (it->second / 90) & 3;
776 output->rotation.erase(it);
777 }
778 }
779
Pawin Vongmasa36653902018-11-15 00:10:25 -0800780 uint32_t transform = 0;
781 switch (quarters) {
782 case 0: // no rotation
783 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
784 break;
785 case 1: // 90 degrees counter-clockwise
786 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
787 : HAL_TRANSFORM_ROT_270;
788 break;
789 case 2: // 180 degrees
790 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
791 break;
792 case 3: // 90 degrees clockwise
793 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
794 : HAL_TRANSFORM_ROT_90;
795 break;
796 }
797
798 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
799 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
800 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
801 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
802 if (surfaceScaling) {
803 videoScalingMode = surfaceScaling->value;
804 }
805
806 // Use dataspace from format as it has the default aspects already applied
807 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
808 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
809
810 // HDR static info
811 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
812 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
813 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
814
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800815 // HDR10 plus info
816 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
817 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
818 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
Yichi Chen54be23c2020-06-15 14:30:53 +0800819 if (hdr10PlusInfo && hdr10PlusInfo->flexCount() == 0) {
820 hdr10PlusInfo.reset();
821 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800822
Pawin Vongmasa36653902018-11-15 00:10:25 -0800823 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
824 if (blocks.size() != 1u) {
825 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
826 return UNKNOWN_ERROR;
827 }
828 const C2ConstGraphicBlock &block = blocks.front();
829
830 // TODO: revisit this after C2Fence implementation.
831 android::IGraphicBufferProducer::QueueBufferInput qbi(
832 timestampNs,
833 false, // droppable
834 dataSpace,
835 Rect(blocks.front().crop().left,
836 blocks.front().crop().top,
837 blocks.front().crop().right(),
838 blocks.front().crop().bottom()),
839 videoScalingMode,
840 transform,
841 Fence::NO_FENCE, 0);
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800842 if (hdrStaticInfo || hdr10PlusInfo) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800843 HdrMetadata hdr;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800844 if (hdrStaticInfo) {
wenchangliuf3f92882020-05-14 00:02:01 +0800845 // If mastering max and min luminance fields are 0, do not use them.
846 // It indicates the value may not be present in the stream.
847 if (hdrStaticInfo->mastering.maxLuminance > 0.0f &&
848 hdrStaticInfo->mastering.minLuminance > 0.0f) {
849 struct android_smpte2086_metadata smpte2086_meta = {
850 .displayPrimaryRed = {
851 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
852 },
853 .displayPrimaryGreen = {
854 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
855 },
856 .displayPrimaryBlue = {
857 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
858 },
859 .whitePoint = {
860 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
861 },
862 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
863 .minLuminance = hdrStaticInfo->mastering.minLuminance,
864 };
Yichi Chen54be23c2020-06-15 14:30:53 +0800865 hdr.validTypes |= HdrMetadata::SMPTE2086;
wenchangliuf3f92882020-05-14 00:02:01 +0800866 hdr.smpte2086 = smpte2086_meta;
867 }
Chong Zhang3bb2a7f2020-04-21 10:35:12 -0700868 // If the content light level fields are 0, do not use them, it
869 // indicates the value may not be present in the stream.
870 if (hdrStaticInfo->maxCll > 0.0f && hdrStaticInfo->maxFall > 0.0f) {
871 struct android_cta861_3_metadata cta861_meta = {
872 .maxContentLightLevel = hdrStaticInfo->maxCll,
873 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
874 };
875 hdr.validTypes |= HdrMetadata::CTA861_3;
876 hdr.cta8613 = cta861_meta;
877 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800878 }
879 if (hdr10PlusInfo) {
880 hdr.validTypes |= HdrMetadata::HDR10PLUS;
881 hdr.hdr10plus.assign(
882 hdr10PlusInfo->m.value,
883 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
884 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800885 qbi.setHdrMetadata(hdr);
886 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800887 // we don't have dirty regions
888 qbi.setSurfaceDamage(Region::INVALID_REGION);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800889 android::IGraphicBufferProducer::QueueBufferOutput qbo;
890 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
891 if (result != OK) {
892 ALOGI("[%s] queueBuffer failed: %d", mName, result);
Sungtak Lee47c018a2020-11-07 01:02:49 -0800893 if (result == NO_INIT) {
894 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
895 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800896 return result;
897 }
898 ALOGV("[%s] queue buffer successful", mName);
899
900 int64_t mediaTimeUs = 0;
901 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
902 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
903
904 return OK;
905}
906
907status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
908 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
909 bool released = false;
910 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700911 Mutexed<Input>::Locked input(mInput);
912 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800913 released = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800914 }
915 }
916 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700917 Mutexed<Output>::Locked output(mOutput);
918 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800919 released = true;
920 }
921 }
922 if (released) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800923 sendOutputBuffers();
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800924 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800925 } else {
926 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
927 }
928 return OK;
929}
930
931void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
932 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700933 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800934
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700935 if (!input->buffers->isArrayMode()) {
936 input->buffers = input->buffers->toArrayMode(input->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800937 }
938
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700939 input->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800940}
941
942void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
943 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700944 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800945
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700946 if (!output->buffers->isArrayMode()) {
947 output->buffers = output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800948 }
949
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700950 output->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800951}
952
953status_t CCodecBufferChannel::start(
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800954 const sp<AMessage> &inputFormat,
955 const sp<AMessage> &outputFormat,
956 bool buffersBoundToCodec) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800957 C2StreamBufferTypeSetting::input iStreamFormat(0u);
958 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kim0379ae82020-11-24 15:01:33 -0800959 C2ComponentKindSetting kind;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800960 C2PortReorderBufferDepthTuning::output reorderDepth;
961 C2PortReorderKeySetting::output reorderKey;
Wonsik Kim078b58e2019-01-09 15:08:06 -0800962 C2PortActualDelayTuning::input inputDelay(0);
963 C2PortActualDelayTuning::output outputDelay(0);
964 C2ActualPipelineDelayTuning pipelineDelay(0);
Sungtak Lee04b30352020-07-27 13:57:25 -0700965 C2SecureModeTuning secureMode(C2Config::SM_UNPROTECTED);
Wonsik Kim078b58e2019-01-09 15:08:06 -0800966
Pawin Vongmasa36653902018-11-15 00:10:25 -0800967 c2_status_t err = mComponent->query(
968 {
969 &iStreamFormat,
970 &oStreamFormat,
Wonsik Kim0379ae82020-11-24 15:01:33 -0800971 &kind,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800972 &reorderDepth,
973 &reorderKey,
Wonsik Kim078b58e2019-01-09 15:08:06 -0800974 &inputDelay,
975 &pipelineDelay,
976 &outputDelay,
Sungtak Lee04b30352020-07-27 13:57:25 -0700977 &secureMode,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800978 },
979 {},
980 C2_DONT_BLOCK,
981 nullptr);
982 if (err == C2_BAD_INDEX) {
Wonsik Kim0379ae82020-11-24 15:01:33 -0800983 if (!iStreamFormat || !oStreamFormat || !kind) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800984 return UNKNOWN_ERROR;
985 }
986 } else if (err != C2_OK) {
987 return UNKNOWN_ERROR;
988 }
989
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -0800990 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
991 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
992 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
993
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700994 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
995 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
Wonsik Kim078b58e2019-01-09 15:08:06 -0800996
Pawin Vongmasa36653902018-11-15 00:10:25 -0800997 // TODO: get this from input format
998 bool secure = mComponent->getName().find(".secure") != std::string::npos;
999
Sungtak Lee04b30352020-07-27 13:57:25 -07001000 // secure mode is a static parameter (shall not change in the executing state)
1001 mSendEncryptedInfoBuffer = secureMode.value == C2Config::SM_READ_PROTECTED_WITH_ENCRYPTED;
1002
Pawin Vongmasa36653902018-11-15 00:10:25 -08001003 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001004 int poolMask = GetCodec2PoolMask();
1005 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001006
1007 if (inputFormat != nullptr) {
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001008 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
Wonsik Kim0379ae82020-11-24 15:01:33 -08001009 bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001010 C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
1011 API_REFLECTION |
1012 API_VALUES |
1013 API_CURRENT_VALUES |
1014 API_DEPENDENCY |
1015 API_SAME_INPUT_BUFFER);
Wonsik Kim0379ae82020-11-24 15:01:33 -08001016 C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
1017 C2StreamSampleRateInfo::input sampleRate(0u);
1018 C2StreamChannelCountInfo::input channelCount(0u);
1019 C2StreamPcmEncodingInfo::input pcmEncoding(0u);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001020 std::shared_ptr<C2BlockPool> pool;
1021 {
1022 Mutexed<BlockPools>::Locked pools(mBlockPools);
1023
1024 // set default allocator ID.
1025 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001026 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001027
1028 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
1029 // from component, create the input block pool with given ID. Otherwise, use default IDs.
1030 std::vector<std::unique_ptr<C2Param>> params;
Wonsik Kimffb889a2020-05-28 11:32:25 -07001031 C2ApiFeaturesSetting featuresSetting{apiFeatures};
Wonsik Kim0379ae82020-11-24 15:01:33 -08001032 std::vector<C2Param *> stackParams({&featuresSetting});
1033 if (audioEncoder) {
1034 stackParams.push_back(&encoderFrameSize);
1035 stackParams.push_back(&sampleRate);
1036 stackParams.push_back(&channelCount);
1037 stackParams.push_back(&pcmEncoding);
1038 } else {
1039 encoderFrameSize.invalidate();
1040 sampleRate.invalidate();
1041 channelCount.invalidate();
1042 pcmEncoding.invalidate();
1043 }
1044 err = mComponent->query(stackParams,
Pawin Vongmasa36653902018-11-15 00:10:25 -08001045 { C2PortAllocatorsTuning::input::PARAM_TYPE },
1046 C2_DONT_BLOCK,
1047 &params);
1048 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1049 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
1050 mName, params.size(), asString(err), err);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001051 } else if (params.size() == 1) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001052 C2PortAllocatorsTuning::input *inputAllocators =
1053 C2PortAllocatorsTuning::input::From(params[0].get());
1054 if (inputAllocators && inputAllocators->flexCount() > 0) {
1055 std::shared_ptr<C2Allocator> allocator;
1056 // verify allocator IDs and resolve default allocator
1057 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
1058 if (allocator) {
1059 pools->inputAllocatorId = allocator->getId();
1060 } else {
1061 ALOGD("[%s] component requested invalid input allocator ID %u",
1062 mName, inputAllocators->m.values[0]);
1063 }
1064 }
1065 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001066 if (featuresSetting) {
1067 apiFeatures = featuresSetting.value;
1068 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001069
1070 // TODO: use C2Component wrapper to associate this pool with ourselves
1071 if ((poolMask >> pools->inputAllocatorId) & 1) {
1072 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
1073 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
1074 mName, pools->inputAllocatorId,
1075 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1076 asString(err), err);
1077 } else {
1078 err = C2_NOT_FOUND;
1079 }
1080 if (err != C2_OK) {
1081 C2BlockPool::local_id_t inputPoolId =
1082 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1083 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
1084 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
1085 mName, (unsigned long long)inputPoolId,
1086 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1087 asString(err), err);
1088 if (err != C2_OK) {
1089 return NO_MEMORY;
1090 }
1091 }
1092 pools->inputPool = pool;
1093 }
1094
Wonsik Kim51051262018-11-28 13:59:05 -08001095 bool forceArrayMode = false;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001096 Mutexed<Input>::Locked input(mInput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001097 input->inputDelay = inputDelayValue;
1098 input->pipelineDelay = pipelineDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001099 input->numSlots = numInputSlots;
1100 input->extraBuffers.flush();
1101 input->numExtraSlots = 0u;
Wonsik Kim0379ae82020-11-24 15:01:33 -08001102 if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
1103 input->frameReassembler.init(
1104 pool,
1105 {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
1106 encoderFrameSize.value,
1107 sampleRate.value,
1108 channelCount.value,
1109 pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
1110 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001111 bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
1112 // For encrypted content, framework decrypts source buffer (ashmem) into
1113 // C2Buffers. Thus non-conforming codecs can process these.
Wonsik Kim0379ae82020-11-24 15:01:33 -08001114 if (!buffersBoundToCodec
1115 && !input->frameReassembler
1116 && (hasCryptoOrDescrambler() || conforming)) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001117 input->buffers.reset(new SlotInputBuffers(mName));
1118 } else if (graphic) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001119 if (mInputSurface) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001120 input->buffers.reset(new DummyInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001121 } else if (mMetaMode == MODE_ANW) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001122 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
Wonsik Kim1221fd12019-07-12 12:52:05 -07001123 // This is to ensure buffers do not get released prematurely.
1124 // TODO: handle this without going into array mode
1125 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001126 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001127 input->buffers.reset(new GraphicInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001128 }
1129 } else {
1130 if (hasCryptoOrDescrambler()) {
1131 int32_t capacity = kLinearBufferSize;
1132 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
1133 if ((size_t)capacity > kMaxLinearBufferSize) {
1134 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
1135 capacity = kMaxLinearBufferSize;
1136 }
1137 if (mDealer == nullptr) {
1138 mDealer = new MemoryDealer(
1139 align(capacity, MemoryDealer::getAllocationAlignment())
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001140 * (numInputSlots + 1),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001141 "EncryptedLinearInputBuffers");
1142 mDecryptDestination = mDealer->allocate((size_t)capacity);
1143 }
1144 if (mCrypto != nullptr && mHeapSeqNum < 0) {
Robert Shih895fba92019-07-16 16:29:44 -07001145 sp<HidlMemory> heap = fromHeap(mDealer->getMemoryHeap());
1146 mHeapSeqNum = mCrypto->setHeap(heap);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001147 } else {
1148 mHeapSeqNum = -1;
1149 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001150 input->buffers.reset(new EncryptedLinearInputBuffers(
Wonsik Kim078b58e2019-01-09 15:08:06 -08001151 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001152 numInputSlots, mName));
Wonsik Kim51051262018-11-28 13:59:05 -08001153 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001154 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001155 input->buffers.reset(new LinearInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001156 }
1157 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001158 input->buffers->setFormat(inputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001159
1160 if (err == C2_OK) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001161 input->buffers->setPool(pool);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001162 } else {
1163 // TODO: error
1164 }
Wonsik Kim51051262018-11-28 13:59:05 -08001165
1166 if (forceArrayMode) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001167 input->buffers = input->buffers->toArrayMode(numInputSlots);
Wonsik Kim51051262018-11-28 13:59:05 -08001168 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001169 }
1170
1171 if (outputFormat != nullptr) {
1172 sp<IGraphicBufferProducer> outputSurface;
1173 uint32_t outputGeneration;
1174 {
1175 Mutexed<OutputSurface>::Locked output(mOutputSurface);
Sungtak Leed7463d12019-09-04 16:01:00 -07001176 output->maxDequeueBuffers = numOutputSlots +
Sungtak Lee7a7b7422019-07-16 17:40:40 -07001177 reorderDepth.value + kRenderingDepth;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001178 outputSurface = output->surface ?
1179 output->surface->getIGraphicBufferProducer() : nullptr;
Wonsik Kimf5e5c832019-02-21 11:36:05 -08001180 if (outputSurface) {
1181 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1182 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001183 outputGeneration = output->generation;
1184 }
1185
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001186 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001187 C2BlockPool::local_id_t outputPoolId_;
David Stevensc3fbb282021-01-18 18:11:20 +09001188 C2BlockPool::local_id_t prevOutputPoolId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001189
1190 {
1191 Mutexed<BlockPools>::Locked pools(mBlockPools);
1192
David Stevensc3fbb282021-01-18 18:11:20 +09001193 prevOutputPoolId = pools->outputPoolId;
1194
Pawin Vongmasa36653902018-11-15 00:10:25 -08001195 // set default allocator ID.
1196 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001197 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001198
1199 // query C2PortAllocatorsTuning::output from component, or use default allocator if
1200 // unsuccessful.
1201 std::vector<std::unique_ptr<C2Param>> params;
1202 err = mComponent->query({ },
1203 { C2PortAllocatorsTuning::output::PARAM_TYPE },
1204 C2_DONT_BLOCK,
1205 &params);
1206 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1207 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
1208 mName, params.size(), asString(err), err);
1209 } else if (err == C2_OK && params.size() == 1) {
1210 C2PortAllocatorsTuning::output *outputAllocators =
1211 C2PortAllocatorsTuning::output::From(params[0].get());
1212 if (outputAllocators && outputAllocators->flexCount() > 0) {
1213 std::shared_ptr<C2Allocator> allocator;
1214 // verify allocator IDs and resolve default allocator
1215 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
1216 if (allocator) {
1217 pools->outputAllocatorId = allocator->getId();
1218 } else {
1219 ALOGD("[%s] component requested invalid output allocator ID %u",
1220 mName, outputAllocators->m.values[0]);
1221 }
1222 }
1223 }
1224
1225 // use bufferqueue if outputting to a surface.
1226 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1227 // if unsuccessful.
1228 if (outputSurface) {
1229 params.clear();
1230 err = mComponent->query({ },
1231 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1232 C2_DONT_BLOCK,
1233 &params);
1234 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1235 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1236 mName, params.size(), asString(err), err);
1237 } else if (err == C2_OK && params.size() == 1) {
1238 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1239 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1240 if (surfaceAllocator) {
1241 std::shared_ptr<C2Allocator> allocator;
1242 // verify allocator IDs and resolve default allocator
1243 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1244 if (allocator) {
1245 pools->outputAllocatorId = allocator->getId();
1246 } else {
1247 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1248 mName, surfaceAllocator->value);
1249 err = C2_BAD_VALUE;
1250 }
1251 }
1252 }
1253 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1254 && err != C2_OK
1255 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1256 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1257 }
1258 }
1259
1260 if ((poolMask >> pools->outputAllocatorId) & 1) {
1261 err = mComponent->createBlockPool(
1262 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1263 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1264 mName, pools->outputAllocatorId,
1265 (unsigned long long)pools->outputPoolId,
1266 asString(err));
1267 } else {
1268 err = C2_NOT_FOUND;
1269 }
1270 if (err != C2_OK) {
1271 // use basic pool instead
1272 pools->outputPoolId =
1273 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1274 }
1275
1276 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1277 // component.
1278 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1279 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1280
1281 std::vector<std::unique_ptr<C2SettingResult>> failures;
1282 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1283 ALOGD("[%s] Configured output block pool ids %llu => %s",
1284 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1285 outputPoolId_ = pools->outputPoolId;
1286 }
1287
David Stevensc3fbb282021-01-18 18:11:20 +09001288 if (prevOutputPoolId != C2BlockPool::BASIC_LINEAR
1289 && prevOutputPoolId != C2BlockPool::BASIC_GRAPHIC) {
1290 c2_status_t err = mComponent->destroyBlockPool(prevOutputPoolId);
1291 if (err != C2_OK) {
1292 ALOGW("Failed to clean up previous block pool %llu - %s (%d)\n",
1293 (unsigned long long) prevOutputPoolId, asString(err), err);
1294 }
1295 }
1296
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001297 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001298 output->outputDelay = outputDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001299 output->numSlots = numOutputSlots;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001300 if (graphic) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001301 if (outputSurface || !buffersBoundToCodec) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001302 output->buffers.reset(new GraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001303 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001304 output->buffers.reset(new RawGraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001305 }
1306 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001307 output->buffers.reset(new LinearOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001308 }
Wonsik Kime4716c02020-02-28 10:42:21 -08001309 output->buffers->setFormat(outputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001310
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001311 output->buffers->clearStash();
1312 if (reorderDepth) {
1313 output->buffers->setReorderDepth(reorderDepth.value);
1314 }
1315 if (reorderKey) {
1316 output->buffers->setReorderKey(reorderKey.value);
1317 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001318
1319 // Try to set output surface to created block pool if given.
1320 if (outputSurface) {
1321 mComponent->setOutputSurface(
1322 outputPoolId_,
1323 outputSurface,
1324 outputGeneration);
1325 }
1326
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001327 if (oStreamFormat.value == C2BufferData::LINEAR) {
Wonsik Kim58713302020-01-29 22:25:23 -08001328 if (buffersBoundToCodec) {
1329 // WORKAROUND: if we're using early CSD workaround we convert to
1330 // array mode, to appease apps assuming the output
1331 // buffers to be of the same size.
1332 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1333 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001334
1335 int32_t channelCount;
1336 int32_t sampleRate;
1337 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1338 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1339 int32_t delay = 0;
1340 int32_t padding = 0;;
1341 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1342 delay = 0;
1343 }
1344 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1345 padding = 0;
1346 }
1347 if (delay || padding) {
1348 // We need write access to the buffers, and we're already in
1349 // array mode.
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001350 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001351 }
1352 }
1353 }
1354 }
1355
1356 // Set up pipeline control. This has to be done after mInputBuffers and
1357 // mOutputBuffers are initialized to make sure that lingering callbacks
1358 // about buffers from the previous generation do not interfere with the
1359 // newly initialized pipeline capacity.
1360
Wonsik Kimab34ed62019-01-31 15:28:46 -08001361 {
1362 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001363 watcher->inputDelay(inputDelayValue)
1364 .pipelineDelay(pipelineDelayValue)
1365 .outputDelay(outputDelayValue)
Wonsik Kimab34ed62019-01-31 15:28:46 -08001366 .smoothnessFactor(kSmoothnessFactor);
1367 watcher->flush();
1368 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001369
1370 mInputMetEos = false;
1371 mSync.start();
1372 return OK;
1373}
1374
1375status_t CCodecBufferChannel::requestInitialInputBuffers() {
1376 if (mInputSurface) {
1377 return OK;
1378 }
1379
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001380 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001381 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
1382 c2_status_t err = mComponent->query({ &oStreamFormat, &prepend }, {}, C2_DONT_BLOCK, nullptr);
1383 if (err != C2_OK && err != C2_BAD_INDEX) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001384 return UNKNOWN_ERROR;
1385 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001386 size_t numInputSlots = mInput.lock()->numSlots;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001387
1388 struct ClientInputBuffer {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001389 size_t index;
1390 sp<MediaCodecBuffer> buffer;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001391 size_t capacity;
1392 };
1393 std::list<ClientInputBuffer> clientInputBuffers;
1394
1395 {
1396 Mutexed<Input>::Locked input(mInput);
1397 while (clientInputBuffers.size() < numInputSlots) {
1398 ClientInputBuffer clientInputBuffer;
1399 if (!input->buffers->requestNewBuffer(&clientInputBuffer.index,
1400 &clientInputBuffer.buffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001401 break;
1402 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001403 clientInputBuffer.capacity = clientInputBuffer.buffer->capacity();
1404 clientInputBuffers.emplace_back(std::move(clientInputBuffer));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001405 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001406 }
1407 if (clientInputBuffers.empty()) {
1408 ALOGW("[%s] start: cannot allocate memory at all", mName);
1409 return NO_MEMORY;
1410 } else if (clientInputBuffers.size() < numInputSlots) {
1411 ALOGD("[%s] start: cannot allocate memory for all slots, "
1412 "only %zu buffers allocated",
1413 mName, clientInputBuffers.size());
1414 } else {
1415 ALOGV("[%s] %zu initial input buffers available",
1416 mName, clientInputBuffers.size());
1417 }
1418 // Sort input buffers by their capacities in increasing order.
1419 clientInputBuffers.sort(
1420 [](const ClientInputBuffer& a, const ClientInputBuffer& b) {
1421 return a.capacity < b.capacity;
1422 });
1423
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001424 std::list<std::unique_ptr<C2Work>> flushedConfigs;
1425 mFlushedConfigs.lock()->swap(flushedConfigs);
1426 if (!flushedConfigs.empty()) {
1427 err = mComponent->queue(&flushedConfigs);
1428 if (err != C2_OK) {
1429 ALOGW("[%s] Error while queueing a flushed config", mName);
1430 return UNKNOWN_ERROR;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001431 }
1432 }
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001433 if (oStreamFormat.value == C2BufferData::LINEAR &&
1434 (!prepend || prepend.value == PREPEND_HEADER_TO_NONE)) {
1435 sp<MediaCodecBuffer> buffer = clientInputBuffers.front().buffer;
1436 // WORKAROUND: Some apps expect CSD available without queueing
1437 // any input. Queue an empty buffer to get the CSD.
1438 buffer->setRange(0, 0);
1439 buffer->meta()->clear();
1440 buffer->meta()->setInt64("timeUs", 0);
1441 if (queueInputBufferInternal(buffer) != OK) {
1442 ALOGW("[%s] Error while queueing an empty buffer to get CSD",
1443 mName);
1444 return UNKNOWN_ERROR;
1445 }
1446 clientInputBuffers.pop_front();
1447 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001448
1449 for (const ClientInputBuffer& clientInputBuffer: clientInputBuffers) {
1450 mCallback->onInputBufferAvailable(
1451 clientInputBuffer.index,
1452 clientInputBuffer.buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001453 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001454
Pawin Vongmasa36653902018-11-15 00:10:25 -08001455 return OK;
1456}
1457
1458void CCodecBufferChannel::stop() {
1459 mSync.stop();
1460 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
1461 if (mInputSurface != nullptr) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001462 mInputSurface.reset();
1463 }
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001464 mPipelineWatcher.lock()->flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001465}
1466
Wonsik Kim936a89c2020-05-08 16:07:50 -07001467void CCodecBufferChannel::reset() {
1468 stop();
1469 {
1470 Mutexed<Input>::Locked input(mInput);
1471 input->buffers.reset(new DummyInputBuffers(""));
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001472 input->extraBuffers.flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001473 }
1474 {
1475 Mutexed<Output>::Locked output(mOutput);
1476 output->buffers.reset();
1477 }
1478}
1479
1480void CCodecBufferChannel::release() {
1481 mComponent.reset();
1482 mInputAllocator.reset();
1483 mOutputSurface.lock()->surface.clear();
1484 {
1485 Mutexed<BlockPools>::Locked blockPools{mBlockPools};
1486 blockPools->inputPool.reset();
1487 blockPools->outputPoolIntf.reset();
1488 }
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001489 setCrypto(nullptr);
1490 setDescrambler(nullptr);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001491}
1492
1493
Pawin Vongmasa36653902018-11-15 00:10:25 -08001494void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1495 ALOGV("[%s] flush", mName);
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001496 std::list<std::unique_ptr<C2Work>> configs;
1497 for (const std::unique_ptr<C2Work> &work : flushedWork) {
1498 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1499 continue;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001500 }
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001501 if (work->input.buffers.empty()
1502 || work->input.buffers.front() == nullptr
1503 || work->input.buffers.front()->data().linearBlocks().empty()) {
1504 ALOGD("[%s] no linear codec config data found", mName);
1505 continue;
1506 }
1507 std::unique_ptr<C2Work> copy(new C2Work);
1508 copy->input.flags = C2FrameData::flags_t(work->input.flags | C2FrameData::FLAG_DROP_FRAME);
1509 copy->input.ordinal = work->input.ordinal;
1510 copy->input.buffers.insert(
1511 copy->input.buffers.begin(),
1512 work->input.buffers.begin(),
1513 work->input.buffers.end());
1514 for (const std::unique_ptr<C2Param> &param : work->input.configUpdate) {
1515 copy->input.configUpdate.push_back(C2Param::Copy(*param));
1516 }
1517 copy->input.infoBuffers.insert(
1518 copy->input.infoBuffers.begin(),
1519 work->input.infoBuffers.begin(),
1520 work->input.infoBuffers.end());
1521 copy->worklets.emplace_back(new C2Worklet);
1522 configs.push_back(std::move(copy));
1523 ALOGV("[%s] stashed flushed codec config data", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001524 }
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001525 mFlushedConfigs.lock()->swap(configs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001526 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001527 Mutexed<Input>::Locked input(mInput);
1528 input->buffers->flush();
1529 input->extraBuffers.flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001530 }
1531 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001532 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001533 if (output->buffers) {
1534 output->buffers->flush(flushedWork);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001535 output->buffers->flushStash();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001536 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001537 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001538 mPipelineWatcher.lock()->flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001539}
1540
1541void CCodecBufferChannel::onWorkDone(
1542 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
Wonsik Kimab34ed62019-01-31 15:28:46 -08001543 const C2StreamInitDataInfo::output *initData) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001544 if (handleWork(std::move(work), outputFormat, initData)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001545 feedInputBufferIfAvailable();
1546 }
1547}
1548
1549void CCodecBufferChannel::onInputBufferDone(
Wonsik Kimab34ed62019-01-31 15:28:46 -08001550 uint64_t frameIndex, size_t arrayIndex) {
Pawin Vongmasa8e2cfb52019-05-15 05:20:52 -07001551 if (mInputSurface) {
1552 return;
1553 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001554 std::shared_ptr<C2Buffer> buffer =
1555 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001556 bool newInputSlotAvailable;
1557 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001558 Mutexed<Input>::Locked input(mInput);
1559 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1560 if (!newInputSlotAvailable) {
1561 (void)input->extraBuffers.expireComponentBuffer(buffer);
1562 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001563 }
1564 if (newInputSlotAvailable) {
1565 feedInputBufferIfAvailable();
1566 }
1567}
1568
1569bool CCodecBufferChannel::handleWork(
1570 std::unique_ptr<C2Work> work,
1571 const sp<AMessage> &outputFormat,
1572 const C2StreamInitDataInfo::output *initData) {
Wonsik Kim936a89c2020-05-08 16:07:50 -07001573 {
Wonsik Kima4e049d2020-04-28 19:42:23 +00001574 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001575 if (!output->buffers) {
1576 return false;
1577 }
Wonsik Kime75a5da2020-02-14 17:29:03 -08001578 }
1579
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001580 // Whether the output buffer should be reported to the client or not.
1581 bool notifyClient = false;
1582
1583 if (work->result == C2_OK){
1584 notifyClient = true;
1585 } else if (work->result == C2_NOT_FOUND) {
1586 ALOGD("[%s] flushed work; ignored.", mName);
1587 } else {
1588 // C2_OK and C2_NOT_FOUND are the only results that we accept for processing
1589 // the config update.
1590 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1591 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1592 return false;
1593 }
1594
1595 if ((work->input.ordinal.frameIndex -
1596 mFirstValidFrameIndex.load()).peek() < 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001597 // Discard frames from previous generation.
1598 ALOGD("[%s] Discard frames from previous generation.", mName);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001599 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001600 }
1601
Wonsik Kim524b0582019-03-12 11:28:57 -07001602 if (mInputSurface == nullptr && (work->worklets.size() != 1u
Pawin Vongmasa36653902018-11-15 00:10:25 -08001603 || !work->worklets.front()
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001604 || !(work->worklets.front()->output.flags &
1605 C2FrameData::FLAG_INCOMPLETE))) {
1606 mPipelineWatcher.lock()->onWorkDone(
1607 work->input.ordinal.frameIndex.peeku());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001608 }
1609
1610 // NOTE: MediaCodec usage supposedly have only one worklet
1611 if (work->worklets.size() != 1u) {
1612 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1613 mName, work->worklets.size());
1614 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1615 return false;
1616 }
1617
1618 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1619
1620 std::shared_ptr<C2Buffer> buffer;
1621 // NOTE: MediaCodec usage supposedly have only one output stream.
1622 if (worklet->output.buffers.size() > 1u) {
1623 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1624 mName, worklet->output.buffers.size());
1625 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1626 return false;
1627 } else if (worklet->output.buffers.size() == 1u) {
1628 buffer = worklet->output.buffers[0];
1629 if (!buffer) {
1630 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1631 }
1632 }
1633
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001634 std::optional<uint32_t> newInputDelay, newPipelineDelay;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001635 bool needMaxDequeueBufferCountUpdate = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001636 while (!worklet->output.configUpdate.empty()) {
1637 std::unique_ptr<C2Param> param;
1638 worklet->output.configUpdate.back().swap(param);
1639 worklet->output.configUpdate.pop_back();
1640 switch (param->coreIndex().coreIndex()) {
1641 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1642 C2PortReorderBufferDepthTuning::output reorderDepth;
1643 if (reorderDepth.updateFrom(*param)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001644 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1645 mName, reorderDepth.value);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001646 mOutput.lock()->buffers->setReorderDepth(reorderDepth.value);
1647 needMaxDequeueBufferCountUpdate = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001648 } else {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001649 ALOGD("[%s] onWorkDone: failed to read reorder depth",
1650 mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001651 }
1652 break;
1653 }
1654 case C2PortReorderKeySetting::CORE_INDEX: {
1655 C2PortReorderKeySetting::output reorderKey;
1656 if (reorderKey.updateFrom(*param)) {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001657 mOutput.lock()->buffers->setReorderKey(reorderKey.value);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001658 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1659 mName, reorderKey.value);
1660 } else {
1661 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1662 }
1663 break;
1664 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001665 case C2PortActualDelayTuning::CORE_INDEX: {
1666 if (param->isGlobal()) {
1667 C2ActualPipelineDelayTuning pipelineDelay;
1668 if (pipelineDelay.updateFrom(*param)) {
1669 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1670 mName, pipelineDelay.value);
1671 newPipelineDelay = pipelineDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001672 (void)mPipelineWatcher.lock()->pipelineDelay(
1673 pipelineDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001674 }
1675 }
1676 if (param->forInput()) {
1677 C2PortActualDelayTuning::input inputDelay;
1678 if (inputDelay.updateFrom(*param)) {
1679 ALOGV("[%s] onWorkDone: updating input delay %u",
1680 mName, inputDelay.value);
1681 newInputDelay = inputDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001682 (void)mPipelineWatcher.lock()->inputDelay(
1683 inputDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001684 }
1685 }
1686 if (param->forOutput()) {
1687 C2PortActualDelayTuning::output outputDelay;
1688 if (outputDelay.updateFrom(*param)) {
1689 ALOGV("[%s] onWorkDone: updating output delay %u",
1690 mName, outputDelay.value);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001691 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
1692 needMaxDequeueBufferCountUpdate = true;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001693
1694 bool outputBuffersChanged = false;
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001695 size_t numOutputSlots = 0;
1696 {
1697 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001698 if (!output->buffers) {
1699 return false;
1700 }
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001701 output->outputDelay = outputDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001702 numOutputSlots = outputDelay.value +
1703 kSmoothnessFactor;
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001704 if (output->numSlots < numOutputSlots) {
1705 output->numSlots = numOutputSlots;
1706 if (output->buffers->isArrayMode()) {
1707 OutputBuffersArray *array =
1708 (OutputBuffersArray *)output->buffers.get();
1709 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
1710 mName, numOutputSlots);
1711 array->grow(numOutputSlots);
1712 outputBuffersChanged = true;
1713 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001714 }
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001715 numOutputSlots = output->numSlots;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001716 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001717
1718 if (outputBuffersChanged) {
1719 mCCodecCallback->onOutputBuffersChanged();
1720 }
1721 }
1722 }
1723 break;
1724 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001725 default:
1726 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
1727 mName, param->index());
1728 break;
1729 }
1730 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001731 if (newInputDelay || newPipelineDelay) {
1732 Mutexed<Input>::Locked input(mInput);
1733 size_t newNumSlots =
1734 newInputDelay.value_or(input->inputDelay) +
1735 newPipelineDelay.value_or(input->pipelineDelay) +
1736 kSmoothnessFactor;
1737 if (input->buffers->isArrayMode()) {
1738 if (input->numSlots >= newNumSlots) {
1739 input->numExtraSlots = 0;
1740 } else {
1741 input->numExtraSlots = newNumSlots - input->numSlots;
1742 }
1743 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
1744 mName, input->numExtraSlots);
1745 } else {
1746 input->numSlots = newNumSlots;
1747 }
1748 }
Wonsik Kim315e40a2020-09-09 14:11:50 -07001749 if (needMaxDequeueBufferCountUpdate) {
1750 size_t numOutputSlots = 0;
1751 uint32_t reorderDepth = 0;
1752 {
1753 Mutexed<Output>::Locked output(mOutput);
1754 numOutputSlots = output->numSlots;
1755 reorderDepth = output->buffers->getReorderDepth();
1756 }
1757 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1758 output->maxDequeueBuffers = numOutputSlots + reorderDepth + kRenderingDepth;
1759 if (output->surface) {
1760 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1761 }
1762 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001763
Pawin Vongmasa36653902018-11-15 00:10:25 -08001764 int32_t flags = 0;
1765 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
1766 flags |= MediaCodec::BUFFER_FLAG_EOS;
1767 ALOGV("[%s] onWorkDone: output EOS", mName);
1768 }
1769
Pawin Vongmasa36653902018-11-15 00:10:25 -08001770 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
1771 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
1772 // the codec input timestamp, but client output timestamp should (reported in timeUs)
1773 // shall correspond to the client input timesamp (in customOrdinal). By using the
1774 // delta between the two, this allows for some timestamp deviation - e.g. if one input
1775 // produces multiple output.
1776 c2_cntr64_t timestamp =
1777 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
1778 - work->input.ordinal.timestamp;
Wonsik Kim95ba0162019-03-19 15:51:54 -07001779 if (mInputSurface != nullptr) {
1780 // When using input surface we need to restore the original input timestamp.
1781 timestamp = work->input.ordinal.customOrdinal;
1782 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001783 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
1784 mName,
1785 work->input.ordinal.customOrdinal.peekll(),
1786 work->input.ordinal.timestamp.peekll(),
1787 worklet->output.ordinal.timestamp.peekll(),
1788 timestamp.peekll());
1789
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001790 // csd cannot be re-ordered and will always arrive first.
Pawin Vongmasa36653902018-11-15 00:10:25 -08001791 if (initData != nullptr) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001792 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001793 if (output->buffers && outputFormat) {
1794 output->buffers->updateSkipCutBuffer(outputFormat);
1795 output->buffers->setFormat(outputFormat);
1796 }
1797 if (!notifyClient) {
1798 return false;
1799 }
1800 size_t index;
1801 sp<MediaCodecBuffer> outBuffer;
Wonsik Kim936a89c2020-05-08 16:07:50 -07001802 if (output->buffers && output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001803 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
1804 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
1805 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
1806
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001807 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001808 mCallback->onOutputBufferAvailable(index, outBuffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001809 } else {
1810 ALOGD("[%s] onWorkDone: unable to register csd", mName);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001811 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001812 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001813 return false;
1814 }
1815 }
1816
ted.sunb8fe01e2020-06-23 14:03:41 +08001817 bool drop = false;
1818 if (worklet->output.flags & C2FrameData::FLAG_DROP_FRAME) {
1819 ALOGV("[%s] onWorkDone: drop buffer but keep metadata", mName);
1820 drop = true;
1821 }
1822
ted.sun04698a32020-06-23 14:03:41 +08001823 if (notifyClient && !buffer && !flags && !(drop && outputFormat)) {
Wonsik Kim35bf5732020-05-14 17:40:29 +00001824 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
Pawin Vongmasa36653902018-11-15 00:10:25 -08001825 mName, work->input.ordinal.frameIndex.peekull());
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001826 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001827 }
1828
1829 if (buffer) {
1830 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
1831 // TODO: properly translate these to metadata
1832 switch (info->coreIndex().coreIndex()) {
1833 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001834 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001835 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
1836 }
1837 break;
1838 default:
1839 break;
1840 }
1841 }
1842 }
1843
1844 {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001845 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimc23cc402020-05-28 14:53:40 -07001846 if (!output->buffers) {
1847 return false;
1848 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001849 output->buffers->pushToStash(
ted.sun04698a32020-06-23 14:03:41 +08001850 drop ? nullptr : buffer,
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001851 notifyClient,
1852 timestamp.peek(),
1853 flags,
1854 outputFormat,
1855 worklet->output.ordinal);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001856 }
1857 sendOutputBuffers();
1858 return true;
1859}
1860
1861void CCodecBufferChannel::sendOutputBuffers() {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001862 OutputBuffers::BufferAction action;
Wonsik Kima4e049d2020-04-28 19:42:23 +00001863 size_t index;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001864 sp<MediaCodecBuffer> outBuffer;
1865 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001866
1867 while (true) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001868 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001869 if (!output->buffers) {
1870 return;
1871 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001872 action = output->buffers->popFromStashAndRegister(
1873 &c2Buffer, &index, &outBuffer);
1874 switch (action) {
1875 case OutputBuffers::SKIP:
1876 return;
1877 case OutputBuffers::DISCARD:
1878 break;
1879 case OutputBuffers::NOTIFY_CLIENT:
Wonsik Kima4e049d2020-04-28 19:42:23 +00001880 output.unlock();
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001881 mCallback->onOutputBufferAvailable(index, outBuffer);
1882 break;
1883 case OutputBuffers::REALLOCATE:
1884 if (!output->buffers->isArrayMode()) {
1885 output->buffers =
1886 output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001887 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001888 static_cast<OutputBuffersArray*>(output->buffers.get())->
1889 realloc(c2Buffer);
1890 output.unlock();
1891 mCCodecCallback->onOutputBuffersChanged();
Wonsik Kim4ada73d2020-05-26 14:58:07 -07001892 break;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001893 case OutputBuffers::RETRY:
1894 ALOGV("[%s] sendOutputBuffers: unable to register output buffer",
1895 mName);
1896 return;
1897 default:
1898 LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: "
1899 "corrupted BufferAction value (%d) "
1900 "returned from popFromStashAndRegister.",
1901 mName, int(action));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001902 return;
1903 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001904 }
1905}
1906
1907status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
1908 static std::atomic_uint32_t surfaceGeneration{0};
1909 uint32_t generation = (getpid() << 10) |
1910 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
1911 & ((1 << 10) - 1));
1912
1913 sp<IGraphicBufferProducer> producer;
1914 if (newSurface) {
1915 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
Sungtak Leeab6f2f32019-02-15 14:43:51 -08001916 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
Sungtak Lee08515812019-06-05 11:16:32 -07001917 newSurface->setMaxDequeuedBufferCount(mOutputSurface.lock()->maxDequeueBuffers);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001918 producer = newSurface->getIGraphicBufferProducer();
1919 producer->setGenerationNumber(generation);
1920 } else {
1921 ALOGE("[%s] setting output surface to null", mName);
1922 return INVALID_OPERATION;
1923 }
1924
1925 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
1926 C2BlockPool::local_id_t outputPoolId;
1927 {
1928 Mutexed<BlockPools>::Locked pools(mBlockPools);
1929 outputPoolId = pools->outputPoolId;
1930 outputPoolIntf = pools->outputPoolIntf;
1931 }
1932
1933 if (outputPoolIntf) {
1934 if (mComponent->setOutputSurface(
1935 outputPoolId,
1936 producer,
1937 generation) != C2_OK) {
1938 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
1939 return INVALID_OPERATION;
1940 }
1941 }
1942
1943 {
1944 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1945 output->surface = newSurface;
1946 output->generation = generation;
1947 }
1948
1949 return OK;
1950}
1951
Wonsik Kimab34ed62019-01-31 15:28:46 -08001952PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001953 // When client pushed EOS, we want all the work to be done quickly.
1954 // Otherwise, component may have stalled work due to input starvation up to
1955 // the sum of the delay in the pipeline.
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001956 size_t n = 0;
1957 if (!mInputMetEos) {
1958 size_t outputDelay = mOutput.lock()->outputDelay;
1959 Mutexed<Input>::Locked input(mInput);
1960 n = input->inputDelay + input->pipelineDelay + outputDelay;
1961 }
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001962 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
Wonsik Kimab34ed62019-01-31 15:28:46 -08001963}
1964
Pawin Vongmasa36653902018-11-15 00:10:25 -08001965void CCodecBufferChannel::setMetaMode(MetaMode mode) {
1966 mMetaMode = mode;
1967}
1968
Wonsik Kim596187e2019-10-25 12:44:10 -07001969void CCodecBufferChannel::setCrypto(const sp<ICrypto> &crypto) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001970 if (mCrypto != nullptr) {
1971 for (std::pair<wp<HidlMemory>, int32_t> entry : mHeapSeqNumMap) {
1972 mCrypto->unsetHeap(entry.second);
1973 }
1974 mHeapSeqNumMap.clear();
1975 if (mHeapSeqNum >= 0) {
1976 mCrypto->unsetHeap(mHeapSeqNum);
1977 mHeapSeqNum = -1;
1978 }
1979 }
Wonsik Kim596187e2019-10-25 12:44:10 -07001980 mCrypto = crypto;
1981}
1982
1983void CCodecBufferChannel::setDescrambler(const sp<IDescrambler> &descrambler) {
1984 mDescrambler = descrambler;
1985}
1986
Pawin Vongmasa36653902018-11-15 00:10:25 -08001987status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
1988 // C2_OK is always translated to OK.
1989 if (c2s == C2_OK) {
1990 return OK;
1991 }
1992
1993 // Operation-dependent translation
1994 // TODO: Add as necessary
1995 switch (c2op) {
1996 case C2_OPERATION_Component_start:
1997 switch (c2s) {
1998 case C2_NO_MEMORY:
1999 return NO_MEMORY;
2000 default:
2001 return UNKNOWN_ERROR;
2002 }
2003 default:
2004 break;
2005 }
2006
2007 // Backup operation-agnostic translation
2008 switch (c2s) {
2009 case C2_BAD_INDEX:
2010 return BAD_INDEX;
2011 case C2_BAD_VALUE:
2012 return BAD_VALUE;
2013 case C2_BLOCKING:
2014 return WOULD_BLOCK;
2015 case C2_DUPLICATE:
2016 return ALREADY_EXISTS;
2017 case C2_NO_INIT:
2018 return NO_INIT;
2019 case C2_NO_MEMORY:
2020 return NO_MEMORY;
2021 case C2_NOT_FOUND:
2022 return NAME_NOT_FOUND;
2023 case C2_TIMED_OUT:
2024 return TIMED_OUT;
2025 case C2_BAD_STATE:
2026 case C2_CANCELED:
2027 case C2_CANNOT_DO:
2028 case C2_CORRUPTED:
2029 case C2_OMITTED:
2030 case C2_REFUSED:
2031 return UNKNOWN_ERROR;
2032 default:
2033 return -static_cast<status_t>(c2s);
2034 }
2035}
2036
2037} // namespace android