blob: 1b21df161d002aa42e26a21655cb405ef8e5b34d [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodecBufferChannel"
19#include <utils/Log.h>
20
21#include <numeric>
22
23#include <C2AllocatorGralloc.h>
24#include <C2PlatformSupport.h>
25#include <C2BlockInternal.h>
26#include <C2Config.h>
27#include <C2Debug.h>
28
29#include <android/hardware/cas/native/1.0/IDescrambler.h>
30#include <android-base/stringprintf.h>
31#include <binder/MemoryDealer.h>
32#include <gui/Surface.h>
33#include <media/openmax/OMX_Core.h>
34#include <media/stagefright/foundation/ABuffer.h>
35#include <media/stagefright/foundation/ALookup.h>
36#include <media/stagefright/foundation/AMessage.h>
37#include <media/stagefright/foundation/AUtils.h>
38#include <media/stagefright/foundation/hexdump.h>
39#include <media/stagefright/MediaCodec.h>
40#include <media/stagefright/MediaCodecConstants.h>
41#include <media/MediaCodecBuffer.h>
42#include <system/window.h>
43
44#include "CCodecBufferChannel.h"
45#include "Codec2Buffer.h"
46#include "SkipCutBuffer.h"
47
48namespace android {
49
50using android::base::StringPrintf;
51using hardware::hidl_handle;
52using hardware::hidl_string;
53using hardware::hidl_vec;
54using namespace hardware::cas::V1_0;
55using namespace hardware::cas::native::V1_0;
56
57using CasStatus = hardware::cas::V1_0::Status;
58
59/**
60 * Base class for representation of buffers at one port.
61 */
62class CCodecBufferChannel::Buffers {
63public:
64 Buffers(const char *componentName, const char *name = "Buffers")
65 : mComponentName(componentName),
66 mChannelName(std::string(componentName) + ":" + name),
67 mName(mChannelName.c_str()) {
68 }
69 virtual ~Buffers() = default;
70
71 /**
72 * Set format for MediaCodec-facing buffers.
73 */
74 void setFormat(const sp<AMessage> &format) {
75 CHECK(format != nullptr);
76 mFormat = format;
77 }
78
79 /**
80 * Return a copy of current format.
81 */
82 sp<AMessage> dupFormat() {
83 return mFormat != nullptr ? mFormat->dup() : nullptr;
84 }
85
86 /**
87 * Returns true if the buffers are operating under array mode.
88 */
89 virtual bool isArrayMode() const { return false; }
90
91 /**
92 * Fills the vector with MediaCodecBuffer's if in array mode; otherwise,
93 * no-op.
94 */
95 virtual void getArray(Vector<sp<MediaCodecBuffer>> *) const {}
96
Wonsik Kimdf5dd142019-02-06 10:15:46 -080097 /**
98 * Return number of buffers the client owns.
99 */
100 virtual size_t numClientBuffers() const = 0;
101
Wonsik Kimc3e5e942019-02-28 11:59:20 -0800102 void handleImageData(const sp<Codec2Buffer> &buffer) {
103 sp<ABuffer> imageDataCandidate = buffer->getImageData();
104 if (imageDataCandidate == nullptr) {
105 return;
106 }
107 sp<ABuffer> imageData;
108 if (!mFormat->findBuffer("image-data", &imageData)
109 || imageDataCandidate->size() != imageData->size()
110 || memcmp(imageDataCandidate->data(), imageData->data(), imageData->size()) != 0) {
111 ALOGD("[%s] updating image-data", mName);
112 sp<AMessage> newFormat = dupFormat();
113 newFormat->setBuffer("image-data", imageDataCandidate);
114 MediaImage2 *img = (MediaImage2*)imageDataCandidate->data();
115 if (img->mNumPlanes > 0 && img->mType != img->MEDIA_IMAGE_TYPE_UNKNOWN) {
116 int32_t stride = img->mPlane[0].mRowInc;
117 newFormat->setInt32(KEY_STRIDE, stride);
118 ALOGD("[%s] updating stride = %d", mName, stride);
119 if (img->mNumPlanes > 1 && stride > 0) {
120 int32_t vstride = (img->mPlane[1].mOffset - img->mPlane[0].mOffset) / stride;
121 newFormat->setInt32(KEY_SLICE_HEIGHT, vstride);
122 ALOGD("[%s] updating vstride = %d", mName, vstride);
123 }
124 }
125 setFormat(newFormat);
126 buffer->setFormat(newFormat);
127 }
128 }
129
Pawin Vongmasa36653902018-11-15 00:10:25 -0800130protected:
131 std::string mComponentName; ///< name of component for debugging
132 std::string mChannelName; ///< name of channel for debugging
133 const char *mName; ///< C-string version of channel name
134 // Format to be used for creating MediaCodec-facing buffers.
135 sp<AMessage> mFormat;
136
137private:
138 DISALLOW_EVIL_CONSTRUCTORS(Buffers);
139};
140
141class CCodecBufferChannel::InputBuffers : public CCodecBufferChannel::Buffers {
142public:
143 InputBuffers(const char *componentName, const char *name = "Input[]")
144 : Buffers(componentName, name) { }
145 virtual ~InputBuffers() = default;
146
147 /**
148 * Set a block pool to obtain input memory blocks.
149 */
150 void setPool(const std::shared_ptr<C2BlockPool> &pool) { mPool = pool; }
151
152 /**
153 * Get a new MediaCodecBuffer for input and its corresponding index.
154 * Returns false if no new buffer can be obtained at the moment.
155 */
156 virtual bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) = 0;
157
158 /**
159 * Release the buffer obtained from requestNewBuffer() and get the
160 * associated C2Buffer object back. Returns true if the buffer was on file
161 * and released successfully.
162 */
163 virtual bool releaseBuffer(
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800164 const sp<MediaCodecBuffer> &buffer,
165 std::shared_ptr<C2Buffer> *c2buffer,
166 bool release) = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800167
168 /**
169 * Release the buffer that is no longer used by the codec process. Return
170 * true if and only if the buffer was on file and released successfully.
171 */
172 virtual bool expireComponentBuffer(
173 const std::shared_ptr<C2Buffer> &c2buffer) = 0;
174
175 /**
176 * Flush internal state. After this call, no index or buffer previously
177 * returned from requestNewBuffer() is valid.
178 */
179 virtual void flush() = 0;
180
181 /**
182 * Return array-backed version of input buffers. The returned object
183 * shall retain the internal state so that it will honor index and
184 * buffer from previous calls of requestNewBuffer().
185 */
186 virtual std::unique_ptr<InputBuffers> toArrayMode(size_t size) = 0;
187
188protected:
189 // Pool to obtain blocks for input buffers.
190 std::shared_ptr<C2BlockPool> mPool;
191
192private:
193 DISALLOW_EVIL_CONSTRUCTORS(InputBuffers);
194};
195
196class CCodecBufferChannel::OutputBuffers : public CCodecBufferChannel::Buffers {
197public:
198 OutputBuffers(const char *componentName, const char *name = "Output")
199 : Buffers(componentName, name) { }
200 virtual ~OutputBuffers() = default;
201
202 /**
203 * Register output C2Buffer from the component and obtain corresponding
204 * index and MediaCodecBuffer object. Returns false if registration
205 * fails.
206 */
207 virtual status_t registerBuffer(
208 const std::shared_ptr<C2Buffer> &buffer,
209 size_t *index,
210 sp<MediaCodecBuffer> *clientBuffer) = 0;
211
212 /**
213 * Register codec specific data as a buffer to be consistent with
214 * MediaCodec behavior.
215 */
216 virtual status_t registerCsd(
Lajos Molnar3bb81cd2019-02-20 15:10:30 -0800217 const C2StreamInitDataInfo::output * /* csd */,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800218 size_t * /* index */,
219 sp<MediaCodecBuffer> * /* clientBuffer */) = 0;
220
221 /**
222 * Release the buffer obtained from registerBuffer() and get the
223 * associated C2Buffer object back. Returns true if the buffer was on file
224 * and released successfully.
225 */
226 virtual bool releaseBuffer(
227 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) = 0;
228
229 /**
230 * Flush internal state. After this call, no index or buffer previously
231 * returned from registerBuffer() is valid.
232 */
233 virtual void flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) = 0;
234
235 /**
236 * Return array-backed version of output buffers. The returned object
237 * shall retain the internal state so that it will honor index and
238 * buffer from previous calls of registerBuffer().
239 */
240 virtual std::unique_ptr<OutputBuffers> toArrayMode(size_t size) = 0;
241
242 /**
243 * Initialize SkipCutBuffer object.
244 */
245 void initSkipCutBuffer(
246 int32_t delay, int32_t padding, int32_t sampleRate, int32_t channelCount) {
247 CHECK(mSkipCutBuffer == nullptr);
248 mDelay = delay;
249 mPadding = padding;
250 mSampleRate = sampleRate;
251 setSkipCutBuffer(delay, padding, channelCount);
252 }
253
254 /**
255 * Update the SkipCutBuffer object. No-op if it's never initialized.
256 */
257 void updateSkipCutBuffer(int32_t sampleRate, int32_t channelCount) {
258 if (mSkipCutBuffer == nullptr) {
259 return;
260 }
261 int32_t delay = mDelay;
262 int32_t padding = mPadding;
263 if (sampleRate != mSampleRate) {
264 delay = ((int64_t)delay * sampleRate) / mSampleRate;
265 padding = ((int64_t)padding * sampleRate) / mSampleRate;
266 }
267 setSkipCutBuffer(delay, padding, channelCount);
268 }
269
270 /**
271 * Submit buffer to SkipCutBuffer object, if initialized.
272 */
273 void submit(const sp<MediaCodecBuffer> &buffer) {
274 if (mSkipCutBuffer != nullptr) {
275 mSkipCutBuffer->submit(buffer);
276 }
277 }
278
279 /**
280 * Transfer SkipCutBuffer object to the other Buffers object.
281 */
282 void transferSkipCutBuffer(const sp<SkipCutBuffer> &scb) {
283 mSkipCutBuffer = scb;
284 }
285
286protected:
287 sp<SkipCutBuffer> mSkipCutBuffer;
288
289private:
290 int32_t mDelay;
291 int32_t mPadding;
292 int32_t mSampleRate;
293
294 void setSkipCutBuffer(int32_t skip, int32_t cut, int32_t channelCount) {
295 if (mSkipCutBuffer != nullptr) {
296 size_t prevSize = mSkipCutBuffer->size();
297 if (prevSize != 0u) {
298 ALOGD("[%s] Replacing SkipCutBuffer holding %zu bytes", mName, prevSize);
299 }
300 }
301 mSkipCutBuffer = new SkipCutBuffer(skip, cut, channelCount);
302 }
303
304 DISALLOW_EVIL_CONSTRUCTORS(OutputBuffers);
305};
306
307namespace {
308
Wonsik Kim078b58e2019-01-09 15:08:06 -0800309const static size_t kSmoothnessFactor = 4;
310const static size_t kRenderingDepth = 3;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800311const static size_t kLinearBufferSize = 1048576;
312// This can fit 4K RGBA frame, and most likely client won't need more than this.
313const static size_t kMaxLinearBufferSize = 3840 * 2160 * 4;
314
315/**
316 * Simple local buffer pool backed by std::vector.
317 */
318class LocalBufferPool : public std::enable_shared_from_this<LocalBufferPool> {
319public:
320 /**
321 * Create a new LocalBufferPool object.
322 *
323 * \param poolCapacity max total size of buffers managed by this pool.
324 *
325 * \return a newly created pool object.
326 */
327 static std::shared_ptr<LocalBufferPool> Create(size_t poolCapacity) {
328 return std::shared_ptr<LocalBufferPool>(new LocalBufferPool(poolCapacity));
329 }
330
331 /**
332 * Return an ABuffer object whose size is at least |capacity|.
333 *
334 * \param capacity requested capacity
335 * \return nullptr if the pool capacity is reached
336 * an ABuffer object otherwise.
337 */
338 sp<ABuffer> newBuffer(size_t capacity) {
339 Mutex::Autolock lock(mMutex);
340 auto it = std::find_if(
341 mPool.begin(), mPool.end(),
342 [capacity](const std::vector<uint8_t> &vec) {
343 return vec.capacity() >= capacity;
344 });
345 if (it != mPool.end()) {
346 sp<ABuffer> buffer = new VectorBuffer(std::move(*it), shared_from_this());
347 mPool.erase(it);
348 return buffer;
349 }
350 if (mUsedSize + capacity > mPoolCapacity) {
351 while (!mPool.empty()) {
352 mUsedSize -= mPool.back().capacity();
353 mPool.pop_back();
354 }
355 if (mUsedSize + capacity > mPoolCapacity) {
356 ALOGD("mUsedSize = %zu, capacity = %zu, mPoolCapacity = %zu",
357 mUsedSize, capacity, mPoolCapacity);
358 return nullptr;
359 }
360 }
361 std::vector<uint8_t> vec(capacity);
362 mUsedSize += vec.capacity();
363 return new VectorBuffer(std::move(vec), shared_from_this());
364 }
365
366private:
367 /**
368 * ABuffer backed by std::vector.
369 */
370 class VectorBuffer : public ::android::ABuffer {
371 public:
372 /**
373 * Construct a VectorBuffer by taking the ownership of supplied vector.
374 *
375 * \param vec backing vector of the buffer. this object takes
376 * ownership at construction.
377 * \param pool a LocalBufferPool object to return the vector at
378 * destruction.
379 */
380 VectorBuffer(std::vector<uint8_t> &&vec, const std::shared_ptr<LocalBufferPool> &pool)
381 : ABuffer(vec.data(), vec.capacity()),
382 mVec(std::move(vec)),
383 mPool(pool) {
384 }
385
386 ~VectorBuffer() override {
387 std::shared_ptr<LocalBufferPool> pool = mPool.lock();
388 if (pool) {
389 // If pool is alive, return the vector back to the pool so that
390 // it can be recycled.
391 pool->returnVector(std::move(mVec));
392 }
393 }
394
395 private:
396 std::vector<uint8_t> mVec;
397 std::weak_ptr<LocalBufferPool> mPool;
398 };
399
400 Mutex mMutex;
401 size_t mPoolCapacity;
402 size_t mUsedSize;
403 std::list<std::vector<uint8_t>> mPool;
404
405 /**
406 * Private constructor to prevent constructing non-managed LocalBufferPool.
407 */
408 explicit LocalBufferPool(size_t poolCapacity)
409 : mPoolCapacity(poolCapacity), mUsedSize(0) {
410 }
411
412 /**
413 * Take back the ownership of vec from the destructed VectorBuffer and put
414 * it in front of the pool.
415 */
416 void returnVector(std::vector<uint8_t> &&vec) {
417 Mutex::Autolock lock(mMutex);
418 mPool.push_front(std::move(vec));
419 }
420
421 DISALLOW_EVIL_CONSTRUCTORS(LocalBufferPool);
422};
423
424sp<GraphicBlockBuffer> AllocateGraphicBuffer(
425 const std::shared_ptr<C2BlockPool> &pool,
426 const sp<AMessage> &format,
427 uint32_t pixelFormat,
428 const C2MemoryUsage &usage,
429 const std::shared_ptr<LocalBufferPool> &localBufferPool) {
430 int32_t width, height;
431 if (!format->findInt32("width", &width) || !format->findInt32("height", &height)) {
432 ALOGD("format lacks width or height");
433 return nullptr;
434 }
435
436 std::shared_ptr<C2GraphicBlock> block;
437 c2_status_t err = pool->fetchGraphicBlock(
438 width, height, pixelFormat, usage, &block);
439 if (err != C2_OK) {
440 ALOGD("fetch graphic block failed: %d", err);
441 return nullptr;
442 }
443
444 return GraphicBlockBuffer::Allocate(
445 format,
446 block,
447 [localBufferPool](size_t capacity) {
448 return localBufferPool->newBuffer(capacity);
449 });
450}
451
452class BuffersArrayImpl;
453
454/**
455 * Flexible buffer slots implementation.
456 */
457class FlexBuffersImpl {
458public:
459 FlexBuffersImpl(const char *name)
460 : mImplName(std::string(name) + ".Impl"),
461 mName(mImplName.c_str()) { }
462
463 /**
464 * Assign an empty slot for a buffer and return the index. If there's no
465 * empty slot, just add one at the end and return it.
466 *
467 * \param buffer[in] a new buffer to assign a slot.
468 * \return index of the assigned slot.
469 */
470 size_t assignSlot(const sp<Codec2Buffer> &buffer) {
471 for (size_t i = 0; i < mBuffers.size(); ++i) {
472 if (mBuffers[i].clientBuffer == nullptr
473 && mBuffers[i].compBuffer.expired()) {
474 mBuffers[i].clientBuffer = buffer;
475 return i;
476 }
477 }
478 mBuffers.push_back({ buffer, std::weak_ptr<C2Buffer>() });
479 return mBuffers.size() - 1;
480 }
481
482 /**
483 * Release the slot from the client, and get the C2Buffer object back from
484 * the previously assigned buffer. Note that the slot is not completely free
485 * until the returned C2Buffer object is freed.
486 *
487 * \param buffer[in] the buffer previously assigned a slot.
488 * \param c2buffer[in,out] pointer to C2Buffer to be populated. Ignored
489 * if null.
490 * \return true if the buffer is successfully released from a slot
491 * false otherwise
492 */
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800493 bool releaseSlot(
494 const sp<MediaCodecBuffer> &buffer,
495 std::shared_ptr<C2Buffer> *c2buffer,
496 bool release) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800497 sp<Codec2Buffer> clientBuffer;
498 size_t index = mBuffers.size();
499 for (size_t i = 0; i < mBuffers.size(); ++i) {
500 if (mBuffers[i].clientBuffer == buffer) {
501 clientBuffer = mBuffers[i].clientBuffer;
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800502 if (release) {
503 mBuffers[i].clientBuffer.clear();
504 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800505 index = i;
506 break;
507 }
508 }
509 if (clientBuffer == nullptr) {
510 ALOGV("[%s] %s: No matching buffer found", mName, __func__);
511 return false;
512 }
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800513 std::shared_ptr<C2Buffer> result = mBuffers[index].compBuffer.lock();
514 if (!result) {
515 result = clientBuffer->asC2Buffer();
516 mBuffers[index].compBuffer = result;
517 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800518 if (c2buffer) {
519 *c2buffer = result;
520 }
521 return true;
522 }
523
524 bool expireComponentBuffer(const std::shared_ptr<C2Buffer> &c2buffer) {
525 for (size_t i = 0; i < mBuffers.size(); ++i) {
526 std::shared_ptr<C2Buffer> compBuffer =
527 mBuffers[i].compBuffer.lock();
528 if (!compBuffer || compBuffer != c2buffer) {
529 continue;
530 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800531 mBuffers[i].compBuffer.reset();
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800532 ALOGV("[%s] codec released buffer #%zu", mName, i);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800533 return true;
534 }
535 ALOGV("[%s] codec released an unknown buffer", mName);
536 return false;
537 }
538
539 void flush() {
540 ALOGV("[%s] buffers are flushed %zu", mName, mBuffers.size());
541 mBuffers.clear();
542 }
543
Wonsik Kimab34ed62019-01-31 15:28:46 -0800544 size_t numClientBuffers() const {
545 return std::count_if(
546 mBuffers.begin(), mBuffers.end(),
547 [](const Entry &entry) {
548 return (entry.clientBuffer != nullptr);
549 });
550 }
551
Pawin Vongmasa36653902018-11-15 00:10:25 -0800552private:
553 friend class BuffersArrayImpl;
554
555 std::string mImplName; ///< name for debugging
556 const char *mName; ///< C-string version of name
557
558 struct Entry {
559 sp<Codec2Buffer> clientBuffer;
560 std::weak_ptr<C2Buffer> compBuffer;
561 };
562 std::vector<Entry> mBuffers;
563};
564
565/**
566 * Static buffer slots implementation based on a fixed-size array.
567 */
568class BuffersArrayImpl {
569public:
570 BuffersArrayImpl()
571 : mImplName("BuffersArrayImpl"),
572 mName(mImplName.c_str()) { }
573
574 /**
575 * Initialize buffer array from the original |impl|. The buffers known by
576 * the client is preserved, and the empty slots are populated so that the
577 * array size is at least |minSize|.
578 *
579 * \param impl[in] FlexBuffersImpl object used so far.
580 * \param minSize[in] minimum size of the buffer array.
581 * \param allocate[in] function to allocate a client buffer for an empty slot.
582 */
583 void initialize(
584 const FlexBuffersImpl &impl,
585 size_t minSize,
586 std::function<sp<Codec2Buffer>()> allocate) {
587 mImplName = impl.mImplName + "[N]";
588 mName = mImplName.c_str();
589 for (size_t i = 0; i < impl.mBuffers.size(); ++i) {
590 sp<Codec2Buffer> clientBuffer = impl.mBuffers[i].clientBuffer;
591 bool ownedByClient = (clientBuffer != nullptr);
592 if (!ownedByClient) {
593 clientBuffer = allocate();
594 }
595 mBuffers.push_back({ clientBuffer, impl.mBuffers[i].compBuffer, ownedByClient });
596 }
597 ALOGV("[%s] converted %zu buffers to array mode of %zu", mName, mBuffers.size(), minSize);
598 for (size_t i = impl.mBuffers.size(); i < minSize; ++i) {
599 mBuffers.push_back({ allocate(), std::weak_ptr<C2Buffer>(), false });
600 }
601 }
602
603 /**
604 * Grab a buffer from the underlying array which matches the criteria.
605 *
606 * \param index[out] index of the slot.
607 * \param buffer[out] the matching buffer.
608 * \param match[in] a function to test whether the buffer matches the
609 * criteria or not.
610 * \return OK if successful,
611 * WOULD_BLOCK if slots are being used,
612 * NO_MEMORY if no slot matches the criteria, even though it's
613 * available
614 */
615 status_t grabBuffer(
616 size_t *index,
617 sp<Codec2Buffer> *buffer,
618 std::function<bool(const sp<Codec2Buffer> &)> match =
619 [](const sp<Codec2Buffer> &) { return true; }) {
620 // allBuffersDontMatch remains true if all buffers are available but
621 // match() returns false for every buffer.
622 bool allBuffersDontMatch = true;
623 for (size_t i = 0; i < mBuffers.size(); ++i) {
624 if (!mBuffers[i].ownedByClient && mBuffers[i].compBuffer.expired()) {
625 if (match(mBuffers[i].clientBuffer)) {
626 mBuffers[i].ownedByClient = true;
627 *buffer = mBuffers[i].clientBuffer;
628 (*buffer)->meta()->clear();
629 (*buffer)->setRange(0, (*buffer)->capacity());
630 *index = i;
631 return OK;
632 }
633 } else {
634 allBuffersDontMatch = false;
635 }
636 }
637 return allBuffersDontMatch ? NO_MEMORY : WOULD_BLOCK;
638 }
639
640 /**
641 * Return the buffer from the client, and get the C2Buffer object back from
642 * the buffer. Note that the slot is not completely free until the returned
643 * C2Buffer object is freed.
644 *
645 * \param buffer[in] the buffer previously grabbed.
646 * \param c2buffer[in,out] pointer to C2Buffer to be populated. Ignored
647 * if null.
648 * \return true if the buffer is successfully returned
649 * false otherwise
650 */
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800651 bool returnBuffer(
652 const sp<MediaCodecBuffer> &buffer,
653 std::shared_ptr<C2Buffer> *c2buffer,
654 bool release) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800655 sp<Codec2Buffer> clientBuffer;
656 size_t index = mBuffers.size();
657 for (size_t i = 0; i < mBuffers.size(); ++i) {
658 if (mBuffers[i].clientBuffer == buffer) {
659 if (!mBuffers[i].ownedByClient) {
660 ALOGD("[%s] Client returned a buffer it does not own according to our record: %zu", mName, i);
661 }
662 clientBuffer = mBuffers[i].clientBuffer;
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800663 if (release) {
664 mBuffers[i].ownedByClient = false;
665 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800666 index = i;
667 break;
668 }
669 }
670 if (clientBuffer == nullptr) {
671 ALOGV("[%s] %s: No matching buffer found", mName, __func__);
672 return false;
673 }
674 ALOGV("[%s] %s: matching buffer found (index=%zu)", mName, __func__, index);
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800675 std::shared_ptr<C2Buffer> result = mBuffers[index].compBuffer.lock();
676 if (!result) {
677 result = clientBuffer->asC2Buffer();
678 mBuffers[index].compBuffer = result;
679 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800680 if (c2buffer) {
681 *c2buffer = result;
682 }
683 return true;
684 }
685
686 bool expireComponentBuffer(const std::shared_ptr<C2Buffer> &c2buffer) {
687 for (size_t i = 0; i < mBuffers.size(); ++i) {
688 std::shared_ptr<C2Buffer> compBuffer =
689 mBuffers[i].compBuffer.lock();
690 if (!compBuffer) {
691 continue;
692 }
693 if (c2buffer == compBuffer) {
694 if (mBuffers[i].ownedByClient) {
695 // This should not happen.
696 ALOGD("[%s] codec released a buffer owned by client "
697 "(index %zu)", mName, i);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800698 }
699 mBuffers[i].compBuffer.reset();
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800700 ALOGV("[%s] codec released buffer #%zu(array mode)", mName, i);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800701 return true;
702 }
703 }
704 ALOGV("[%s] codec released an unknown buffer (array mode)", mName);
705 return false;
706 }
707
708 /**
709 * Populate |array| with the underlying buffer array.
710 *
711 * \param array[out] an array to be filled with the underlying buffer array.
712 */
713 void getArray(Vector<sp<MediaCodecBuffer>> *array) const {
714 array->clear();
715 for (const Entry &entry : mBuffers) {
716 array->push(entry.clientBuffer);
717 }
718 }
719
720 /**
721 * The client abandoned all known buffers, so reclaim the ownership.
722 */
723 void flush() {
724 for (Entry &entry : mBuffers) {
725 entry.ownedByClient = false;
726 }
727 }
728
729 void realloc(std::function<sp<Codec2Buffer>()> alloc) {
730 size_t size = mBuffers.size();
731 mBuffers.clear();
732 for (size_t i = 0; i < size; ++i) {
733 mBuffers.push_back({ alloc(), std::weak_ptr<C2Buffer>(), false });
734 }
735 }
736
Wonsik Kimab34ed62019-01-31 15:28:46 -0800737 size_t numClientBuffers() const {
738 return std::count_if(
739 mBuffers.begin(), mBuffers.end(),
740 [](const Entry &entry) {
741 return entry.ownedByClient;
742 });
743 }
744
Pawin Vongmasa36653902018-11-15 00:10:25 -0800745private:
746 std::string mImplName; ///< name for debugging
747 const char *mName; ///< C-string version of name
748
749 struct Entry {
750 const sp<Codec2Buffer> clientBuffer;
751 std::weak_ptr<C2Buffer> compBuffer;
752 bool ownedByClient;
753 };
754 std::vector<Entry> mBuffers;
755};
756
757class InputBuffersArray : public CCodecBufferChannel::InputBuffers {
758public:
759 InputBuffersArray(const char *componentName, const char *name = "Input[N]")
760 : InputBuffers(componentName, name) { }
761 ~InputBuffersArray() override = default;
762
763 void initialize(
764 const FlexBuffersImpl &impl,
765 size_t minSize,
766 std::function<sp<Codec2Buffer>()> allocate) {
767 mImpl.initialize(impl, minSize, allocate);
768 }
769
770 bool isArrayMode() const final { return true; }
771
772 std::unique_ptr<CCodecBufferChannel::InputBuffers> toArrayMode(
773 size_t) final {
774 return nullptr;
775 }
776
777 void getArray(Vector<sp<MediaCodecBuffer>> *array) const final {
778 mImpl.getArray(array);
779 }
780
781 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override {
782 sp<Codec2Buffer> c2Buffer;
783 status_t err = mImpl.grabBuffer(index, &c2Buffer);
784 if (err == OK) {
785 c2Buffer->setFormat(mFormat);
Wonsik Kimc3e5e942019-02-28 11:59:20 -0800786 handleImageData(c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800787 *buffer = c2Buffer;
788 return true;
789 }
790 return false;
791 }
792
793 bool releaseBuffer(
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800794 const sp<MediaCodecBuffer> &buffer,
795 std::shared_ptr<C2Buffer> *c2buffer,
796 bool release) override {
797 return mImpl.returnBuffer(buffer, c2buffer, release);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800798 }
799
800 bool expireComponentBuffer(
801 const std::shared_ptr<C2Buffer> &c2buffer) override {
802 return mImpl.expireComponentBuffer(c2buffer);
803 }
804
805 void flush() override {
806 mImpl.flush();
807 }
808
Wonsik Kimab34ed62019-01-31 15:28:46 -0800809 size_t numClientBuffers() const final {
810 return mImpl.numClientBuffers();
811 }
812
Pawin Vongmasa36653902018-11-15 00:10:25 -0800813private:
814 BuffersArrayImpl mImpl;
815};
816
817class LinearInputBuffers : public CCodecBufferChannel::InputBuffers {
818public:
819 LinearInputBuffers(const char *componentName, const char *name = "1D-Input")
820 : InputBuffers(componentName, name),
821 mImpl(mName) { }
822
823 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override {
824 int32_t capacity = kLinearBufferSize;
825 (void)mFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
826 if ((size_t)capacity > kMaxLinearBufferSize) {
827 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
828 capacity = kMaxLinearBufferSize;
829 }
830 // TODO: proper max input size
831 // TODO: read usage from intf
832 sp<Codec2Buffer> newBuffer = alloc((size_t)capacity);
833 if (newBuffer == nullptr) {
834 return false;
835 }
836 *index = mImpl.assignSlot(newBuffer);
837 *buffer = newBuffer;
838 return true;
839 }
840
841 bool releaseBuffer(
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800842 const sp<MediaCodecBuffer> &buffer,
843 std::shared_ptr<C2Buffer> *c2buffer,
844 bool release) override {
845 return mImpl.releaseSlot(buffer, c2buffer, release);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800846 }
847
848 bool expireComponentBuffer(
849 const std::shared_ptr<C2Buffer> &c2buffer) override {
850 return mImpl.expireComponentBuffer(c2buffer);
851 }
852
853 void flush() override {
854 // This is no-op by default unless we're in array mode where we need to keep
855 // track of the flushed work.
856 mImpl.flush();
857 }
858
859 std::unique_ptr<CCodecBufferChannel::InputBuffers> toArrayMode(
860 size_t size) final {
861 int32_t capacity = kLinearBufferSize;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800862 (void)mFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
863 if ((size_t)capacity > kMaxLinearBufferSize) {
864 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
865 capacity = kMaxLinearBufferSize;
866 }
867 // TODO: proper max input size
868 // TODO: read usage from intf
Pawin Vongmasa36653902018-11-15 00:10:25 -0800869 std::unique_ptr<InputBuffersArray> array(
870 new InputBuffersArray(mComponentName.c_str(), "1D-Input[N]"));
871 array->setPool(mPool);
872 array->setFormat(mFormat);
873 array->initialize(
874 mImpl,
875 size,
876 [this, capacity] () -> sp<Codec2Buffer> { return alloc(capacity); });
877 return std::move(array);
878 }
879
Wonsik Kimab34ed62019-01-31 15:28:46 -0800880 size_t numClientBuffers() const final {
881 return mImpl.numClientBuffers();
882 }
883
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800884 virtual sp<Codec2Buffer> alloc(size_t size) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800885 C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
886 std::shared_ptr<C2LinearBlock> block;
887
888 c2_status_t err = mPool->fetchLinearBlock(size, usage, &block);
889 if (err != C2_OK) {
890 return nullptr;
891 }
892
893 return LinearBlockBuffer::Allocate(mFormat, block);
894 }
895
896private:
897 FlexBuffersImpl mImpl;
898};
899
900class EncryptedLinearInputBuffers : public LinearInputBuffers {
901public:
902 EncryptedLinearInputBuffers(
903 bool secure,
904 const sp<MemoryDealer> &dealer,
905 const sp<ICrypto> &crypto,
906 int32_t heapSeqNum,
907 size_t capacity,
Wonsik Kim078b58e2019-01-09 15:08:06 -0800908 size_t numInputSlots,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800909 const char *componentName, const char *name = "EncryptedInput")
910 : LinearInputBuffers(componentName, name),
911 mUsage({0, 0}),
912 mDealer(dealer),
913 mCrypto(crypto),
914 mHeapSeqNum(heapSeqNum) {
915 if (secure) {
916 mUsage = { C2MemoryUsage::READ_PROTECTED, 0 };
917 } else {
918 mUsage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
919 }
Wonsik Kim078b58e2019-01-09 15:08:06 -0800920 for (size_t i = 0; i < numInputSlots; ++i) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800921 sp<IMemory> memory = mDealer->allocate(capacity);
922 if (memory == nullptr) {
923 ALOGD("[%s] Failed to allocate memory from dealer: only %zu slots allocated", mName, i);
924 break;
925 }
926 mMemoryVector.push_back({std::weak_ptr<C2LinearBlock>(), memory});
927 }
928 }
929
930 ~EncryptedLinearInputBuffers() override {
931 }
932
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800933 sp<Codec2Buffer> alloc(size_t size) override {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800934 sp<IMemory> memory;
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800935 size_t slot = 0;
936 for (; slot < mMemoryVector.size(); ++slot) {
937 if (mMemoryVector[slot].block.expired()) {
938 memory = mMemoryVector[slot].memory;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800939 break;
940 }
941 }
942 if (memory == nullptr) {
943 return nullptr;
944 }
945
946 std::shared_ptr<C2LinearBlock> block;
947 c2_status_t err = mPool->fetchLinearBlock(size, mUsage, &block);
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800948 if (err != C2_OK || block == nullptr) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800949 return nullptr;
950 }
951
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800952 mMemoryVector[slot].block = block;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800953 return new EncryptedLinearBlockBuffer(mFormat, block, memory, mHeapSeqNum);
954 }
955
956private:
957 C2MemoryUsage mUsage;
958 sp<MemoryDealer> mDealer;
959 sp<ICrypto> mCrypto;
960 int32_t mHeapSeqNum;
961 struct Entry {
962 std::weak_ptr<C2LinearBlock> block;
963 sp<IMemory> memory;
964 };
965 std::vector<Entry> mMemoryVector;
966};
967
968class GraphicMetadataInputBuffers : public CCodecBufferChannel::InputBuffers {
969public:
970 GraphicMetadataInputBuffers(const char *componentName, const char *name = "2D-MetaInput")
971 : InputBuffers(componentName, name),
972 mImpl(mName),
973 mStore(GetCodec2PlatformAllocatorStore()) { }
974 ~GraphicMetadataInputBuffers() override = default;
975
976 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override {
977 std::shared_ptr<C2Allocator> alloc;
978 c2_status_t err = mStore->fetchAllocator(mPool->getAllocatorId(), &alloc);
979 if (err != C2_OK) {
980 return false;
981 }
982 sp<GraphicMetadataBuffer> newBuffer = new GraphicMetadataBuffer(mFormat, alloc);
983 if (newBuffer == nullptr) {
984 return false;
985 }
986 *index = mImpl.assignSlot(newBuffer);
987 *buffer = newBuffer;
988 return true;
989 }
990
991 bool releaseBuffer(
Pawin Vongmasa1f213362019-01-24 06:59:16 -0800992 const sp<MediaCodecBuffer> &buffer,
993 std::shared_ptr<C2Buffer> *c2buffer,
994 bool release) override {
995 return mImpl.releaseSlot(buffer, c2buffer, release);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800996 }
997
998 bool expireComponentBuffer(
999 const std::shared_ptr<C2Buffer> &c2buffer) override {
1000 return mImpl.expireComponentBuffer(c2buffer);
1001 }
1002
1003 void flush() override {
1004 // This is no-op by default unless we're in array mode where we need to keep
1005 // track of the flushed work.
1006 }
1007
1008 std::unique_ptr<CCodecBufferChannel::InputBuffers> toArrayMode(
1009 size_t size) final {
1010 std::shared_ptr<C2Allocator> alloc;
1011 c2_status_t err = mStore->fetchAllocator(mPool->getAllocatorId(), &alloc);
1012 if (err != C2_OK) {
1013 return nullptr;
1014 }
1015 std::unique_ptr<InputBuffersArray> array(
1016 new InputBuffersArray(mComponentName.c_str(), "2D-MetaInput[N]"));
1017 array->setPool(mPool);
1018 array->setFormat(mFormat);
1019 array->initialize(
1020 mImpl,
1021 size,
1022 [format = mFormat, alloc]() -> sp<Codec2Buffer> {
1023 return new GraphicMetadataBuffer(format, alloc);
1024 });
1025 return std::move(array);
1026 }
1027
Wonsik Kimab34ed62019-01-31 15:28:46 -08001028 size_t numClientBuffers() const final {
1029 return mImpl.numClientBuffers();
1030 }
1031
Pawin Vongmasa36653902018-11-15 00:10:25 -08001032private:
1033 FlexBuffersImpl mImpl;
1034 std::shared_ptr<C2AllocatorStore> mStore;
1035};
1036
1037class GraphicInputBuffers : public CCodecBufferChannel::InputBuffers {
1038public:
Wonsik Kim078b58e2019-01-09 15:08:06 -08001039 GraphicInputBuffers(
1040 size_t numInputSlots, const char *componentName, const char *name = "2D-BB-Input")
Pawin Vongmasa36653902018-11-15 00:10:25 -08001041 : InputBuffers(componentName, name),
1042 mImpl(mName),
1043 mLocalBufferPool(LocalBufferPool::Create(
Wonsik Kim078b58e2019-01-09 15:08:06 -08001044 kMaxLinearBufferSize * numInputSlots)) { }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001045 ~GraphicInputBuffers() override = default;
1046
1047 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override {
1048 // TODO: proper max input size
1049 // TODO: read usage from intf
1050 C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
1051 sp<GraphicBlockBuffer> newBuffer = AllocateGraphicBuffer(
1052 mPool, mFormat, HAL_PIXEL_FORMAT_YV12, usage, mLocalBufferPool);
1053 if (newBuffer == nullptr) {
1054 return false;
1055 }
1056 *index = mImpl.assignSlot(newBuffer);
Wonsik Kimc3e5e942019-02-28 11:59:20 -08001057 handleImageData(newBuffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001058 *buffer = newBuffer;
1059 return true;
1060 }
1061
1062 bool releaseBuffer(
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001063 const sp<MediaCodecBuffer> &buffer,
1064 std::shared_ptr<C2Buffer> *c2buffer,
1065 bool release) override {
1066 return mImpl.releaseSlot(buffer, c2buffer, release);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001067 }
1068
1069 bool expireComponentBuffer(
1070 const std::shared_ptr<C2Buffer> &c2buffer) override {
1071 return mImpl.expireComponentBuffer(c2buffer);
1072 }
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001073
Pawin Vongmasa36653902018-11-15 00:10:25 -08001074 void flush() override {
1075 // This is no-op by default unless we're in array mode where we need to keep
1076 // track of the flushed work.
1077 }
1078
1079 std::unique_ptr<CCodecBufferChannel::InputBuffers> toArrayMode(
1080 size_t size) final {
1081 std::unique_ptr<InputBuffersArray> array(
1082 new InputBuffersArray(mComponentName.c_str(), "2D-BB-Input[N]"));
1083 array->setPool(mPool);
1084 array->setFormat(mFormat);
1085 array->initialize(
1086 mImpl,
1087 size,
1088 [pool = mPool, format = mFormat, lbp = mLocalBufferPool]() -> sp<Codec2Buffer> {
1089 C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
1090 return AllocateGraphicBuffer(
1091 pool, format, HAL_PIXEL_FORMAT_YV12, usage, lbp);
1092 });
1093 return std::move(array);
1094 }
1095
Wonsik Kimab34ed62019-01-31 15:28:46 -08001096 size_t numClientBuffers() const final {
1097 return mImpl.numClientBuffers();
1098 }
1099
Pawin Vongmasa36653902018-11-15 00:10:25 -08001100private:
1101 FlexBuffersImpl mImpl;
1102 std::shared_ptr<LocalBufferPool> mLocalBufferPool;
1103};
1104
1105class DummyInputBuffers : public CCodecBufferChannel::InputBuffers {
1106public:
1107 DummyInputBuffers(const char *componentName, const char *name = "2D-Input")
1108 : InputBuffers(componentName, name) { }
1109
1110 bool requestNewBuffer(size_t *, sp<MediaCodecBuffer> *) override {
1111 return false;
1112 }
1113
1114 bool releaseBuffer(
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001115 const sp<MediaCodecBuffer> &, std::shared_ptr<C2Buffer> *, bool) override {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001116 return false;
1117 }
1118
1119 bool expireComponentBuffer(const std::shared_ptr<C2Buffer> &) override {
1120 return false;
1121 }
1122 void flush() override {
1123 }
1124
1125 std::unique_ptr<CCodecBufferChannel::InputBuffers> toArrayMode(
1126 size_t) final {
1127 return nullptr;
1128 }
1129
1130 bool isArrayMode() const final { return true; }
1131
1132 void getArray(Vector<sp<MediaCodecBuffer>> *array) const final {
1133 array->clear();
1134 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001135
1136 size_t numClientBuffers() const final {
1137 return 0u;
1138 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001139};
1140
1141class OutputBuffersArray : public CCodecBufferChannel::OutputBuffers {
1142public:
1143 OutputBuffersArray(const char *componentName, const char *name = "Output[N]")
1144 : OutputBuffers(componentName, name) { }
1145 ~OutputBuffersArray() override = default;
1146
1147 void initialize(
1148 const FlexBuffersImpl &impl,
1149 size_t minSize,
1150 std::function<sp<Codec2Buffer>()> allocate) {
1151 mImpl.initialize(impl, minSize, allocate);
1152 }
1153
1154 bool isArrayMode() const final { return true; }
1155
1156 std::unique_ptr<CCodecBufferChannel::OutputBuffers> toArrayMode(
1157 size_t) final {
1158 return nullptr;
1159 }
1160
1161 status_t registerBuffer(
1162 const std::shared_ptr<C2Buffer> &buffer,
1163 size_t *index,
1164 sp<MediaCodecBuffer> *clientBuffer) final {
1165 sp<Codec2Buffer> c2Buffer;
1166 status_t err = mImpl.grabBuffer(
1167 index,
1168 &c2Buffer,
1169 [buffer](const sp<Codec2Buffer> &clientBuffer) {
1170 return clientBuffer->canCopy(buffer);
1171 });
1172 if (err == WOULD_BLOCK) {
1173 ALOGV("[%s] buffers temporarily not available", mName);
1174 return err;
1175 } else if (err != OK) {
1176 ALOGD("[%s] grabBuffer failed: %d", mName, err);
1177 return err;
1178 }
1179 c2Buffer->setFormat(mFormat);
1180 if (!c2Buffer->copy(buffer)) {
1181 ALOGD("[%s] copy buffer failed", mName);
1182 return WOULD_BLOCK;
1183 }
1184 submit(c2Buffer);
Wonsik Kimc48ddcf2019-02-11 16:16:57 -08001185 handleImageData(c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001186 *clientBuffer = c2Buffer;
1187 ALOGV("[%s] grabbed buffer %zu", mName, *index);
1188 return OK;
1189 }
1190
1191 status_t registerCsd(
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001192 const C2StreamInitDataInfo::output *csd,
Pawin Vongmasa36653902018-11-15 00:10:25 -08001193 size_t *index,
1194 sp<MediaCodecBuffer> *clientBuffer) final {
1195 sp<Codec2Buffer> c2Buffer;
1196 status_t err = mImpl.grabBuffer(
1197 index,
1198 &c2Buffer,
1199 [csd](const sp<Codec2Buffer> &clientBuffer) {
1200 return clientBuffer->base() != nullptr
1201 && clientBuffer->capacity() >= csd->flexCount();
1202 });
1203 if (err != OK) {
1204 return err;
1205 }
1206 memcpy(c2Buffer->base(), csd->m.value, csd->flexCount());
1207 c2Buffer->setRange(0, csd->flexCount());
1208 c2Buffer->setFormat(mFormat);
1209 *clientBuffer = c2Buffer;
1210 return OK;
1211 }
1212
1213 bool releaseBuffer(
1214 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) override {
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001215 return mImpl.returnBuffer(buffer, c2buffer, true);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001216 }
1217
1218 void flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) override {
1219 (void)flushedWork;
1220 mImpl.flush();
1221 if (mSkipCutBuffer != nullptr) {
1222 mSkipCutBuffer->clear();
1223 }
1224 }
1225
1226 void getArray(Vector<sp<MediaCodecBuffer>> *array) const final {
1227 mImpl.getArray(array);
1228 }
1229
1230 void realloc(const std::shared_ptr<C2Buffer> &c2buffer) {
1231 std::function<sp<Codec2Buffer>()> alloc;
1232 switch (c2buffer->data().type()) {
1233 case C2BufferData::LINEAR: {
1234 uint32_t size = kLinearBufferSize;
1235 const C2ConstLinearBlock &block = c2buffer->data().linearBlocks().front();
1236 if (block.size() < kMaxLinearBufferSize / 2) {
1237 size = block.size() * 2;
1238 } else {
1239 size = kMaxLinearBufferSize;
1240 }
1241 alloc = [format = mFormat, size] {
1242 return new LocalLinearBuffer(format, new ABuffer(size));
1243 };
1244 break;
1245 }
1246
1247 // TODO: add support
1248 case C2BufferData::GRAPHIC: FALLTHROUGH_INTENDED;
1249
1250 case C2BufferData::INVALID: FALLTHROUGH_INTENDED;
1251 case C2BufferData::LINEAR_CHUNKS: FALLTHROUGH_INTENDED;
1252 case C2BufferData::GRAPHIC_CHUNKS: FALLTHROUGH_INTENDED;
1253 default:
1254 ALOGD("Unsupported type: %d", (int)c2buffer->data().type());
1255 return;
1256 }
1257 mImpl.realloc(alloc);
1258 }
1259
Wonsik Kimdf5dd142019-02-06 10:15:46 -08001260 size_t numClientBuffers() const final {
1261 return mImpl.numClientBuffers();
1262 }
1263
Pawin Vongmasa36653902018-11-15 00:10:25 -08001264private:
1265 BuffersArrayImpl mImpl;
1266};
1267
1268class FlexOutputBuffers : public CCodecBufferChannel::OutputBuffers {
1269public:
1270 FlexOutputBuffers(const char *componentName, const char *name = "Output[]")
1271 : OutputBuffers(componentName, name),
1272 mImpl(mName) { }
1273
1274 status_t registerBuffer(
1275 const std::shared_ptr<C2Buffer> &buffer,
1276 size_t *index,
1277 sp<MediaCodecBuffer> *clientBuffer) override {
1278 sp<Codec2Buffer> newBuffer = wrap(buffer);
Wonsik Kim186fdbf2019-01-29 13:30:01 -08001279 if (newBuffer == nullptr) {
1280 return NO_MEMORY;
1281 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001282 newBuffer->setFormat(mFormat);
1283 *index = mImpl.assignSlot(newBuffer);
Wonsik Kimc48ddcf2019-02-11 16:16:57 -08001284 handleImageData(newBuffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001285 *clientBuffer = newBuffer;
1286 ALOGV("[%s] registered buffer %zu", mName, *index);
1287 return OK;
1288 }
1289
1290 status_t registerCsd(
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001291 const C2StreamInitDataInfo::output *csd,
Pawin Vongmasa36653902018-11-15 00:10:25 -08001292 size_t *index,
1293 sp<MediaCodecBuffer> *clientBuffer) final {
1294 sp<Codec2Buffer> newBuffer = new LocalLinearBuffer(
1295 mFormat, ABuffer::CreateAsCopy(csd->m.value, csd->flexCount()));
1296 *index = mImpl.assignSlot(newBuffer);
1297 *clientBuffer = newBuffer;
1298 return OK;
1299 }
1300
1301 bool releaseBuffer(
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001302 const sp<MediaCodecBuffer> &buffer,
1303 std::shared_ptr<C2Buffer> *c2buffer) override {
1304 return mImpl.releaseSlot(buffer, c2buffer, true);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001305 }
1306
1307 void flush(
1308 const std::list<std::unique_ptr<C2Work>> &flushedWork) override {
1309 (void) flushedWork;
1310 // This is no-op by default unless we're in array mode where we need to keep
1311 // track of the flushed work.
1312 }
1313
1314 std::unique_ptr<CCodecBufferChannel::OutputBuffers> toArrayMode(
1315 size_t size) override {
1316 std::unique_ptr<OutputBuffersArray> array(new OutputBuffersArray(mComponentName.c_str()));
1317 array->setFormat(mFormat);
1318 array->transferSkipCutBuffer(mSkipCutBuffer);
1319 array->initialize(
1320 mImpl,
1321 size,
1322 [this]() { return allocateArrayBuffer(); });
1323 return std::move(array);
1324 }
1325
Wonsik Kimdf5dd142019-02-06 10:15:46 -08001326 size_t numClientBuffers() const final {
1327 return mImpl.numClientBuffers();
1328 }
1329
Pawin Vongmasa36653902018-11-15 00:10:25 -08001330 /**
1331 * Return an appropriate Codec2Buffer object for the type of buffers.
1332 *
1333 * \param buffer C2Buffer object to wrap.
1334 *
1335 * \return appropriate Codec2Buffer object to wrap |buffer|.
1336 */
1337 virtual sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) = 0;
1338
1339 /**
1340 * Return an appropriate Codec2Buffer object for the type of buffers, to be
1341 * used as an empty array buffer.
1342 *
1343 * \return appropriate Codec2Buffer object which can copy() from C2Buffers.
1344 */
1345 virtual sp<Codec2Buffer> allocateArrayBuffer() = 0;
1346
1347private:
1348 FlexBuffersImpl mImpl;
1349};
1350
1351class LinearOutputBuffers : public FlexOutputBuffers {
1352public:
1353 LinearOutputBuffers(const char *componentName, const char *name = "1D-Output")
1354 : FlexOutputBuffers(componentName, name) { }
1355
1356 void flush(
1357 const std::list<std::unique_ptr<C2Work>> &flushedWork) override {
1358 if (mSkipCutBuffer != nullptr) {
1359 mSkipCutBuffer->clear();
1360 }
1361 FlexOutputBuffers::flush(flushedWork);
1362 }
1363
1364 sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) override {
1365 if (buffer == nullptr) {
1366 ALOGV("[%s] using a dummy buffer", mName);
1367 return new LocalLinearBuffer(mFormat, new ABuffer(0));
1368 }
1369 if (buffer->data().type() != C2BufferData::LINEAR) {
1370 ALOGV("[%s] non-linear buffer %d", mName, buffer->data().type());
1371 // We expect linear output buffers from the component.
1372 return nullptr;
1373 }
1374 if (buffer->data().linearBlocks().size() != 1u) {
1375 ALOGV("[%s] no linear buffers", mName);
1376 // We expect one and only one linear block from the component.
1377 return nullptr;
1378 }
1379 sp<Codec2Buffer> clientBuffer = ConstLinearBlockBuffer::Allocate(mFormat, buffer);
Wonsik Kim186fdbf2019-01-29 13:30:01 -08001380 if (clientBuffer == nullptr) {
1381 ALOGD("[%s] ConstLinearBlockBuffer::Allocate failed", mName);
1382 return nullptr;
1383 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001384 submit(clientBuffer);
1385 return clientBuffer;
1386 }
1387
1388 sp<Codec2Buffer> allocateArrayBuffer() override {
1389 // TODO: proper max output size
1390 return new LocalLinearBuffer(mFormat, new ABuffer(kLinearBufferSize));
1391 }
1392};
1393
1394class GraphicOutputBuffers : public FlexOutputBuffers {
1395public:
1396 GraphicOutputBuffers(const char *componentName, const char *name = "2D-Output")
1397 : FlexOutputBuffers(componentName, name) { }
1398
1399 sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) override {
1400 return new DummyContainerBuffer(mFormat, buffer);
1401 }
1402
1403 sp<Codec2Buffer> allocateArrayBuffer() override {
1404 return new DummyContainerBuffer(mFormat);
1405 }
1406};
1407
1408class RawGraphicOutputBuffers : public FlexOutputBuffers {
1409public:
Wonsik Kim078b58e2019-01-09 15:08:06 -08001410 RawGraphicOutputBuffers(
1411 size_t numOutputSlots, const char *componentName, const char *name = "2D-BB-Output")
Pawin Vongmasa36653902018-11-15 00:10:25 -08001412 : FlexOutputBuffers(componentName, name),
1413 mLocalBufferPool(LocalBufferPool::Create(
Wonsik Kim078b58e2019-01-09 15:08:06 -08001414 kMaxLinearBufferSize * numOutputSlots)) { }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001415 ~RawGraphicOutputBuffers() override = default;
1416
1417 sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) override {
1418 if (buffer == nullptr) {
1419 sp<Codec2Buffer> c2buffer = ConstGraphicBlockBuffer::AllocateEmpty(
1420 mFormat,
1421 [lbp = mLocalBufferPool](size_t capacity) {
1422 return lbp->newBuffer(capacity);
1423 });
Wonsik Kim186fdbf2019-01-29 13:30:01 -08001424 if (c2buffer == nullptr) {
1425 ALOGD("[%s] ConstGraphicBlockBuffer::AllocateEmpty failed", mName);
1426 return nullptr;
1427 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001428 c2buffer->setRange(0, 0);
1429 return c2buffer;
1430 } else {
1431 return ConstGraphicBlockBuffer::Allocate(
1432 mFormat,
1433 buffer,
1434 [lbp = mLocalBufferPool](size_t capacity) {
1435 return lbp->newBuffer(capacity);
1436 });
1437 }
1438 }
1439
1440 sp<Codec2Buffer> allocateArrayBuffer() override {
1441 return ConstGraphicBlockBuffer::AllocateEmpty(
1442 mFormat,
1443 [lbp = mLocalBufferPool](size_t capacity) {
1444 return lbp->newBuffer(capacity);
1445 });
1446 }
1447
1448private:
1449 std::shared_ptr<LocalBufferPool> mLocalBufferPool;
1450};
1451
1452} // namespace
1453
1454CCodecBufferChannel::QueueGuard::QueueGuard(
1455 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
1456 Mutex::Autolock l(mSync.mGuardLock);
1457 // At this point it's guaranteed that mSync is not under state transition,
1458 // as we are holding its mutex.
1459
1460 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
1461 if (count->value == -1) {
1462 mRunning = false;
1463 } else {
1464 ++count->value;
1465 mRunning = true;
1466 }
1467}
1468
1469CCodecBufferChannel::QueueGuard::~QueueGuard() {
1470 if (mRunning) {
1471 // We are not holding mGuardLock at this point so that QueueSync::stop() can
1472 // keep holding the lock until mCount reaches zero.
1473 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
1474 --count->value;
1475 count->cond.broadcast();
1476 }
1477}
1478
1479void CCodecBufferChannel::QueueSync::start() {
1480 Mutex::Autolock l(mGuardLock);
1481 // If stopped, it goes to running state; otherwise no-op.
1482 Mutexed<Counter>::Locked count(mCount);
1483 if (count->value == -1) {
1484 count->value = 0;
1485 }
1486}
1487
1488void CCodecBufferChannel::QueueSync::stop() {
1489 Mutex::Autolock l(mGuardLock);
1490 Mutexed<Counter>::Locked count(mCount);
1491 if (count->value == -1) {
1492 // no-op
1493 return;
1494 }
1495 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
1496 // mCount can only decrement. In other words, threads that acquired the lock
1497 // are allowed to finish execution but additional threads trying to acquire
1498 // the lock at this point will block, and then get QueueGuard at STOPPED
1499 // state.
1500 while (count->value != 0) {
1501 count.waitForCondition(count->cond);
1502 }
1503 count->value = -1;
1504}
1505
Pawin Vongmasa36653902018-11-15 00:10:25 -08001506// CCodecBufferChannel::ReorderStash
1507
1508CCodecBufferChannel::ReorderStash::ReorderStash() {
1509 clear();
1510}
1511
1512void CCodecBufferChannel::ReorderStash::clear() {
1513 mPending.clear();
1514 mStash.clear();
1515 mDepth = 0;
1516 mKey = C2Config::ORDINAL;
1517}
1518
Wonsik Kim6897f222019-01-30 13:29:24 -08001519void CCodecBufferChannel::ReorderStash::flush() {
1520 mPending.clear();
1521 mStash.clear();
1522}
1523
Pawin Vongmasa36653902018-11-15 00:10:25 -08001524void CCodecBufferChannel::ReorderStash::setDepth(uint32_t depth) {
1525 mPending.splice(mPending.end(), mStash);
1526 mDepth = depth;
1527}
Wonsik Kim66427432019-03-21 15:06:22 -07001528
Pawin Vongmasa36653902018-11-15 00:10:25 -08001529void CCodecBufferChannel::ReorderStash::setKey(C2Config::ordinal_key_t key) {
1530 mPending.splice(mPending.end(), mStash);
1531 mKey = key;
1532}
1533
1534bool CCodecBufferChannel::ReorderStash::pop(Entry *entry) {
1535 if (mPending.empty()) {
1536 return false;
1537 }
1538 entry->buffer = mPending.front().buffer;
1539 entry->timestamp = mPending.front().timestamp;
1540 entry->flags = mPending.front().flags;
1541 entry->ordinal = mPending.front().ordinal;
1542 mPending.pop_front();
1543 return true;
1544}
1545
1546void CCodecBufferChannel::ReorderStash::emplace(
1547 const std::shared_ptr<C2Buffer> &buffer,
1548 int64_t timestamp,
1549 int32_t flags,
1550 const C2WorkOrdinalStruct &ordinal) {
Wonsik Kim66427432019-03-21 15:06:22 -07001551 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
1552 if (!buffer && eos) {
1553 // TRICKY: we may be violating ordering of the stash here. Because we
1554 // don't expect any more emplace() calls after this, the ordering should
1555 // not matter.
1556 mStash.emplace_back(buffer, timestamp, flags, ordinal);
1557 } else {
1558 flags = flags & ~MediaCodec::BUFFER_FLAG_EOS;
1559 auto it = mStash.begin();
1560 for (; it != mStash.end(); ++it) {
1561 if (less(ordinal, it->ordinal)) {
1562 break;
1563 }
1564 }
1565 mStash.emplace(it, buffer, timestamp, flags, ordinal);
1566 if (eos) {
1567 mStash.back().flags = mStash.back().flags | MediaCodec::BUFFER_FLAG_EOS;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001568 }
1569 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001570 while (!mStash.empty() && mStash.size() > mDepth) {
1571 mPending.push_back(mStash.front());
1572 mStash.pop_front();
1573 }
1574}
1575
1576void CCodecBufferChannel::ReorderStash::defer(
1577 const CCodecBufferChannel::ReorderStash::Entry &entry) {
1578 mPending.push_front(entry);
1579}
1580
1581bool CCodecBufferChannel::ReorderStash::hasPending() const {
1582 return !mPending.empty();
1583}
1584
1585bool CCodecBufferChannel::ReorderStash::less(
1586 const C2WorkOrdinalStruct &o1, const C2WorkOrdinalStruct &o2) {
1587 switch (mKey) {
1588 case C2Config::ORDINAL: return o1.frameIndex < o2.frameIndex;
1589 case C2Config::TIMESTAMP: return o1.timestamp < o2.timestamp;
1590 case C2Config::CUSTOM: return o1.customOrdinal < o2.customOrdinal;
1591 default:
1592 ALOGD("Unrecognized key; default to timestamp");
1593 return o1.frameIndex < o2.frameIndex;
1594 }
1595}
1596
1597// CCodecBufferChannel
1598
1599CCodecBufferChannel::CCodecBufferChannel(
1600 const std::shared_ptr<CCodecCallback> &callback)
1601 : mHeapSeqNum(-1),
1602 mCCodecCallback(callback),
Wonsik Kim078b58e2019-01-09 15:08:06 -08001603 mNumInputSlots(kSmoothnessFactor),
1604 mNumOutputSlots(kSmoothnessFactor),
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001605 mDelay(0),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001606 mFrameIndex(0u),
1607 mFirstValidFrameIndex(0u),
1608 mMetaMode(MODE_NONE),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001609 mInputMetEos(false) {
Wonsik Kimf5e5c832019-02-21 11:36:05 -08001610 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001611 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
1612 buffers->reset(new DummyInputBuffers(""));
1613}
1614
1615CCodecBufferChannel::~CCodecBufferChannel() {
1616 if (mCrypto != nullptr && mDealer != nullptr && mHeapSeqNum >= 0) {
1617 mCrypto->unsetHeap(mHeapSeqNum);
1618 }
1619}
1620
1621void CCodecBufferChannel::setComponent(
1622 const std::shared_ptr<Codec2Client::Component> &component) {
1623 mComponent = component;
1624 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
1625 mName = mComponentName.c_str();
1626}
1627
1628status_t CCodecBufferChannel::setInputSurface(
1629 const std::shared_ptr<InputSurfaceWrapper> &surface) {
1630 ALOGV("[%s] setInputSurface", mName);
1631 mInputSurface = surface;
1632 return mInputSurface->connect(mComponent);
1633}
1634
1635status_t CCodecBufferChannel::signalEndOfInputStream() {
1636 if (mInputSurface == nullptr) {
1637 return INVALID_OPERATION;
1638 }
1639 return mInputSurface->signalEndOfInputStream();
1640}
1641
1642status_t CCodecBufferChannel::queueInputBufferInternal(const sp<MediaCodecBuffer> &buffer) {
1643 int64_t timeUs;
1644 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1645
1646 if (mInputMetEos) {
1647 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
1648 return OK;
1649 }
1650
1651 int32_t flags = 0;
1652 int32_t tmp = 0;
1653 bool eos = false;
1654 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
1655 eos = true;
1656 mInputMetEos = true;
1657 ALOGV("[%s] input EOS", mName);
1658 }
1659 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
1660 flags |= C2FrameData::FLAG_CODEC_CONFIG;
1661 }
1662 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
1663 std::unique_ptr<C2Work> work(new C2Work);
1664 work->input.ordinal.timestamp = timeUs;
1665 work->input.ordinal.frameIndex = mFrameIndex++;
1666 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
1667 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
1668 // Keep client timestamp in customOrdinal
1669 work->input.ordinal.customOrdinal = timeUs;
1670 work->input.buffers.clear();
1671
Wonsik Kimab34ed62019-01-31 15:28:46 -08001672 uint64_t queuedFrameIndex = work->input.ordinal.frameIndex.peeku();
1673 std::vector<std::shared_ptr<C2Buffer>> queuedBuffers;
1674
Pawin Vongmasa36653902018-11-15 00:10:25 -08001675 if (buffer->size() > 0u) {
1676 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
1677 std::shared_ptr<C2Buffer> c2buffer;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001678 if (!(*buffers)->releaseBuffer(buffer, &c2buffer, false)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001679 return -ENOENT;
1680 }
1681 work->input.buffers.push_back(c2buffer);
Wonsik Kimab34ed62019-01-31 15:28:46 -08001682 queuedBuffers.push_back(c2buffer);
1683 } else if (eos) {
1684 flags |= C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001685 }
1686 work->input.flags = (C2FrameData::flags_t)flags;
1687 // TODO: fill info's
1688
1689 work->input.configUpdate = std::move(mParamsToBeSet);
1690 work->worklets.clear();
1691 work->worklets.emplace_back(new C2Worklet);
1692
1693 std::list<std::unique_ptr<C2Work>> items;
1694 items.push_back(std::move(work));
Wonsik Kimab34ed62019-01-31 15:28:46 -08001695 mPipelineWatcher.lock()->onWorkQueued(
1696 queuedFrameIndex,
1697 std::move(queuedBuffers),
1698 PipelineWatcher::Clock::now());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001699 c2_status_t err = mComponent->queue(&items);
Wonsik Kimab34ed62019-01-31 15:28:46 -08001700 if (err != C2_OK) {
1701 mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex);
1702 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001703
1704 if (err == C2_OK && eos && buffer->size() > 0u) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001705 work.reset(new C2Work);
1706 work->input.ordinal.timestamp = timeUs;
1707 work->input.ordinal.frameIndex = mFrameIndex++;
1708 // WORKAROUND: keep client timestamp in customOrdinal
1709 work->input.ordinal.customOrdinal = timeUs;
1710 work->input.buffers.clear();
1711 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa1c75a232019-01-09 04:41:52 -08001712 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001713
Wonsik Kimab34ed62019-01-31 15:28:46 -08001714 queuedFrameIndex = work->input.ordinal.frameIndex.peeku();
1715 queuedBuffers.clear();
1716
Pawin Vongmasa36653902018-11-15 00:10:25 -08001717 items.clear();
1718 items.push_back(std::move(work));
Wonsik Kimab34ed62019-01-31 15:28:46 -08001719
1720 mPipelineWatcher.lock()->onWorkQueued(
1721 queuedFrameIndex,
1722 std::move(queuedBuffers),
1723 PipelineWatcher::Clock::now());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001724 err = mComponent->queue(&items);
Wonsik Kimab34ed62019-01-31 15:28:46 -08001725 if (err != C2_OK) {
1726 mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex);
1727 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001728 }
1729 if (err == C2_OK) {
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001730 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
1731 bool released = (*buffers)->releaseBuffer(buffer, nullptr, true);
1732 ALOGV("[%s] queueInputBuffer: buffer %sreleased", mName, released ? "" : "not ");
Pawin Vongmasa36653902018-11-15 00:10:25 -08001733 }
1734
1735 feedInputBufferIfAvailableInternal();
1736 return err;
1737}
1738
1739status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> &params) {
1740 QueueGuard guard(mSync);
1741 if (!guard.isRunning()) {
1742 ALOGD("[%s] setParameters is only supported in the running state.", mName);
1743 return -ENOSYS;
1744 }
1745 mParamsToBeSet.insert(mParamsToBeSet.end(),
1746 std::make_move_iterator(params.begin()),
1747 std::make_move_iterator(params.end()));
1748 params.clear();
1749 return OK;
1750}
1751
1752status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
1753 QueueGuard guard(mSync);
1754 if (!guard.isRunning()) {
1755 ALOGD("[%s] No more buffers should be queued at current state.", mName);
1756 return -ENOSYS;
1757 }
1758 return queueInputBufferInternal(buffer);
1759}
1760
1761status_t CCodecBufferChannel::queueSecureInputBuffer(
1762 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
1763 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
1764 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
1765 AString *errorDetailMsg) {
1766 QueueGuard guard(mSync);
1767 if (!guard.isRunning()) {
1768 ALOGD("[%s] No more buffers should be queued at current state.", mName);
1769 return -ENOSYS;
1770 }
1771
1772 if (!hasCryptoOrDescrambler()) {
1773 return -ENOSYS;
1774 }
1775 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
1776
1777 ssize_t result = -1;
1778 ssize_t codecDataOffset = 0;
1779 if (mCrypto != nullptr) {
1780 ICrypto::DestinationBuffer destination;
1781 if (secure) {
1782 destination.mType = ICrypto::kDestinationTypeNativeHandle;
1783 destination.mHandle = encryptedBuffer->handle();
1784 } else {
1785 destination.mType = ICrypto::kDestinationTypeSharedMemory;
1786 destination.mSharedMemory = mDecryptDestination;
1787 }
1788 ICrypto::SourceBuffer source;
1789 encryptedBuffer->fillSourceBuffer(&source);
1790 result = mCrypto->decrypt(
1791 key, iv, mode, pattern, source, buffer->offset(),
1792 subSamples, numSubSamples, destination, errorDetailMsg);
1793 if (result < 0) {
1794 return result;
1795 }
1796 if (destination.mType == ICrypto::kDestinationTypeSharedMemory) {
1797 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
1798 }
1799 } else {
1800 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
1801 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
1802 hidl_vec<SubSample> hidlSubSamples;
1803 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
1804
1805 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
1806 encryptedBuffer->fillSourceBuffer(&srcBuffer);
1807
1808 DestinationBuffer dstBuffer;
1809 if (secure) {
1810 dstBuffer.type = BufferType::NATIVE_HANDLE;
1811 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
1812 } else {
1813 dstBuffer.type = BufferType::SHARED_MEMORY;
1814 dstBuffer.nonsecureMemory = srcBuffer;
1815 }
1816
1817 CasStatus status = CasStatus::OK;
1818 hidl_string detailedError;
1819 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
1820
1821 if (key != nullptr) {
1822 sctrl = (ScramblingControl)key[0];
1823 // Adjust for the PES offset
1824 codecDataOffset = key[2] | (key[3] << 8);
1825 }
1826
1827 auto returnVoid = mDescrambler->descramble(
1828 sctrl,
1829 hidlSubSamples,
1830 srcBuffer,
1831 0,
1832 dstBuffer,
1833 0,
1834 [&status, &result, &detailedError] (
1835 CasStatus _status, uint32_t _bytesWritten,
1836 const hidl_string& _detailedError) {
1837 status = _status;
1838 result = (ssize_t)_bytesWritten;
1839 detailedError = _detailedError;
1840 });
1841
1842 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
1843 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
1844 mName, returnVoid.description().c_str(), status, result);
1845 return UNKNOWN_ERROR;
1846 }
1847
1848 if (result < codecDataOffset) {
1849 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
1850 return BAD_VALUE;
1851 }
1852
1853 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
1854
1855 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
1856 encryptedBuffer->copyDecryptedContentFromMemory(result);
1857 }
1858 }
1859
1860 buffer->setRange(codecDataOffset, result - codecDataOffset);
1861 return queueInputBufferInternal(buffer);
1862}
1863
1864void CCodecBufferChannel::feedInputBufferIfAvailable() {
1865 QueueGuard guard(mSync);
1866 if (!guard.isRunning()) {
1867 ALOGV("[%s] We're not running --- no input buffer reported", mName);
1868 return;
1869 }
1870 feedInputBufferIfAvailableInternal();
1871}
1872
1873void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
Wonsik Kimdf5dd142019-02-06 10:15:46 -08001874 if (mInputMetEos ||
1875 mReorderStash.lock()->hasPending() ||
1876 mPipelineWatcher.lock()->pipelineFull()) {
1877 return;
1878 } else {
1879 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
1880 if ((*buffers)->numClientBuffers() >= mNumOutputSlots) {
1881 return;
1882 }
1883 }
1884 for (size_t i = 0; i < mNumInputSlots; ++i) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001885 sp<MediaCodecBuffer> inBuffer;
1886 size_t index;
1887 {
1888 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
Wonsik Kimab34ed62019-01-31 15:28:46 -08001889 if ((*buffers)->numClientBuffers() >= mNumInputSlots) {
1890 return;
1891 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001892 if (!(*buffers)->requestNewBuffer(&index, &inBuffer)) {
1893 ALOGV("[%s] no new buffer available", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001894 break;
1895 }
1896 }
1897 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
1898 mCallback->onInputBufferAvailable(index, inBuffer);
1899 }
1900}
1901
1902status_t CCodecBufferChannel::renderOutputBuffer(
1903 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001904 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001905 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001906 bool released = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001907 {
1908 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
1909 if (*buffers) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001910 released = (*buffers)->releaseBuffer(buffer, &c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001911 }
1912 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001913 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
1914 // set to true.
1915 sendOutputBuffers();
1916 // input buffer feeding may have been gated by pending output buffers
1917 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001918 if (!c2Buffer) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001919 if (released) {
Wonsik Kimf7529dd2019-04-18 17:35:53 -07001920 std::call_once(mRenderWarningFlag, [this] {
1921 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
1922 "timestamp or render=true with non-video buffers. Apps should "
1923 "call releaseOutputBuffer() with render=false for those.",
1924 mName);
1925 });
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001926 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001927 return INVALID_OPERATION;
1928 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001929
1930#if 0
1931 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
1932 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
1933 for (const std::shared_ptr<const C2Info> &info : infoParams) {
1934 AString res;
1935 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
1936 if (ix) res.append(", ");
1937 res.append(*((int32_t*)info.get() + (ix / 4)));
1938 }
1939 ALOGV(" [%s]", res.c_str());
1940 }
1941#endif
1942 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
1943 std::static_pointer_cast<const C2StreamRotationInfo::output>(
1944 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
1945 bool flip = rotation && (rotation->flip & 1);
1946 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
1947 uint32_t transform = 0;
1948 switch (quarters) {
1949 case 0: // no rotation
1950 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
1951 break;
1952 case 1: // 90 degrees counter-clockwise
1953 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
1954 : HAL_TRANSFORM_ROT_270;
1955 break;
1956 case 2: // 180 degrees
1957 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
1958 break;
1959 case 3: // 90 degrees clockwise
1960 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
1961 : HAL_TRANSFORM_ROT_90;
1962 break;
1963 }
1964
1965 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
1966 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
1967 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
1968 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
1969 if (surfaceScaling) {
1970 videoScalingMode = surfaceScaling->value;
1971 }
1972
1973 // Use dataspace from format as it has the default aspects already applied
1974 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
1975 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
1976
1977 // HDR static info
1978 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
1979 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
1980 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
1981
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001982 // HDR10 plus info
1983 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
1984 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
1985 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
1986
Pawin Vongmasa36653902018-11-15 00:10:25 -08001987 {
1988 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1989 if (output->surface == nullptr) {
1990 ALOGI("[%s] cannot render buffer without surface", mName);
1991 return OK;
1992 }
1993 }
1994
1995 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
1996 if (blocks.size() != 1u) {
1997 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
1998 return UNKNOWN_ERROR;
1999 }
2000 const C2ConstGraphicBlock &block = blocks.front();
2001
2002 // TODO: revisit this after C2Fence implementation.
2003 android::IGraphicBufferProducer::QueueBufferInput qbi(
2004 timestampNs,
2005 false, // droppable
2006 dataSpace,
2007 Rect(blocks.front().crop().left,
2008 blocks.front().crop().top,
2009 blocks.front().crop().right(),
2010 blocks.front().crop().bottom()),
2011 videoScalingMode,
2012 transform,
2013 Fence::NO_FENCE, 0);
Pawin Vongmasa8be93112018-12-11 14:01:42 -08002014 if (hdrStaticInfo || hdr10PlusInfo) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002015 HdrMetadata hdr;
Pawin Vongmasa8be93112018-12-11 14:01:42 -08002016 if (hdrStaticInfo) {
2017 struct android_smpte2086_metadata smpte2086_meta = {
2018 .displayPrimaryRed = {
2019 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
2020 },
2021 .displayPrimaryGreen = {
2022 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
2023 },
2024 .displayPrimaryBlue = {
2025 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
2026 },
2027 .whitePoint = {
2028 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
2029 },
2030 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
2031 .minLuminance = hdrStaticInfo->mastering.minLuminance,
2032 };
2033
2034 struct android_cta861_3_metadata cta861_meta = {
2035 .maxContentLightLevel = hdrStaticInfo->maxCll,
2036 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
2037 };
2038
2039 hdr.validTypes = HdrMetadata::SMPTE2086 | HdrMetadata::CTA861_3;
2040 hdr.smpte2086 = smpte2086_meta;
2041 hdr.cta8613 = cta861_meta;
2042 }
2043 if (hdr10PlusInfo) {
2044 hdr.validTypes |= HdrMetadata::HDR10PLUS;
2045 hdr.hdr10plus.assign(
2046 hdr10PlusInfo->m.value,
2047 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
2048 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002049 qbi.setHdrMetadata(hdr);
2050 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -08002051 // we don't have dirty regions
2052 qbi.setSurfaceDamage(Region::INVALID_REGION);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002053 android::IGraphicBufferProducer::QueueBufferOutput qbo;
2054 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
2055 if (result != OK) {
2056 ALOGI("[%s] queueBuffer failed: %d", mName, result);
2057 return result;
2058 }
2059 ALOGV("[%s] queue buffer successful", mName);
2060
2061 int64_t mediaTimeUs = 0;
2062 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
2063 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
2064
2065 return OK;
2066}
2067
2068status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
2069 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
2070 bool released = false;
2071 {
2072 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
Pawin Vongmasa1f213362019-01-24 06:59:16 -08002073 if (*buffers && (*buffers)->releaseBuffer(buffer, nullptr, true)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002074 released = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002075 }
2076 }
2077 {
2078 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2079 if (*buffers && (*buffers)->releaseBuffer(buffer, nullptr)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002080 released = true;
2081 }
2082 }
2083 if (released) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002084 sendOutputBuffers();
Pawin Vongmasa8be93112018-12-11 14:01:42 -08002085 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -08002086 } else {
2087 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
2088 }
2089 return OK;
2090}
2091
2092void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
2093 array->clear();
2094 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
2095
2096 if (!(*buffers)->isArrayMode()) {
Wonsik Kim078b58e2019-01-09 15:08:06 -08002097 *buffers = (*buffers)->toArrayMode(mNumInputSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002098 }
2099
2100 (*buffers)->getArray(array);
2101}
2102
2103void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
2104 array->clear();
2105 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2106
2107 if (!(*buffers)->isArrayMode()) {
Wonsik Kim078b58e2019-01-09 15:08:06 -08002108 *buffers = (*buffers)->toArrayMode(mNumOutputSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002109 }
2110
2111 (*buffers)->getArray(array);
2112}
2113
2114status_t CCodecBufferChannel::start(
2115 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) {
2116 C2StreamBufferTypeSetting::input iStreamFormat(0u);
2117 C2StreamBufferTypeSetting::output oStreamFormat(0u);
2118 C2PortReorderBufferDepthTuning::output reorderDepth;
2119 C2PortReorderKeySetting::output reorderKey;
Wonsik Kim078b58e2019-01-09 15:08:06 -08002120 C2PortActualDelayTuning::input inputDelay(0);
2121 C2PortActualDelayTuning::output outputDelay(0);
2122 C2ActualPipelineDelayTuning pipelineDelay(0);
2123
Pawin Vongmasa36653902018-11-15 00:10:25 -08002124 c2_status_t err = mComponent->query(
2125 {
2126 &iStreamFormat,
2127 &oStreamFormat,
2128 &reorderDepth,
2129 &reorderKey,
Wonsik Kim078b58e2019-01-09 15:08:06 -08002130 &inputDelay,
2131 &pipelineDelay,
2132 &outputDelay,
Pawin Vongmasa36653902018-11-15 00:10:25 -08002133 },
2134 {},
2135 C2_DONT_BLOCK,
2136 nullptr);
2137 if (err == C2_BAD_INDEX) {
2138 if (!iStreamFormat || !oStreamFormat) {
2139 return UNKNOWN_ERROR;
2140 }
2141 } else if (err != C2_OK) {
2142 return UNKNOWN_ERROR;
2143 }
2144
2145 {
2146 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
2147 reorder->clear();
2148 if (reorderDepth) {
2149 reorder->setDepth(reorderDepth.value);
2150 }
2151 if (reorderKey) {
2152 reorder->setKey(reorderKey.value);
2153 }
2154 }
Wonsik Kim078b58e2019-01-09 15:08:06 -08002155
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08002156 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
2157 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
2158 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
2159
2160 mNumInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
2161 mNumOutputSlots = outputDelayValue + kSmoothnessFactor;
2162 mDelay = inputDelayValue + pipelineDelayValue + outputDelayValue;
Wonsik Kim078b58e2019-01-09 15:08:06 -08002163
Pawin Vongmasa36653902018-11-15 00:10:25 -08002164 // TODO: get this from input format
2165 bool secure = mComponent->getName().find(".secure") != std::string::npos;
2166
2167 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
2168 int poolMask = property_get_int32(
2169 "debug.stagefright.c2-poolmask",
2170 1 << C2PlatformAllocatorStore::ION |
2171 1 << C2PlatformAllocatorStore::BUFFERQUEUE);
2172
2173 if (inputFormat != nullptr) {
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08002174 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002175 std::shared_ptr<C2BlockPool> pool;
2176 {
2177 Mutexed<BlockPools>::Locked pools(mBlockPools);
2178
2179 // set default allocator ID.
2180 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
2181 : C2PlatformAllocatorStore::ION;
2182
2183 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
2184 // from component, create the input block pool with given ID. Otherwise, use default IDs.
2185 std::vector<std::unique_ptr<C2Param>> params;
2186 err = mComponent->query({ },
2187 { C2PortAllocatorsTuning::input::PARAM_TYPE },
2188 C2_DONT_BLOCK,
2189 &params);
2190 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
2191 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
2192 mName, params.size(), asString(err), err);
2193 } else if (err == C2_OK && params.size() == 1) {
2194 C2PortAllocatorsTuning::input *inputAllocators =
2195 C2PortAllocatorsTuning::input::From(params[0].get());
2196 if (inputAllocators && inputAllocators->flexCount() > 0) {
2197 std::shared_ptr<C2Allocator> allocator;
2198 // verify allocator IDs and resolve default allocator
2199 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
2200 if (allocator) {
2201 pools->inputAllocatorId = allocator->getId();
2202 } else {
2203 ALOGD("[%s] component requested invalid input allocator ID %u",
2204 mName, inputAllocators->m.values[0]);
2205 }
2206 }
2207 }
2208
2209 // TODO: use C2Component wrapper to associate this pool with ourselves
2210 if ((poolMask >> pools->inputAllocatorId) & 1) {
2211 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
2212 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
2213 mName, pools->inputAllocatorId,
2214 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
2215 asString(err), err);
2216 } else {
2217 err = C2_NOT_FOUND;
2218 }
2219 if (err != C2_OK) {
2220 C2BlockPool::local_id_t inputPoolId =
2221 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
2222 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
2223 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
2224 mName, (unsigned long long)inputPoolId,
2225 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
2226 asString(err), err);
2227 if (err != C2_OK) {
2228 return NO_MEMORY;
2229 }
2230 }
2231 pools->inputPool = pool;
2232 }
2233
Wonsik Kim51051262018-11-28 13:59:05 -08002234 bool forceArrayMode = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002235 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
2236 if (graphic) {
2237 if (mInputSurface) {
2238 buffers->reset(new DummyInputBuffers(mName));
2239 } else if (mMetaMode == MODE_ANW) {
2240 buffers->reset(new GraphicMetadataInputBuffers(mName));
2241 } else {
Wonsik Kim078b58e2019-01-09 15:08:06 -08002242 buffers->reset(new GraphicInputBuffers(mNumInputSlots, mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08002243 }
2244 } else {
2245 if (hasCryptoOrDescrambler()) {
2246 int32_t capacity = kLinearBufferSize;
2247 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
2248 if ((size_t)capacity > kMaxLinearBufferSize) {
2249 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
2250 capacity = kMaxLinearBufferSize;
2251 }
2252 if (mDealer == nullptr) {
2253 mDealer = new MemoryDealer(
2254 align(capacity, MemoryDealer::getAllocationAlignment())
Wonsik Kim078b58e2019-01-09 15:08:06 -08002255 * (mNumInputSlots + 1),
Pawin Vongmasa36653902018-11-15 00:10:25 -08002256 "EncryptedLinearInputBuffers");
2257 mDecryptDestination = mDealer->allocate((size_t)capacity);
2258 }
2259 if (mCrypto != nullptr && mHeapSeqNum < 0) {
2260 mHeapSeqNum = mCrypto->setHeap(mDealer->getMemoryHeap());
2261 } else {
2262 mHeapSeqNum = -1;
2263 }
2264 buffers->reset(new EncryptedLinearInputBuffers(
Wonsik Kim078b58e2019-01-09 15:08:06 -08002265 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
2266 mNumInputSlots, mName));
Wonsik Kim51051262018-11-28 13:59:05 -08002267 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002268 } else {
2269 buffers->reset(new LinearInputBuffers(mName));
2270 }
2271 }
2272 (*buffers)->setFormat(inputFormat);
2273
2274 if (err == C2_OK) {
2275 (*buffers)->setPool(pool);
2276 } else {
2277 // TODO: error
2278 }
Wonsik Kim51051262018-11-28 13:59:05 -08002279
2280 if (forceArrayMode) {
Wonsik Kim078b58e2019-01-09 15:08:06 -08002281 *buffers = (*buffers)->toArrayMode(mNumInputSlots);
Wonsik Kim51051262018-11-28 13:59:05 -08002282 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002283 }
2284
2285 if (outputFormat != nullptr) {
2286 sp<IGraphicBufferProducer> outputSurface;
2287 uint32_t outputGeneration;
2288 {
2289 Mutexed<OutputSurface>::Locked output(mOutputSurface);
Wonsik Kimf5e5c832019-02-21 11:36:05 -08002290 output->maxDequeueBuffers = mNumOutputSlots + reorderDepth.value + kRenderingDepth;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002291 outputSurface = output->surface ?
2292 output->surface->getIGraphicBufferProducer() : nullptr;
Wonsik Kimf5e5c832019-02-21 11:36:05 -08002293 if (outputSurface) {
2294 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
2295 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002296 outputGeneration = output->generation;
2297 }
2298
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08002299 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002300 C2BlockPool::local_id_t outputPoolId_;
2301
2302 {
2303 Mutexed<BlockPools>::Locked pools(mBlockPools);
2304
2305 // set default allocator ID.
2306 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
2307 : C2PlatformAllocatorStore::ION;
2308
2309 // query C2PortAllocatorsTuning::output from component, or use default allocator if
2310 // unsuccessful.
2311 std::vector<std::unique_ptr<C2Param>> params;
2312 err = mComponent->query({ },
2313 { C2PortAllocatorsTuning::output::PARAM_TYPE },
2314 C2_DONT_BLOCK,
2315 &params);
2316 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
2317 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
2318 mName, params.size(), asString(err), err);
2319 } else if (err == C2_OK && params.size() == 1) {
2320 C2PortAllocatorsTuning::output *outputAllocators =
2321 C2PortAllocatorsTuning::output::From(params[0].get());
2322 if (outputAllocators && outputAllocators->flexCount() > 0) {
2323 std::shared_ptr<C2Allocator> allocator;
2324 // verify allocator IDs and resolve default allocator
2325 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
2326 if (allocator) {
2327 pools->outputAllocatorId = allocator->getId();
2328 } else {
2329 ALOGD("[%s] component requested invalid output allocator ID %u",
2330 mName, outputAllocators->m.values[0]);
2331 }
2332 }
2333 }
2334
2335 // use bufferqueue if outputting to a surface.
2336 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
2337 // if unsuccessful.
2338 if (outputSurface) {
2339 params.clear();
2340 err = mComponent->query({ },
2341 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
2342 C2_DONT_BLOCK,
2343 &params);
2344 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
2345 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
2346 mName, params.size(), asString(err), err);
2347 } else if (err == C2_OK && params.size() == 1) {
2348 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
2349 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
2350 if (surfaceAllocator) {
2351 std::shared_ptr<C2Allocator> allocator;
2352 // verify allocator IDs and resolve default allocator
2353 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
2354 if (allocator) {
2355 pools->outputAllocatorId = allocator->getId();
2356 } else {
2357 ALOGD("[%s] component requested invalid surface output allocator ID %u",
2358 mName, surfaceAllocator->value);
2359 err = C2_BAD_VALUE;
2360 }
2361 }
2362 }
2363 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
2364 && err != C2_OK
2365 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
2366 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
2367 }
2368 }
2369
2370 if ((poolMask >> pools->outputAllocatorId) & 1) {
2371 err = mComponent->createBlockPool(
2372 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
2373 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
2374 mName, pools->outputAllocatorId,
2375 (unsigned long long)pools->outputPoolId,
2376 asString(err));
2377 } else {
2378 err = C2_NOT_FOUND;
2379 }
2380 if (err != C2_OK) {
2381 // use basic pool instead
2382 pools->outputPoolId =
2383 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
2384 }
2385
2386 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
2387 // component.
2388 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
2389 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
2390
2391 std::vector<std::unique_ptr<C2SettingResult>> failures;
2392 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
2393 ALOGD("[%s] Configured output block pool ids %llu => %s",
2394 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
2395 outputPoolId_ = pools->outputPoolId;
2396 }
2397
2398 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2399
2400 if (graphic) {
2401 if (outputSurface) {
2402 buffers->reset(new GraphicOutputBuffers(mName));
2403 } else {
Wonsik Kim078b58e2019-01-09 15:08:06 -08002404 buffers->reset(new RawGraphicOutputBuffers(mNumOutputSlots, mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08002405 }
2406 } else {
2407 buffers->reset(new LinearOutputBuffers(mName));
2408 }
2409 (*buffers)->setFormat(outputFormat->dup());
2410
2411
2412 // Try to set output surface to created block pool if given.
2413 if (outputSurface) {
2414 mComponent->setOutputSurface(
2415 outputPoolId_,
2416 outputSurface,
2417 outputGeneration);
2418 }
2419
2420 if (oStreamFormat.value == C2BufferData::LINEAR
2421 && mComponentName.find("c2.qti.") == std::string::npos) {
2422 // WORKAROUND: if we're using early CSD workaround we convert to
2423 // array mode, to appease apps assuming the output
2424 // buffers to be of the same size.
Wonsik Kim078b58e2019-01-09 15:08:06 -08002425 (*buffers) = (*buffers)->toArrayMode(mNumOutputSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002426
2427 int32_t channelCount;
2428 int32_t sampleRate;
2429 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
2430 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
2431 int32_t delay = 0;
2432 int32_t padding = 0;;
2433 if (!outputFormat->findInt32("encoder-delay", &delay)) {
2434 delay = 0;
2435 }
2436 if (!outputFormat->findInt32("encoder-padding", &padding)) {
2437 padding = 0;
2438 }
2439 if (delay || padding) {
2440 // We need write access to the buffers, and we're already in
2441 // array mode.
2442 (*buffers)->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
2443 }
2444 }
2445 }
2446 }
2447
2448 // Set up pipeline control. This has to be done after mInputBuffers and
2449 // mOutputBuffers are initialized to make sure that lingering callbacks
2450 // about buffers from the previous generation do not interfere with the
2451 // newly initialized pipeline capacity.
2452
Wonsik Kimab34ed62019-01-31 15:28:46 -08002453 {
2454 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08002455 watcher->inputDelay(inputDelayValue)
2456 .pipelineDelay(pipelineDelayValue)
2457 .outputDelay(outputDelayValue)
Wonsik Kimab34ed62019-01-31 15:28:46 -08002458 .smoothnessFactor(kSmoothnessFactor);
2459 watcher->flush();
2460 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002461
2462 mInputMetEos = false;
2463 mSync.start();
2464 return OK;
2465}
2466
2467status_t CCodecBufferChannel::requestInitialInputBuffers() {
2468 if (mInputSurface) {
2469 return OK;
2470 }
2471
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08002472 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002473 c2_status_t err = mComponent->query({ &oStreamFormat }, {}, C2_DONT_BLOCK, nullptr);
2474 if (err != C2_OK) {
2475 return UNKNOWN_ERROR;
2476 }
2477 std::vector<sp<MediaCodecBuffer>> toBeQueued;
2478 // TODO: use proper buffer depth instead of this random value
Wonsik Kim078b58e2019-01-09 15:08:06 -08002479 for (size_t i = 0; i < mNumInputSlots; ++i) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002480 size_t index;
2481 sp<MediaCodecBuffer> buffer;
2482 {
2483 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
2484 if (!(*buffers)->requestNewBuffer(&index, &buffer)) {
2485 if (i == 0) {
2486 ALOGW("[%s] start: cannot allocate memory at all", mName);
2487 return NO_MEMORY;
2488 } else {
2489 ALOGV("[%s] start: cannot allocate memory, only %zu buffers allocated",
2490 mName, i);
2491 }
2492 break;
2493 }
2494 }
2495 if (buffer) {
2496 Mutexed<std::list<sp<ABuffer>>>::Locked configs(mFlushedConfigs);
2497 ALOGV("[%s] input buffer %zu available", mName, index);
2498 bool post = true;
2499 if (!configs->empty()) {
2500 sp<ABuffer> config = configs->front();
Pawin Vongmasa472c7382019-03-26 18:13:58 -07002501 configs->pop_front();
Pawin Vongmasa36653902018-11-15 00:10:25 -08002502 if (buffer->capacity() >= config->size()) {
2503 memcpy(buffer->base(), config->data(), config->size());
2504 buffer->setRange(0, config->size());
2505 buffer->meta()->clear();
2506 buffer->meta()->setInt64("timeUs", 0);
2507 buffer->meta()->setInt32("csd", 1);
2508 post = false;
2509 } else {
2510 ALOGD("[%s] buffer capacity too small for the config (%zu < %zu)",
2511 mName, buffer->capacity(), config->size());
2512 }
2513 } else if (oStreamFormat.value == C2BufferData::LINEAR && i == 0
2514 && mComponentName.find("c2.qti.") == std::string::npos) {
2515 // WORKAROUND: Some apps expect CSD available without queueing
2516 // any input. Queue an empty buffer to get the CSD.
2517 buffer->setRange(0, 0);
2518 buffer->meta()->clear();
2519 buffer->meta()->setInt64("timeUs", 0);
2520 post = false;
2521 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08002522 if (post) {
2523 mCallback->onInputBufferAvailable(index, buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002524 } else {
Wonsik Kimab34ed62019-01-31 15:28:46 -08002525 toBeQueued.emplace_back(buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002526 }
2527 }
2528 }
2529 for (const sp<MediaCodecBuffer> &buffer : toBeQueued) {
2530 if (queueInputBufferInternal(buffer) != OK) {
Wonsik Kimab34ed62019-01-31 15:28:46 -08002531 ALOGV("[%s] Error while queueing initial buffers", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002532 }
2533 }
2534 return OK;
2535}
2536
2537void CCodecBufferChannel::stop() {
2538 mSync.stop();
2539 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
2540 if (mInputSurface != nullptr) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002541 mInputSurface.reset();
2542 }
2543}
2544
2545void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
2546 ALOGV("[%s] flush", mName);
2547 {
2548 Mutexed<std::list<sp<ABuffer>>>::Locked configs(mFlushedConfigs);
2549 for (const std::unique_ptr<C2Work> &work : flushedWork) {
2550 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
2551 continue;
2552 }
2553 if (work->input.buffers.empty()
2554 || work->input.buffers.front()->data().linearBlocks().empty()) {
2555 ALOGD("[%s] no linear codec config data found", mName);
2556 continue;
2557 }
2558 C2ReadView view =
2559 work->input.buffers.front()->data().linearBlocks().front().map().get();
2560 if (view.error() != C2_OK) {
2561 ALOGD("[%s] failed to map flushed codec config data: %d", mName, view.error());
2562 continue;
2563 }
2564 configs->push_back(ABuffer::CreateAsCopy(view.data(), view.capacity()));
2565 ALOGV("[%s] stashed flushed codec config data (size=%u)", mName, view.capacity());
2566 }
2567 }
2568 {
2569 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
2570 (*buffers)->flush();
2571 }
2572 {
2573 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2574 (*buffers)->flush(flushedWork);
2575 }
Wonsik Kim6897f222019-01-30 13:29:24 -08002576 mReorderStash.lock()->flush();
Wonsik Kimab34ed62019-01-31 15:28:46 -08002577 mPipelineWatcher.lock()->flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08002578}
2579
2580void CCodecBufferChannel::onWorkDone(
2581 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
Wonsik Kimab34ed62019-01-31 15:28:46 -08002582 const C2StreamInitDataInfo::output *initData) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002583 if (handleWork(std::move(work), outputFormat, initData)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002584 feedInputBufferIfAvailable();
2585 }
2586}
2587
2588void CCodecBufferChannel::onInputBufferDone(
Wonsik Kimab34ed62019-01-31 15:28:46 -08002589 uint64_t frameIndex, size_t arrayIndex) {
2590 std::shared_ptr<C2Buffer> buffer =
2591 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002592 bool newInputSlotAvailable;
2593 {
2594 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
2595 newInputSlotAvailable = (*buffers)->expireComponentBuffer(buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002596 }
2597 if (newInputSlotAvailable) {
2598 feedInputBufferIfAvailable();
2599 }
2600}
2601
2602bool CCodecBufferChannel::handleWork(
2603 std::unique_ptr<C2Work> work,
2604 const sp<AMessage> &outputFormat,
2605 const C2StreamInitDataInfo::output *initData) {
2606 if ((work->input.ordinal.frameIndex - mFirstValidFrameIndex.load()).peek() < 0) {
2607 // Discard frames from previous generation.
2608 ALOGD("[%s] Discard frames from previous generation.", mName);
2609 return false;
2610 }
2611
Wonsik Kim524b0582019-03-12 11:28:57 -07002612 if (mInputSurface == nullptr && (work->worklets.size() != 1u
Pawin Vongmasa36653902018-11-15 00:10:25 -08002613 || !work->worklets.front()
Wonsik Kim524b0582019-03-12 11:28:57 -07002614 || !(work->worklets.front()->output.flags & C2FrameData::FLAG_INCOMPLETE))) {
Wonsik Kimab34ed62019-01-31 15:28:46 -08002615 mPipelineWatcher.lock()->onWorkDone(work->input.ordinal.frameIndex.peeku());
Pawin Vongmasa36653902018-11-15 00:10:25 -08002616 }
2617
2618 if (work->result == C2_NOT_FOUND) {
2619 ALOGD("[%s] flushed work; ignored.", mName);
2620 return true;
2621 }
2622
2623 if (work->result != C2_OK) {
2624 ALOGD("[%s] work failed to complete: %d", mName, work->result);
2625 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
2626 return false;
2627 }
2628
2629 // NOTE: MediaCodec usage supposedly have only one worklet
2630 if (work->worklets.size() != 1u) {
2631 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
2632 mName, work->worklets.size());
2633 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2634 return false;
2635 }
2636
2637 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
2638
2639 std::shared_ptr<C2Buffer> buffer;
2640 // NOTE: MediaCodec usage supposedly have only one output stream.
2641 if (worklet->output.buffers.size() > 1u) {
2642 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
2643 mName, worklet->output.buffers.size());
2644 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2645 return false;
2646 } else if (worklet->output.buffers.size() == 1u) {
2647 buffer = worklet->output.buffers[0];
2648 if (!buffer) {
2649 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
2650 }
2651 }
2652
2653 while (!worklet->output.configUpdate.empty()) {
2654 std::unique_ptr<C2Param> param;
2655 worklet->output.configUpdate.back().swap(param);
2656 worklet->output.configUpdate.pop_back();
2657 switch (param->coreIndex().coreIndex()) {
2658 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
2659 C2PortReorderBufferDepthTuning::output reorderDepth;
2660 if (reorderDepth.updateFrom(*param)) {
2661 mReorderStash.lock()->setDepth(reorderDepth.value);
2662 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
2663 mName, reorderDepth.value);
Wonsik Kimf5e5c832019-02-21 11:36:05 -08002664 Mutexed<OutputSurface>::Locked output(mOutputSurface);
2665 output->maxDequeueBuffers = mNumOutputSlots + reorderDepth.value + kRenderingDepth;
2666 if (output->surface) {
2667 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
2668 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002669 } else {
2670 ALOGD("[%s] onWorkDone: failed to read reorder depth", mName);
2671 }
2672 break;
2673 }
2674 case C2PortReorderKeySetting::CORE_INDEX: {
2675 C2PortReorderKeySetting::output reorderKey;
2676 if (reorderKey.updateFrom(*param)) {
2677 mReorderStash.lock()->setKey(reorderKey.value);
2678 ALOGV("[%s] onWorkDone: updated reorder key to %u",
2679 mName, reorderKey.value);
2680 } else {
2681 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
2682 }
2683 break;
2684 }
2685 default:
2686 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
2687 mName, param->index());
2688 break;
2689 }
2690 }
2691
2692 if (outputFormat != nullptr) {
2693 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2694 ALOGD("[%s] onWorkDone: output format changed to %s",
2695 mName, outputFormat->debugString().c_str());
2696 (*buffers)->setFormat(outputFormat);
2697
2698 AString mediaType;
2699 if (outputFormat->findString(KEY_MIME, &mediaType)
2700 && mediaType == MIMETYPE_AUDIO_RAW) {
2701 int32_t channelCount;
2702 int32_t sampleRate;
2703 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
2704 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
2705 (*buffers)->updateSkipCutBuffer(sampleRate, channelCount);
2706 }
2707 }
2708 }
2709
2710 int32_t flags = 0;
2711 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
2712 flags |= MediaCodec::BUFFER_FLAG_EOS;
2713 ALOGV("[%s] onWorkDone: output EOS", mName);
2714 }
2715
2716 sp<MediaCodecBuffer> outBuffer;
2717 size_t index;
2718
2719 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
2720 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
2721 // the codec input timestamp, but client output timestamp should (reported in timeUs)
2722 // shall correspond to the client input timesamp (in customOrdinal). By using the
2723 // delta between the two, this allows for some timestamp deviation - e.g. if one input
2724 // produces multiple output.
2725 c2_cntr64_t timestamp =
2726 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
2727 - work->input.ordinal.timestamp;
Wonsik Kim95ba0162019-03-19 15:51:54 -07002728 if (mInputSurface != nullptr) {
2729 // When using input surface we need to restore the original input timestamp.
2730 timestamp = work->input.ordinal.customOrdinal;
2731 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002732 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
2733 mName,
2734 work->input.ordinal.customOrdinal.peekll(),
2735 work->input.ordinal.timestamp.peekll(),
2736 worklet->output.ordinal.timestamp.peekll(),
2737 timestamp.peekll());
2738
2739 if (initData != nullptr) {
2740 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2741 if ((*buffers)->registerCsd(initData, &index, &outBuffer) == OK) {
2742 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
2743 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
2744 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
2745
2746 buffers.unlock();
2747 mCallback->onOutputBufferAvailable(index, outBuffer);
2748 buffers.lock();
2749 } else {
2750 ALOGD("[%s] onWorkDone: unable to register csd", mName);
2751 buffers.unlock();
2752 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2753 buffers.lock();
2754 return false;
2755 }
2756 }
2757
2758 if (!buffer && !flags) {
2759 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
2760 mName, work->input.ordinal.frameIndex.peekull());
2761 return true;
2762 }
2763
2764 if (buffer) {
2765 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
2766 // TODO: properly translate these to metadata
2767 switch (info->coreIndex().coreIndex()) {
2768 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08002769 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002770 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
2771 }
2772 break;
2773 default:
2774 break;
2775 }
2776 }
2777 }
2778
2779 {
2780 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
2781 reorder->emplace(buffer, timestamp.peek(), flags, worklet->output.ordinal);
2782 if (flags & MediaCodec::BUFFER_FLAG_EOS) {
2783 // Flush reorder stash
2784 reorder->setDepth(0);
2785 }
2786 }
2787 sendOutputBuffers();
2788 return true;
2789}
2790
2791void CCodecBufferChannel::sendOutputBuffers() {
2792 ReorderStash::Entry entry;
2793 sp<MediaCodecBuffer> outBuffer;
2794 size_t index;
2795
2796 while (true) {
Wonsik Kim38ad3412019-02-01 15:13:23 -08002797 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
2798 if (!reorder->hasPending()) {
2799 break;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002800 }
Wonsik Kim38ad3412019-02-01 15:13:23 -08002801 if (!reorder->pop(&entry)) {
2802 break;
2803 }
2804
Pawin Vongmasa36653902018-11-15 00:10:25 -08002805 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2806 status_t err = (*buffers)->registerBuffer(entry.buffer, &index, &outBuffer);
2807 if (err != OK) {
Wonsik Kim38ad3412019-02-01 15:13:23 -08002808 bool outputBuffersChanged = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002809 if (err != WOULD_BLOCK) {
Wonsik Kim186fdbf2019-01-29 13:30:01 -08002810 if (!(*buffers)->isArrayMode()) {
2811 *buffers = (*buffers)->toArrayMode(mNumOutputSlots);
2812 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002813 OutputBuffersArray *array = (OutputBuffersArray *)buffers->get();
2814 array->realloc(entry.buffer);
Wonsik Kim38ad3412019-02-01 15:13:23 -08002815 outputBuffersChanged = true;
2816 }
2817 ALOGV("[%s] sendOutputBuffers: unable to register output buffer", mName);
2818 reorder->defer(entry);
2819
2820 buffers.unlock();
2821 reorder.unlock();
2822
2823 if (outputBuffersChanged) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002824 mCCodecCallback->onOutputBuffersChanged();
2825 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002826 return;
2827 }
2828 buffers.unlock();
Wonsik Kim38ad3412019-02-01 15:13:23 -08002829 reorder.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08002830
2831 outBuffer->meta()->setInt64("timeUs", entry.timestamp);
2832 outBuffer->meta()->setInt32("flags", entry.flags);
Wonsik Kim66427432019-03-21 15:06:22 -07002833 ALOGV("[%s] sendOutputBuffers: out buffer index = %zu [%p] => %p + %zu (%lld)",
2834 mName, index, outBuffer.get(), outBuffer->data(), outBuffer->size(),
2835 (long long)entry.timestamp);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002836 mCallback->onOutputBufferAvailable(index, outBuffer);
2837 }
2838}
2839
2840status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
2841 static std::atomic_uint32_t surfaceGeneration{0};
2842 uint32_t generation = (getpid() << 10) |
2843 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
2844 & ((1 << 10) - 1));
2845
2846 sp<IGraphicBufferProducer> producer;
2847 if (newSurface) {
2848 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002849 producer = newSurface->getIGraphicBufferProducer();
2850 producer->setGenerationNumber(generation);
2851 } else {
2852 ALOGE("[%s] setting output surface to null", mName);
2853 return INVALID_OPERATION;
2854 }
2855
2856 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
2857 C2BlockPool::local_id_t outputPoolId;
2858 {
2859 Mutexed<BlockPools>::Locked pools(mBlockPools);
2860 outputPoolId = pools->outputPoolId;
2861 outputPoolIntf = pools->outputPoolIntf;
2862 }
2863
2864 if (outputPoolIntf) {
2865 if (mComponent->setOutputSurface(
2866 outputPoolId,
2867 producer,
2868 generation) != C2_OK) {
2869 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
2870 return INVALID_OPERATION;
2871 }
2872 }
2873
2874 {
2875 Mutexed<OutputSurface>::Locked output(mOutputSurface);
Wonsik Kimf5e5c832019-02-21 11:36:05 -08002876 newSurface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002877 output->surface = newSurface;
2878 output->generation = generation;
2879 }
2880
2881 return OK;
2882}
2883
Wonsik Kimab34ed62019-01-31 15:28:46 -08002884PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08002885 // When client pushed EOS, we want all the work to be done quickly.
2886 // Otherwise, component may have stalled work due to input starvation up to
2887 // the sum of the delay in the pipeline.
2888 size_t n = mInputMetEos ? 0 : mDelay;
2889 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
Wonsik Kimab34ed62019-01-31 15:28:46 -08002890}
2891
Pawin Vongmasa36653902018-11-15 00:10:25 -08002892void CCodecBufferChannel::setMetaMode(MetaMode mode) {
2893 mMetaMode = mode;
2894}
2895
2896status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
2897 // C2_OK is always translated to OK.
2898 if (c2s == C2_OK) {
2899 return OK;
2900 }
2901
2902 // Operation-dependent translation
2903 // TODO: Add as necessary
2904 switch (c2op) {
2905 case C2_OPERATION_Component_start:
2906 switch (c2s) {
2907 case C2_NO_MEMORY:
2908 return NO_MEMORY;
2909 default:
2910 return UNKNOWN_ERROR;
2911 }
2912 default:
2913 break;
2914 }
2915
2916 // Backup operation-agnostic translation
2917 switch (c2s) {
2918 case C2_BAD_INDEX:
2919 return BAD_INDEX;
2920 case C2_BAD_VALUE:
2921 return BAD_VALUE;
2922 case C2_BLOCKING:
2923 return WOULD_BLOCK;
2924 case C2_DUPLICATE:
2925 return ALREADY_EXISTS;
2926 case C2_NO_INIT:
2927 return NO_INIT;
2928 case C2_NO_MEMORY:
2929 return NO_MEMORY;
2930 case C2_NOT_FOUND:
2931 return NAME_NOT_FOUND;
2932 case C2_TIMED_OUT:
2933 return TIMED_OUT;
2934 case C2_BAD_STATE:
2935 case C2_CANCELED:
2936 case C2_CANNOT_DO:
2937 case C2_CORRUPTED:
2938 case C2_OMITTED:
2939 case C2_REFUSED:
2940 return UNKNOWN_ERROR;
2941 default:
2942 return -static_cast<status_t>(c2s);
2943 }
2944}
2945
2946} // namespace android