blob: 53ae5858934c9ef1aea873611c320ddee4b488c1 [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodecBufferChannel"
19#include <utils/Log.h>
20
21#include <numeric>
22
23#include <C2AllocatorGralloc.h>
24#include <C2PlatformSupport.h>
25#include <C2BlockInternal.h>
26#include <C2Config.h>
27#include <C2Debug.h>
28
29#include <android/hardware/cas/native/1.0/IDescrambler.h>
30#include <android-base/stringprintf.h>
31#include <binder/MemoryDealer.h>
32#include <gui/Surface.h>
33#include <media/openmax/OMX_Core.h>
34#include <media/stagefright/foundation/ABuffer.h>
35#include <media/stagefright/foundation/ALookup.h>
36#include <media/stagefright/foundation/AMessage.h>
37#include <media/stagefright/foundation/AUtils.h>
38#include <media/stagefright/foundation/hexdump.h>
39#include <media/stagefright/MediaCodec.h>
40#include <media/stagefright/MediaCodecConstants.h>
41#include <media/MediaCodecBuffer.h>
42#include <system/window.h>
43
44#include "CCodecBufferChannel.h"
45#include "Codec2Buffer.h"
46#include "SkipCutBuffer.h"
47
48namespace android {
49
50using android::base::StringPrintf;
51using hardware::hidl_handle;
52using hardware::hidl_string;
53using hardware::hidl_vec;
54using namespace hardware::cas::V1_0;
55using namespace hardware::cas::native::V1_0;
56
57using CasStatus = hardware::cas::V1_0::Status;
58
59/**
60 * Base class for representation of buffers at one port.
61 */
62class CCodecBufferChannel::Buffers {
63public:
64 Buffers(const char *componentName, const char *name = "Buffers")
65 : mComponentName(componentName),
66 mChannelName(std::string(componentName) + ":" + name),
67 mName(mChannelName.c_str()) {
68 }
69 virtual ~Buffers() = default;
70
71 /**
72 * Set format for MediaCodec-facing buffers.
73 */
74 void setFormat(const sp<AMessage> &format) {
75 CHECK(format != nullptr);
76 mFormat = format;
77 }
78
79 /**
80 * Return a copy of current format.
81 */
82 sp<AMessage> dupFormat() {
83 return mFormat != nullptr ? mFormat->dup() : nullptr;
84 }
85
86 /**
87 * Returns true if the buffers are operating under array mode.
88 */
89 virtual bool isArrayMode() const { return false; }
90
91 /**
92 * Fills the vector with MediaCodecBuffer's if in array mode; otherwise,
93 * no-op.
94 */
95 virtual void getArray(Vector<sp<MediaCodecBuffer>> *) const {}
96
97protected:
98 std::string mComponentName; ///< name of component for debugging
99 std::string mChannelName; ///< name of channel for debugging
100 const char *mName; ///< C-string version of channel name
101 // Format to be used for creating MediaCodec-facing buffers.
102 sp<AMessage> mFormat;
103
104private:
105 DISALLOW_EVIL_CONSTRUCTORS(Buffers);
106};
107
108class CCodecBufferChannel::InputBuffers : public CCodecBufferChannel::Buffers {
109public:
110 InputBuffers(const char *componentName, const char *name = "Input[]")
111 : Buffers(componentName, name) { }
112 virtual ~InputBuffers() = default;
113
114 /**
115 * Set a block pool to obtain input memory blocks.
116 */
117 void setPool(const std::shared_ptr<C2BlockPool> &pool) { mPool = pool; }
118
119 /**
120 * Get a new MediaCodecBuffer for input and its corresponding index.
121 * Returns false if no new buffer can be obtained at the moment.
122 */
123 virtual bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) = 0;
124
125 /**
126 * Release the buffer obtained from requestNewBuffer() and get the
127 * associated C2Buffer object back. Returns true if the buffer was on file
128 * and released successfully.
129 */
130 virtual bool releaseBuffer(
131 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) = 0;
132
133 /**
134 * Release the buffer that is no longer used by the codec process. Return
135 * true if and only if the buffer was on file and released successfully.
136 */
137 virtual bool expireComponentBuffer(
138 const std::shared_ptr<C2Buffer> &c2buffer) = 0;
139
140 /**
141 * Flush internal state. After this call, no index or buffer previously
142 * returned from requestNewBuffer() is valid.
143 */
144 virtual void flush() = 0;
145
146 /**
147 * Return array-backed version of input buffers. The returned object
148 * shall retain the internal state so that it will honor index and
149 * buffer from previous calls of requestNewBuffer().
150 */
151 virtual std::unique_ptr<InputBuffers> toArrayMode(size_t size) = 0;
152
153protected:
154 // Pool to obtain blocks for input buffers.
155 std::shared_ptr<C2BlockPool> mPool;
156
157private:
158 DISALLOW_EVIL_CONSTRUCTORS(InputBuffers);
159};
160
161class CCodecBufferChannel::OutputBuffers : public CCodecBufferChannel::Buffers {
162public:
163 OutputBuffers(const char *componentName, const char *name = "Output")
164 : Buffers(componentName, name) { }
165 virtual ~OutputBuffers() = default;
166
167 /**
168 * Register output C2Buffer from the component and obtain corresponding
169 * index and MediaCodecBuffer object. Returns false if registration
170 * fails.
171 */
172 virtual status_t registerBuffer(
173 const std::shared_ptr<C2Buffer> &buffer,
174 size_t *index,
175 sp<MediaCodecBuffer> *clientBuffer) = 0;
176
177 /**
178 * Register codec specific data as a buffer to be consistent with
179 * MediaCodec behavior.
180 */
181 virtual status_t registerCsd(
182 const C2StreamCsdInfo::output * /* csd */,
183 size_t * /* index */,
184 sp<MediaCodecBuffer> * /* clientBuffer */) = 0;
185
186 /**
187 * Release the buffer obtained from registerBuffer() and get the
188 * associated C2Buffer object back. Returns true if the buffer was on file
189 * and released successfully.
190 */
191 virtual bool releaseBuffer(
192 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) = 0;
193
194 /**
195 * Flush internal state. After this call, no index or buffer previously
196 * returned from registerBuffer() is valid.
197 */
198 virtual void flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) = 0;
199
200 /**
201 * Return array-backed version of output buffers. The returned object
202 * shall retain the internal state so that it will honor index and
203 * buffer from previous calls of registerBuffer().
204 */
205 virtual std::unique_ptr<OutputBuffers> toArrayMode(size_t size) = 0;
206
207 /**
208 * Initialize SkipCutBuffer object.
209 */
210 void initSkipCutBuffer(
211 int32_t delay, int32_t padding, int32_t sampleRate, int32_t channelCount) {
212 CHECK(mSkipCutBuffer == nullptr);
213 mDelay = delay;
214 mPadding = padding;
215 mSampleRate = sampleRate;
216 setSkipCutBuffer(delay, padding, channelCount);
217 }
218
219 /**
220 * Update the SkipCutBuffer object. No-op if it's never initialized.
221 */
222 void updateSkipCutBuffer(int32_t sampleRate, int32_t channelCount) {
223 if (mSkipCutBuffer == nullptr) {
224 return;
225 }
226 int32_t delay = mDelay;
227 int32_t padding = mPadding;
228 if (sampleRate != mSampleRate) {
229 delay = ((int64_t)delay * sampleRate) / mSampleRate;
230 padding = ((int64_t)padding * sampleRate) / mSampleRate;
231 }
232 setSkipCutBuffer(delay, padding, channelCount);
233 }
234
235 /**
236 * Submit buffer to SkipCutBuffer object, if initialized.
237 */
238 void submit(const sp<MediaCodecBuffer> &buffer) {
239 if (mSkipCutBuffer != nullptr) {
240 mSkipCutBuffer->submit(buffer);
241 }
242 }
243
244 /**
245 * Transfer SkipCutBuffer object to the other Buffers object.
246 */
247 void transferSkipCutBuffer(const sp<SkipCutBuffer> &scb) {
248 mSkipCutBuffer = scb;
249 }
250
251protected:
252 sp<SkipCutBuffer> mSkipCutBuffer;
253
254private:
255 int32_t mDelay;
256 int32_t mPadding;
257 int32_t mSampleRate;
258
259 void setSkipCutBuffer(int32_t skip, int32_t cut, int32_t channelCount) {
260 if (mSkipCutBuffer != nullptr) {
261 size_t prevSize = mSkipCutBuffer->size();
262 if (prevSize != 0u) {
263 ALOGD("[%s] Replacing SkipCutBuffer holding %zu bytes", mName, prevSize);
264 }
265 }
266 mSkipCutBuffer = new SkipCutBuffer(skip, cut, channelCount);
267 }
268
269 DISALLOW_EVIL_CONSTRUCTORS(OutputBuffers);
270};
271
272namespace {
273
Wonsik Kim078b58e2019-01-09 15:08:06 -0800274const static size_t kSmoothnessFactor = 4;
275const static size_t kRenderingDepth = 3;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800276const static size_t kLinearBufferSize = 1048576;
277// This can fit 4K RGBA frame, and most likely client won't need more than this.
278const static size_t kMaxLinearBufferSize = 3840 * 2160 * 4;
279
280/**
281 * Simple local buffer pool backed by std::vector.
282 */
283class LocalBufferPool : public std::enable_shared_from_this<LocalBufferPool> {
284public:
285 /**
286 * Create a new LocalBufferPool object.
287 *
288 * \param poolCapacity max total size of buffers managed by this pool.
289 *
290 * \return a newly created pool object.
291 */
292 static std::shared_ptr<LocalBufferPool> Create(size_t poolCapacity) {
293 return std::shared_ptr<LocalBufferPool>(new LocalBufferPool(poolCapacity));
294 }
295
296 /**
297 * Return an ABuffer object whose size is at least |capacity|.
298 *
299 * \param capacity requested capacity
300 * \return nullptr if the pool capacity is reached
301 * an ABuffer object otherwise.
302 */
303 sp<ABuffer> newBuffer(size_t capacity) {
304 Mutex::Autolock lock(mMutex);
305 auto it = std::find_if(
306 mPool.begin(), mPool.end(),
307 [capacity](const std::vector<uint8_t> &vec) {
308 return vec.capacity() >= capacity;
309 });
310 if (it != mPool.end()) {
311 sp<ABuffer> buffer = new VectorBuffer(std::move(*it), shared_from_this());
312 mPool.erase(it);
313 return buffer;
314 }
315 if (mUsedSize + capacity > mPoolCapacity) {
316 while (!mPool.empty()) {
317 mUsedSize -= mPool.back().capacity();
318 mPool.pop_back();
319 }
320 if (mUsedSize + capacity > mPoolCapacity) {
321 ALOGD("mUsedSize = %zu, capacity = %zu, mPoolCapacity = %zu",
322 mUsedSize, capacity, mPoolCapacity);
323 return nullptr;
324 }
325 }
326 std::vector<uint8_t> vec(capacity);
327 mUsedSize += vec.capacity();
328 return new VectorBuffer(std::move(vec), shared_from_this());
329 }
330
331private:
332 /**
333 * ABuffer backed by std::vector.
334 */
335 class VectorBuffer : public ::android::ABuffer {
336 public:
337 /**
338 * Construct a VectorBuffer by taking the ownership of supplied vector.
339 *
340 * \param vec backing vector of the buffer. this object takes
341 * ownership at construction.
342 * \param pool a LocalBufferPool object to return the vector at
343 * destruction.
344 */
345 VectorBuffer(std::vector<uint8_t> &&vec, const std::shared_ptr<LocalBufferPool> &pool)
346 : ABuffer(vec.data(), vec.capacity()),
347 mVec(std::move(vec)),
348 mPool(pool) {
349 }
350
351 ~VectorBuffer() override {
352 std::shared_ptr<LocalBufferPool> pool = mPool.lock();
353 if (pool) {
354 // If pool is alive, return the vector back to the pool so that
355 // it can be recycled.
356 pool->returnVector(std::move(mVec));
357 }
358 }
359
360 private:
361 std::vector<uint8_t> mVec;
362 std::weak_ptr<LocalBufferPool> mPool;
363 };
364
365 Mutex mMutex;
366 size_t mPoolCapacity;
367 size_t mUsedSize;
368 std::list<std::vector<uint8_t>> mPool;
369
370 /**
371 * Private constructor to prevent constructing non-managed LocalBufferPool.
372 */
373 explicit LocalBufferPool(size_t poolCapacity)
374 : mPoolCapacity(poolCapacity), mUsedSize(0) {
375 }
376
377 /**
378 * Take back the ownership of vec from the destructed VectorBuffer and put
379 * it in front of the pool.
380 */
381 void returnVector(std::vector<uint8_t> &&vec) {
382 Mutex::Autolock lock(mMutex);
383 mPool.push_front(std::move(vec));
384 }
385
386 DISALLOW_EVIL_CONSTRUCTORS(LocalBufferPool);
387};
388
389sp<GraphicBlockBuffer> AllocateGraphicBuffer(
390 const std::shared_ptr<C2BlockPool> &pool,
391 const sp<AMessage> &format,
392 uint32_t pixelFormat,
393 const C2MemoryUsage &usage,
394 const std::shared_ptr<LocalBufferPool> &localBufferPool) {
395 int32_t width, height;
396 if (!format->findInt32("width", &width) || !format->findInt32("height", &height)) {
397 ALOGD("format lacks width or height");
398 return nullptr;
399 }
400
401 std::shared_ptr<C2GraphicBlock> block;
402 c2_status_t err = pool->fetchGraphicBlock(
403 width, height, pixelFormat, usage, &block);
404 if (err != C2_OK) {
405 ALOGD("fetch graphic block failed: %d", err);
406 return nullptr;
407 }
408
409 return GraphicBlockBuffer::Allocate(
410 format,
411 block,
412 [localBufferPool](size_t capacity) {
413 return localBufferPool->newBuffer(capacity);
414 });
415}
416
417class BuffersArrayImpl;
418
419/**
420 * Flexible buffer slots implementation.
421 */
422class FlexBuffersImpl {
423public:
424 FlexBuffersImpl(const char *name)
425 : mImplName(std::string(name) + ".Impl"),
426 mName(mImplName.c_str()) { }
427
428 /**
429 * Assign an empty slot for a buffer and return the index. If there's no
430 * empty slot, just add one at the end and return it.
431 *
432 * \param buffer[in] a new buffer to assign a slot.
433 * \return index of the assigned slot.
434 */
435 size_t assignSlot(const sp<Codec2Buffer> &buffer) {
436 for (size_t i = 0; i < mBuffers.size(); ++i) {
437 if (mBuffers[i].clientBuffer == nullptr
438 && mBuffers[i].compBuffer.expired()) {
439 mBuffers[i].clientBuffer = buffer;
440 return i;
441 }
442 }
443 mBuffers.push_back({ buffer, std::weak_ptr<C2Buffer>() });
444 return mBuffers.size() - 1;
445 }
446
447 /**
448 * Release the slot from the client, and get the C2Buffer object back from
449 * the previously assigned buffer. Note that the slot is not completely free
450 * until the returned C2Buffer object is freed.
451 *
452 * \param buffer[in] the buffer previously assigned a slot.
453 * \param c2buffer[in,out] pointer to C2Buffer to be populated. Ignored
454 * if null.
455 * \return true if the buffer is successfully released from a slot
456 * false otherwise
457 */
458 bool releaseSlot(const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) {
459 sp<Codec2Buffer> clientBuffer;
460 size_t index = mBuffers.size();
461 for (size_t i = 0; i < mBuffers.size(); ++i) {
462 if (mBuffers[i].clientBuffer == buffer) {
463 clientBuffer = mBuffers[i].clientBuffer;
464 mBuffers[i].clientBuffer.clear();
465 index = i;
466 break;
467 }
468 }
469 if (clientBuffer == nullptr) {
470 ALOGV("[%s] %s: No matching buffer found", mName, __func__);
471 return false;
472 }
473 std::shared_ptr<C2Buffer> result = clientBuffer->asC2Buffer();
474 mBuffers[index].compBuffer = result;
475 if (c2buffer) {
476 *c2buffer = result;
477 }
478 return true;
479 }
480
481 bool expireComponentBuffer(const std::shared_ptr<C2Buffer> &c2buffer) {
482 for (size_t i = 0; i < mBuffers.size(); ++i) {
483 std::shared_ptr<C2Buffer> compBuffer =
484 mBuffers[i].compBuffer.lock();
485 if (!compBuffer || compBuffer != c2buffer) {
486 continue;
487 }
488 mBuffers[i].clientBuffer = nullptr;
489 mBuffers[i].compBuffer.reset();
490 return true;
491 }
492 ALOGV("[%s] codec released an unknown buffer", mName);
493 return false;
494 }
495
496 void flush() {
497 ALOGV("[%s] buffers are flushed %zu", mName, mBuffers.size());
498 mBuffers.clear();
499 }
500
501private:
502 friend class BuffersArrayImpl;
503
504 std::string mImplName; ///< name for debugging
505 const char *mName; ///< C-string version of name
506
507 struct Entry {
508 sp<Codec2Buffer> clientBuffer;
509 std::weak_ptr<C2Buffer> compBuffer;
510 };
511 std::vector<Entry> mBuffers;
512};
513
514/**
515 * Static buffer slots implementation based on a fixed-size array.
516 */
517class BuffersArrayImpl {
518public:
519 BuffersArrayImpl()
520 : mImplName("BuffersArrayImpl"),
521 mName(mImplName.c_str()) { }
522
523 /**
524 * Initialize buffer array from the original |impl|. The buffers known by
525 * the client is preserved, and the empty slots are populated so that the
526 * array size is at least |minSize|.
527 *
528 * \param impl[in] FlexBuffersImpl object used so far.
529 * \param minSize[in] minimum size of the buffer array.
530 * \param allocate[in] function to allocate a client buffer for an empty slot.
531 */
532 void initialize(
533 const FlexBuffersImpl &impl,
534 size_t minSize,
535 std::function<sp<Codec2Buffer>()> allocate) {
536 mImplName = impl.mImplName + "[N]";
537 mName = mImplName.c_str();
538 for (size_t i = 0; i < impl.mBuffers.size(); ++i) {
539 sp<Codec2Buffer> clientBuffer = impl.mBuffers[i].clientBuffer;
540 bool ownedByClient = (clientBuffer != nullptr);
541 if (!ownedByClient) {
542 clientBuffer = allocate();
543 }
544 mBuffers.push_back({ clientBuffer, impl.mBuffers[i].compBuffer, ownedByClient });
545 }
546 ALOGV("[%s] converted %zu buffers to array mode of %zu", mName, mBuffers.size(), minSize);
547 for (size_t i = impl.mBuffers.size(); i < minSize; ++i) {
548 mBuffers.push_back({ allocate(), std::weak_ptr<C2Buffer>(), false });
549 }
550 }
551
552 /**
553 * Grab a buffer from the underlying array which matches the criteria.
554 *
555 * \param index[out] index of the slot.
556 * \param buffer[out] the matching buffer.
557 * \param match[in] a function to test whether the buffer matches the
558 * criteria or not.
559 * \return OK if successful,
560 * WOULD_BLOCK if slots are being used,
561 * NO_MEMORY if no slot matches the criteria, even though it's
562 * available
563 */
564 status_t grabBuffer(
565 size_t *index,
566 sp<Codec2Buffer> *buffer,
567 std::function<bool(const sp<Codec2Buffer> &)> match =
568 [](const sp<Codec2Buffer> &) { return true; }) {
569 // allBuffersDontMatch remains true if all buffers are available but
570 // match() returns false for every buffer.
571 bool allBuffersDontMatch = true;
572 for (size_t i = 0; i < mBuffers.size(); ++i) {
573 if (!mBuffers[i].ownedByClient && mBuffers[i].compBuffer.expired()) {
574 if (match(mBuffers[i].clientBuffer)) {
575 mBuffers[i].ownedByClient = true;
576 *buffer = mBuffers[i].clientBuffer;
577 (*buffer)->meta()->clear();
578 (*buffer)->setRange(0, (*buffer)->capacity());
579 *index = i;
580 return OK;
581 }
582 } else {
583 allBuffersDontMatch = false;
584 }
585 }
586 return allBuffersDontMatch ? NO_MEMORY : WOULD_BLOCK;
587 }
588
589 /**
590 * Return the buffer from the client, and get the C2Buffer object back from
591 * the buffer. Note that the slot is not completely free until the returned
592 * C2Buffer object is freed.
593 *
594 * \param buffer[in] the buffer previously grabbed.
595 * \param c2buffer[in,out] pointer to C2Buffer to be populated. Ignored
596 * if null.
597 * \return true if the buffer is successfully returned
598 * false otherwise
599 */
600 bool returnBuffer(const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) {
601 sp<Codec2Buffer> clientBuffer;
602 size_t index = mBuffers.size();
603 for (size_t i = 0; i < mBuffers.size(); ++i) {
604 if (mBuffers[i].clientBuffer == buffer) {
605 if (!mBuffers[i].ownedByClient) {
606 ALOGD("[%s] Client returned a buffer it does not own according to our record: %zu", mName, i);
607 }
608 clientBuffer = mBuffers[i].clientBuffer;
609 mBuffers[i].ownedByClient = false;
610 index = i;
611 break;
612 }
613 }
614 if (clientBuffer == nullptr) {
615 ALOGV("[%s] %s: No matching buffer found", mName, __func__);
616 return false;
617 }
618 ALOGV("[%s] %s: matching buffer found (index=%zu)", mName, __func__, index);
619 std::shared_ptr<C2Buffer> result = clientBuffer->asC2Buffer();
620 mBuffers[index].compBuffer = result;
621 if (c2buffer) {
622 *c2buffer = result;
623 }
624 return true;
625 }
626
627 bool expireComponentBuffer(const std::shared_ptr<C2Buffer> &c2buffer) {
628 for (size_t i = 0; i < mBuffers.size(); ++i) {
629 std::shared_ptr<C2Buffer> compBuffer =
630 mBuffers[i].compBuffer.lock();
631 if (!compBuffer) {
632 continue;
633 }
634 if (c2buffer == compBuffer) {
635 if (mBuffers[i].ownedByClient) {
636 // This should not happen.
637 ALOGD("[%s] codec released a buffer owned by client "
638 "(index %zu)", mName, i);
639 mBuffers[i].ownedByClient = false;
640 }
641 mBuffers[i].compBuffer.reset();
642 return true;
643 }
644 }
645 ALOGV("[%s] codec released an unknown buffer (array mode)", mName);
646 return false;
647 }
648
649 /**
650 * Populate |array| with the underlying buffer array.
651 *
652 * \param array[out] an array to be filled with the underlying buffer array.
653 */
654 void getArray(Vector<sp<MediaCodecBuffer>> *array) const {
655 array->clear();
656 for (const Entry &entry : mBuffers) {
657 array->push(entry.clientBuffer);
658 }
659 }
660
661 /**
662 * The client abandoned all known buffers, so reclaim the ownership.
663 */
664 void flush() {
665 for (Entry &entry : mBuffers) {
666 entry.ownedByClient = false;
667 }
668 }
669
670 void realloc(std::function<sp<Codec2Buffer>()> alloc) {
671 size_t size = mBuffers.size();
672 mBuffers.clear();
673 for (size_t i = 0; i < size; ++i) {
674 mBuffers.push_back({ alloc(), std::weak_ptr<C2Buffer>(), false });
675 }
676 }
677
678private:
679 std::string mImplName; ///< name for debugging
680 const char *mName; ///< C-string version of name
681
682 struct Entry {
683 const sp<Codec2Buffer> clientBuffer;
684 std::weak_ptr<C2Buffer> compBuffer;
685 bool ownedByClient;
686 };
687 std::vector<Entry> mBuffers;
688};
689
690class InputBuffersArray : public CCodecBufferChannel::InputBuffers {
691public:
692 InputBuffersArray(const char *componentName, const char *name = "Input[N]")
693 : InputBuffers(componentName, name) { }
694 ~InputBuffersArray() override = default;
695
696 void initialize(
697 const FlexBuffersImpl &impl,
698 size_t minSize,
699 std::function<sp<Codec2Buffer>()> allocate) {
700 mImpl.initialize(impl, minSize, allocate);
701 }
702
703 bool isArrayMode() const final { return true; }
704
705 std::unique_ptr<CCodecBufferChannel::InputBuffers> toArrayMode(
706 size_t) final {
707 return nullptr;
708 }
709
710 void getArray(Vector<sp<MediaCodecBuffer>> *array) const final {
711 mImpl.getArray(array);
712 }
713
714 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override {
715 sp<Codec2Buffer> c2Buffer;
716 status_t err = mImpl.grabBuffer(index, &c2Buffer);
717 if (err == OK) {
718 c2Buffer->setFormat(mFormat);
719 *buffer = c2Buffer;
720 return true;
721 }
722 return false;
723 }
724
725 bool releaseBuffer(
726 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) override {
727 return mImpl.returnBuffer(buffer, c2buffer);
728 }
729
730 bool expireComponentBuffer(
731 const std::shared_ptr<C2Buffer> &c2buffer) override {
732 return mImpl.expireComponentBuffer(c2buffer);
733 }
734
735 void flush() override {
736 mImpl.flush();
737 }
738
739private:
740 BuffersArrayImpl mImpl;
741};
742
743class LinearInputBuffers : public CCodecBufferChannel::InputBuffers {
744public:
745 LinearInputBuffers(const char *componentName, const char *name = "1D-Input")
746 : InputBuffers(componentName, name),
747 mImpl(mName) { }
748
749 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override {
750 int32_t capacity = kLinearBufferSize;
751 (void)mFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
752 if ((size_t)capacity > kMaxLinearBufferSize) {
753 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
754 capacity = kMaxLinearBufferSize;
755 }
756 // TODO: proper max input size
757 // TODO: read usage from intf
758 sp<Codec2Buffer> newBuffer = alloc((size_t)capacity);
759 if (newBuffer == nullptr) {
760 return false;
761 }
762 *index = mImpl.assignSlot(newBuffer);
763 *buffer = newBuffer;
764 return true;
765 }
766
767 bool releaseBuffer(
768 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) override {
769 return mImpl.releaseSlot(buffer, c2buffer);
770 }
771
772 bool expireComponentBuffer(
773 const std::shared_ptr<C2Buffer> &c2buffer) override {
774 return mImpl.expireComponentBuffer(c2buffer);
775 }
776
777 void flush() override {
778 // This is no-op by default unless we're in array mode where we need to keep
779 // track of the flushed work.
780 mImpl.flush();
781 }
782
783 std::unique_ptr<CCodecBufferChannel::InputBuffers> toArrayMode(
784 size_t size) final {
785 int32_t capacity = kLinearBufferSize;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800786 (void)mFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
787 if ((size_t)capacity > kMaxLinearBufferSize) {
788 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
789 capacity = kMaxLinearBufferSize;
790 }
791 // TODO: proper max input size
792 // TODO: read usage from intf
Pawin Vongmasa36653902018-11-15 00:10:25 -0800793 std::unique_ptr<InputBuffersArray> array(
794 new InputBuffersArray(mComponentName.c_str(), "1D-Input[N]"));
795 array->setPool(mPool);
796 array->setFormat(mFormat);
797 array->initialize(
798 mImpl,
799 size,
800 [this, capacity] () -> sp<Codec2Buffer> { return alloc(capacity); });
801 return std::move(array);
802 }
803
804 virtual sp<Codec2Buffer> alloc(size_t size) const {
805 C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
806 std::shared_ptr<C2LinearBlock> block;
807
808 c2_status_t err = mPool->fetchLinearBlock(size, usage, &block);
809 if (err != C2_OK) {
810 return nullptr;
811 }
812
813 return LinearBlockBuffer::Allocate(mFormat, block);
814 }
815
816private:
817 FlexBuffersImpl mImpl;
818};
819
820class EncryptedLinearInputBuffers : public LinearInputBuffers {
821public:
822 EncryptedLinearInputBuffers(
823 bool secure,
824 const sp<MemoryDealer> &dealer,
825 const sp<ICrypto> &crypto,
826 int32_t heapSeqNum,
827 size_t capacity,
Wonsik Kim078b58e2019-01-09 15:08:06 -0800828 size_t numInputSlots,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800829 const char *componentName, const char *name = "EncryptedInput")
830 : LinearInputBuffers(componentName, name),
831 mUsage({0, 0}),
832 mDealer(dealer),
833 mCrypto(crypto),
834 mHeapSeqNum(heapSeqNum) {
835 if (secure) {
836 mUsage = { C2MemoryUsage::READ_PROTECTED, 0 };
837 } else {
838 mUsage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
839 }
Wonsik Kim078b58e2019-01-09 15:08:06 -0800840 for (size_t i = 0; i < numInputSlots; ++i) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800841 sp<IMemory> memory = mDealer->allocate(capacity);
842 if (memory == nullptr) {
843 ALOGD("[%s] Failed to allocate memory from dealer: only %zu slots allocated", mName, i);
844 break;
845 }
846 mMemoryVector.push_back({std::weak_ptr<C2LinearBlock>(), memory});
847 }
848 }
849
850 ~EncryptedLinearInputBuffers() override {
851 }
852
853 sp<Codec2Buffer> alloc(size_t size) const override {
854 sp<IMemory> memory;
855 for (const Entry &entry : mMemoryVector) {
856 if (entry.block.expired()) {
857 memory = entry.memory;
858 break;
859 }
860 }
861 if (memory == nullptr) {
862 return nullptr;
863 }
864
865 std::shared_ptr<C2LinearBlock> block;
866 c2_status_t err = mPool->fetchLinearBlock(size, mUsage, &block);
867 if (err != C2_OK) {
868 return nullptr;
869 }
870
871 return new EncryptedLinearBlockBuffer(mFormat, block, memory, mHeapSeqNum);
872 }
873
874private:
875 C2MemoryUsage mUsage;
876 sp<MemoryDealer> mDealer;
877 sp<ICrypto> mCrypto;
878 int32_t mHeapSeqNum;
879 struct Entry {
880 std::weak_ptr<C2LinearBlock> block;
881 sp<IMemory> memory;
882 };
883 std::vector<Entry> mMemoryVector;
884};
885
886class GraphicMetadataInputBuffers : public CCodecBufferChannel::InputBuffers {
887public:
888 GraphicMetadataInputBuffers(const char *componentName, const char *name = "2D-MetaInput")
889 : InputBuffers(componentName, name),
890 mImpl(mName),
891 mStore(GetCodec2PlatformAllocatorStore()) { }
892 ~GraphicMetadataInputBuffers() override = default;
893
894 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override {
895 std::shared_ptr<C2Allocator> alloc;
896 c2_status_t err = mStore->fetchAllocator(mPool->getAllocatorId(), &alloc);
897 if (err != C2_OK) {
898 return false;
899 }
900 sp<GraphicMetadataBuffer> newBuffer = new GraphicMetadataBuffer(mFormat, alloc);
901 if (newBuffer == nullptr) {
902 return false;
903 }
904 *index = mImpl.assignSlot(newBuffer);
905 *buffer = newBuffer;
906 return true;
907 }
908
909 bool releaseBuffer(
910 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) override {
911 return mImpl.releaseSlot(buffer, c2buffer);
912 }
913
914 bool expireComponentBuffer(
915 const std::shared_ptr<C2Buffer> &c2buffer) override {
916 return mImpl.expireComponentBuffer(c2buffer);
917 }
918
919 void flush() override {
920 // This is no-op by default unless we're in array mode where we need to keep
921 // track of the flushed work.
922 }
923
924 std::unique_ptr<CCodecBufferChannel::InputBuffers> toArrayMode(
925 size_t size) final {
926 std::shared_ptr<C2Allocator> alloc;
927 c2_status_t err = mStore->fetchAllocator(mPool->getAllocatorId(), &alloc);
928 if (err != C2_OK) {
929 return nullptr;
930 }
931 std::unique_ptr<InputBuffersArray> array(
932 new InputBuffersArray(mComponentName.c_str(), "2D-MetaInput[N]"));
933 array->setPool(mPool);
934 array->setFormat(mFormat);
935 array->initialize(
936 mImpl,
937 size,
938 [format = mFormat, alloc]() -> sp<Codec2Buffer> {
939 return new GraphicMetadataBuffer(format, alloc);
940 });
941 return std::move(array);
942 }
943
944private:
945 FlexBuffersImpl mImpl;
946 std::shared_ptr<C2AllocatorStore> mStore;
947};
948
949class GraphicInputBuffers : public CCodecBufferChannel::InputBuffers {
950public:
Wonsik Kim078b58e2019-01-09 15:08:06 -0800951 GraphicInputBuffers(
952 size_t numInputSlots, const char *componentName, const char *name = "2D-BB-Input")
Pawin Vongmasa36653902018-11-15 00:10:25 -0800953 : InputBuffers(componentName, name),
954 mImpl(mName),
955 mLocalBufferPool(LocalBufferPool::Create(
Wonsik Kim078b58e2019-01-09 15:08:06 -0800956 kMaxLinearBufferSize * numInputSlots)) { }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800957 ~GraphicInputBuffers() override = default;
958
959 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override {
960 // TODO: proper max input size
961 // TODO: read usage from intf
962 C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
963 sp<GraphicBlockBuffer> newBuffer = AllocateGraphicBuffer(
964 mPool, mFormat, HAL_PIXEL_FORMAT_YV12, usage, mLocalBufferPool);
965 if (newBuffer == nullptr) {
966 return false;
967 }
968 *index = mImpl.assignSlot(newBuffer);
969 *buffer = newBuffer;
970 return true;
971 }
972
973 bool releaseBuffer(
974 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) override {
975 return mImpl.releaseSlot(buffer, c2buffer);
976 }
977
978 bool expireComponentBuffer(
979 const std::shared_ptr<C2Buffer> &c2buffer) override {
980 return mImpl.expireComponentBuffer(c2buffer);
981 }
982 void flush() override {
983 // This is no-op by default unless we're in array mode where we need to keep
984 // track of the flushed work.
985 }
986
987 std::unique_ptr<CCodecBufferChannel::InputBuffers> toArrayMode(
988 size_t size) final {
989 std::unique_ptr<InputBuffersArray> array(
990 new InputBuffersArray(mComponentName.c_str(), "2D-BB-Input[N]"));
991 array->setPool(mPool);
992 array->setFormat(mFormat);
993 array->initialize(
994 mImpl,
995 size,
996 [pool = mPool, format = mFormat, lbp = mLocalBufferPool]() -> sp<Codec2Buffer> {
997 C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
998 return AllocateGraphicBuffer(
999 pool, format, HAL_PIXEL_FORMAT_YV12, usage, lbp);
1000 });
1001 return std::move(array);
1002 }
1003
1004private:
1005 FlexBuffersImpl mImpl;
1006 std::shared_ptr<LocalBufferPool> mLocalBufferPool;
1007};
1008
1009class DummyInputBuffers : public CCodecBufferChannel::InputBuffers {
1010public:
1011 DummyInputBuffers(const char *componentName, const char *name = "2D-Input")
1012 : InputBuffers(componentName, name) { }
1013
1014 bool requestNewBuffer(size_t *, sp<MediaCodecBuffer> *) override {
1015 return false;
1016 }
1017
1018 bool releaseBuffer(
1019 const sp<MediaCodecBuffer> &, std::shared_ptr<C2Buffer> *) override {
1020 return false;
1021 }
1022
1023 bool expireComponentBuffer(const std::shared_ptr<C2Buffer> &) override {
1024 return false;
1025 }
1026 void flush() override {
1027 }
1028
1029 std::unique_ptr<CCodecBufferChannel::InputBuffers> toArrayMode(
1030 size_t) final {
1031 return nullptr;
1032 }
1033
1034 bool isArrayMode() const final { return true; }
1035
1036 void getArray(Vector<sp<MediaCodecBuffer>> *array) const final {
1037 array->clear();
1038 }
1039};
1040
1041class OutputBuffersArray : public CCodecBufferChannel::OutputBuffers {
1042public:
1043 OutputBuffersArray(const char *componentName, const char *name = "Output[N]")
1044 : OutputBuffers(componentName, name) { }
1045 ~OutputBuffersArray() override = default;
1046
1047 void initialize(
1048 const FlexBuffersImpl &impl,
1049 size_t minSize,
1050 std::function<sp<Codec2Buffer>()> allocate) {
1051 mImpl.initialize(impl, minSize, allocate);
1052 }
1053
1054 bool isArrayMode() const final { return true; }
1055
1056 std::unique_ptr<CCodecBufferChannel::OutputBuffers> toArrayMode(
1057 size_t) final {
1058 return nullptr;
1059 }
1060
1061 status_t registerBuffer(
1062 const std::shared_ptr<C2Buffer> &buffer,
1063 size_t *index,
1064 sp<MediaCodecBuffer> *clientBuffer) final {
1065 sp<Codec2Buffer> c2Buffer;
1066 status_t err = mImpl.grabBuffer(
1067 index,
1068 &c2Buffer,
1069 [buffer](const sp<Codec2Buffer> &clientBuffer) {
1070 return clientBuffer->canCopy(buffer);
1071 });
1072 if (err == WOULD_BLOCK) {
1073 ALOGV("[%s] buffers temporarily not available", mName);
1074 return err;
1075 } else if (err != OK) {
1076 ALOGD("[%s] grabBuffer failed: %d", mName, err);
1077 return err;
1078 }
1079 c2Buffer->setFormat(mFormat);
1080 if (!c2Buffer->copy(buffer)) {
1081 ALOGD("[%s] copy buffer failed", mName);
1082 return WOULD_BLOCK;
1083 }
1084 submit(c2Buffer);
1085 *clientBuffer = c2Buffer;
1086 ALOGV("[%s] grabbed buffer %zu", mName, *index);
1087 return OK;
1088 }
1089
1090 status_t registerCsd(
1091 const C2StreamCsdInfo::output *csd,
1092 size_t *index,
1093 sp<MediaCodecBuffer> *clientBuffer) final {
1094 sp<Codec2Buffer> c2Buffer;
1095 status_t err = mImpl.grabBuffer(
1096 index,
1097 &c2Buffer,
1098 [csd](const sp<Codec2Buffer> &clientBuffer) {
1099 return clientBuffer->base() != nullptr
1100 && clientBuffer->capacity() >= csd->flexCount();
1101 });
1102 if (err != OK) {
1103 return err;
1104 }
1105 memcpy(c2Buffer->base(), csd->m.value, csd->flexCount());
1106 c2Buffer->setRange(0, csd->flexCount());
1107 c2Buffer->setFormat(mFormat);
1108 *clientBuffer = c2Buffer;
1109 return OK;
1110 }
1111
1112 bool releaseBuffer(
1113 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) override {
1114 return mImpl.returnBuffer(buffer, c2buffer);
1115 }
1116
1117 void flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) override {
1118 (void)flushedWork;
1119 mImpl.flush();
1120 if (mSkipCutBuffer != nullptr) {
1121 mSkipCutBuffer->clear();
1122 }
1123 }
1124
1125 void getArray(Vector<sp<MediaCodecBuffer>> *array) const final {
1126 mImpl.getArray(array);
1127 }
1128
1129 void realloc(const std::shared_ptr<C2Buffer> &c2buffer) {
1130 std::function<sp<Codec2Buffer>()> alloc;
1131 switch (c2buffer->data().type()) {
1132 case C2BufferData::LINEAR: {
1133 uint32_t size = kLinearBufferSize;
1134 const C2ConstLinearBlock &block = c2buffer->data().linearBlocks().front();
1135 if (block.size() < kMaxLinearBufferSize / 2) {
1136 size = block.size() * 2;
1137 } else {
1138 size = kMaxLinearBufferSize;
1139 }
1140 alloc = [format = mFormat, size] {
1141 return new LocalLinearBuffer(format, new ABuffer(size));
1142 };
1143 break;
1144 }
1145
1146 // TODO: add support
1147 case C2BufferData::GRAPHIC: FALLTHROUGH_INTENDED;
1148
1149 case C2BufferData::INVALID: FALLTHROUGH_INTENDED;
1150 case C2BufferData::LINEAR_CHUNKS: FALLTHROUGH_INTENDED;
1151 case C2BufferData::GRAPHIC_CHUNKS: FALLTHROUGH_INTENDED;
1152 default:
1153 ALOGD("Unsupported type: %d", (int)c2buffer->data().type());
1154 return;
1155 }
1156 mImpl.realloc(alloc);
1157 }
1158
1159private:
1160 BuffersArrayImpl mImpl;
1161};
1162
1163class FlexOutputBuffers : public CCodecBufferChannel::OutputBuffers {
1164public:
1165 FlexOutputBuffers(const char *componentName, const char *name = "Output[]")
1166 : OutputBuffers(componentName, name),
1167 mImpl(mName) { }
1168
1169 status_t registerBuffer(
1170 const std::shared_ptr<C2Buffer> &buffer,
1171 size_t *index,
1172 sp<MediaCodecBuffer> *clientBuffer) override {
1173 sp<Codec2Buffer> newBuffer = wrap(buffer);
Wonsik Kim186fdbf2019-01-29 13:30:01 -08001174 if (newBuffer == nullptr) {
1175 return NO_MEMORY;
1176 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001177 newBuffer->setFormat(mFormat);
1178 *index = mImpl.assignSlot(newBuffer);
1179 *clientBuffer = newBuffer;
1180 ALOGV("[%s] registered buffer %zu", mName, *index);
1181 return OK;
1182 }
1183
1184 status_t registerCsd(
1185 const C2StreamCsdInfo::output *csd,
1186 size_t *index,
1187 sp<MediaCodecBuffer> *clientBuffer) final {
1188 sp<Codec2Buffer> newBuffer = new LocalLinearBuffer(
1189 mFormat, ABuffer::CreateAsCopy(csd->m.value, csd->flexCount()));
1190 *index = mImpl.assignSlot(newBuffer);
1191 *clientBuffer = newBuffer;
1192 return OK;
1193 }
1194
1195 bool releaseBuffer(
1196 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) override {
1197 return mImpl.releaseSlot(buffer, c2buffer);
1198 }
1199
1200 void flush(
1201 const std::list<std::unique_ptr<C2Work>> &flushedWork) override {
1202 (void) flushedWork;
1203 // This is no-op by default unless we're in array mode where we need to keep
1204 // track of the flushed work.
1205 }
1206
1207 std::unique_ptr<CCodecBufferChannel::OutputBuffers> toArrayMode(
1208 size_t size) override {
1209 std::unique_ptr<OutputBuffersArray> array(new OutputBuffersArray(mComponentName.c_str()));
1210 array->setFormat(mFormat);
1211 array->transferSkipCutBuffer(mSkipCutBuffer);
1212 array->initialize(
1213 mImpl,
1214 size,
1215 [this]() { return allocateArrayBuffer(); });
1216 return std::move(array);
1217 }
1218
1219 /**
1220 * Return an appropriate Codec2Buffer object for the type of buffers.
1221 *
1222 * \param buffer C2Buffer object to wrap.
1223 *
1224 * \return appropriate Codec2Buffer object to wrap |buffer|.
1225 */
1226 virtual sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) = 0;
1227
1228 /**
1229 * Return an appropriate Codec2Buffer object for the type of buffers, to be
1230 * used as an empty array buffer.
1231 *
1232 * \return appropriate Codec2Buffer object which can copy() from C2Buffers.
1233 */
1234 virtual sp<Codec2Buffer> allocateArrayBuffer() = 0;
1235
1236private:
1237 FlexBuffersImpl mImpl;
1238};
1239
1240class LinearOutputBuffers : public FlexOutputBuffers {
1241public:
1242 LinearOutputBuffers(const char *componentName, const char *name = "1D-Output")
1243 : FlexOutputBuffers(componentName, name) { }
1244
1245 void flush(
1246 const std::list<std::unique_ptr<C2Work>> &flushedWork) override {
1247 if (mSkipCutBuffer != nullptr) {
1248 mSkipCutBuffer->clear();
1249 }
1250 FlexOutputBuffers::flush(flushedWork);
1251 }
1252
1253 sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) override {
1254 if (buffer == nullptr) {
1255 ALOGV("[%s] using a dummy buffer", mName);
1256 return new LocalLinearBuffer(mFormat, new ABuffer(0));
1257 }
1258 if (buffer->data().type() != C2BufferData::LINEAR) {
1259 ALOGV("[%s] non-linear buffer %d", mName, buffer->data().type());
1260 // We expect linear output buffers from the component.
1261 return nullptr;
1262 }
1263 if (buffer->data().linearBlocks().size() != 1u) {
1264 ALOGV("[%s] no linear buffers", mName);
1265 // We expect one and only one linear block from the component.
1266 return nullptr;
1267 }
1268 sp<Codec2Buffer> clientBuffer = ConstLinearBlockBuffer::Allocate(mFormat, buffer);
Wonsik Kim186fdbf2019-01-29 13:30:01 -08001269 if (clientBuffer == nullptr) {
1270 ALOGD("[%s] ConstLinearBlockBuffer::Allocate failed", mName);
1271 return nullptr;
1272 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001273 submit(clientBuffer);
1274 return clientBuffer;
1275 }
1276
1277 sp<Codec2Buffer> allocateArrayBuffer() override {
1278 // TODO: proper max output size
1279 return new LocalLinearBuffer(mFormat, new ABuffer(kLinearBufferSize));
1280 }
1281};
1282
1283class GraphicOutputBuffers : public FlexOutputBuffers {
1284public:
1285 GraphicOutputBuffers(const char *componentName, const char *name = "2D-Output")
1286 : FlexOutputBuffers(componentName, name) { }
1287
1288 sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) override {
1289 return new DummyContainerBuffer(mFormat, buffer);
1290 }
1291
1292 sp<Codec2Buffer> allocateArrayBuffer() override {
1293 return new DummyContainerBuffer(mFormat);
1294 }
1295};
1296
1297class RawGraphicOutputBuffers : public FlexOutputBuffers {
1298public:
Wonsik Kim078b58e2019-01-09 15:08:06 -08001299 RawGraphicOutputBuffers(
1300 size_t numOutputSlots, const char *componentName, const char *name = "2D-BB-Output")
Pawin Vongmasa36653902018-11-15 00:10:25 -08001301 : FlexOutputBuffers(componentName, name),
1302 mLocalBufferPool(LocalBufferPool::Create(
Wonsik Kim078b58e2019-01-09 15:08:06 -08001303 kMaxLinearBufferSize * numOutputSlots)) { }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001304 ~RawGraphicOutputBuffers() override = default;
1305
1306 sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) override {
1307 if (buffer == nullptr) {
1308 sp<Codec2Buffer> c2buffer = ConstGraphicBlockBuffer::AllocateEmpty(
1309 mFormat,
1310 [lbp = mLocalBufferPool](size_t capacity) {
1311 return lbp->newBuffer(capacity);
1312 });
Wonsik Kim186fdbf2019-01-29 13:30:01 -08001313 if (c2buffer == nullptr) {
1314 ALOGD("[%s] ConstGraphicBlockBuffer::AllocateEmpty failed", mName);
1315 return nullptr;
1316 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001317 c2buffer->setRange(0, 0);
1318 return c2buffer;
1319 } else {
1320 return ConstGraphicBlockBuffer::Allocate(
1321 mFormat,
1322 buffer,
1323 [lbp = mLocalBufferPool](size_t capacity) {
1324 return lbp->newBuffer(capacity);
1325 });
1326 }
1327 }
1328
1329 sp<Codec2Buffer> allocateArrayBuffer() override {
1330 return ConstGraphicBlockBuffer::AllocateEmpty(
1331 mFormat,
1332 [lbp = mLocalBufferPool](size_t capacity) {
1333 return lbp->newBuffer(capacity);
1334 });
1335 }
1336
1337private:
1338 std::shared_ptr<LocalBufferPool> mLocalBufferPool;
1339};
1340
1341} // namespace
1342
1343CCodecBufferChannel::QueueGuard::QueueGuard(
1344 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
1345 Mutex::Autolock l(mSync.mGuardLock);
1346 // At this point it's guaranteed that mSync is not under state transition,
1347 // as we are holding its mutex.
1348
1349 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
1350 if (count->value == -1) {
1351 mRunning = false;
1352 } else {
1353 ++count->value;
1354 mRunning = true;
1355 }
1356}
1357
1358CCodecBufferChannel::QueueGuard::~QueueGuard() {
1359 if (mRunning) {
1360 // We are not holding mGuardLock at this point so that QueueSync::stop() can
1361 // keep holding the lock until mCount reaches zero.
1362 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
1363 --count->value;
1364 count->cond.broadcast();
1365 }
1366}
1367
1368void CCodecBufferChannel::QueueSync::start() {
1369 Mutex::Autolock l(mGuardLock);
1370 // If stopped, it goes to running state; otherwise no-op.
1371 Mutexed<Counter>::Locked count(mCount);
1372 if (count->value == -1) {
1373 count->value = 0;
1374 }
1375}
1376
1377void CCodecBufferChannel::QueueSync::stop() {
1378 Mutex::Autolock l(mGuardLock);
1379 Mutexed<Counter>::Locked count(mCount);
1380 if (count->value == -1) {
1381 // no-op
1382 return;
1383 }
1384 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
1385 // mCount can only decrement. In other words, threads that acquired the lock
1386 // are allowed to finish execution but additional threads trying to acquire
1387 // the lock at this point will block, and then get QueueGuard at STOPPED
1388 // state.
1389 while (count->value != 0) {
1390 count.waitForCondition(count->cond);
1391 }
1392 count->value = -1;
1393}
1394
1395// CCodecBufferChannel::PipelineCapacity
1396
1397CCodecBufferChannel::PipelineCapacity::PipelineCapacity()
1398 : input(0), component(0),
1399 mName("<UNKNOWN COMPONENT>") {
1400}
1401
1402void CCodecBufferChannel::PipelineCapacity::initialize(
1403 int newInput,
1404 int newComponent,
1405 const char* newName,
1406 const char* callerTag) {
1407 input.store(newInput, std::memory_order_relaxed);
1408 component.store(newComponent, std::memory_order_relaxed);
1409 mName = newName;
1410 ALOGV("[%s] %s -- PipelineCapacity::initialize(): "
1411 "pipeline availability initialized ==> "
1412 "input = %d, component = %d",
1413 mName, callerTag ? callerTag : "*",
1414 newInput, newComponent);
1415}
1416
1417bool CCodecBufferChannel::PipelineCapacity::allocate(const char* callerTag) {
1418 int prevInput = input.fetch_sub(1, std::memory_order_relaxed);
1419 int prevComponent = component.fetch_sub(1, std::memory_order_relaxed);
1420 if (prevInput > 0 && prevComponent > 0) {
1421 ALOGV("[%s] %s -- PipelineCapacity::allocate() returns true: "
1422 "pipeline availability -1 all ==> "
1423 "input = %d, component = %d",
1424 mName, callerTag ? callerTag : "*",
1425 prevInput - 1,
1426 prevComponent - 1);
1427 return true;
1428 }
1429 input.fetch_add(1, std::memory_order_relaxed);
1430 component.fetch_add(1, std::memory_order_relaxed);
1431 ALOGV("[%s] %s -- PipelineCapacity::allocate() returns false: "
1432 "pipeline availability unchanged ==> "
1433 "input = %d, component = %d",
1434 mName, callerTag ? callerTag : "*",
1435 prevInput,
1436 prevComponent);
1437 return false;
1438}
1439
1440void CCodecBufferChannel::PipelineCapacity::free(const char* callerTag) {
1441 int prevInput = input.fetch_add(1, std::memory_order_relaxed);
1442 int prevComponent = component.fetch_add(1, std::memory_order_relaxed);
1443 ALOGV("[%s] %s -- PipelineCapacity::free(): "
1444 "pipeline availability +1 all ==> "
1445 "input = %d, component = %d",
1446 mName, callerTag ? callerTag : "*",
1447 prevInput + 1,
1448 prevComponent + 1);
1449}
1450
1451int CCodecBufferChannel::PipelineCapacity::freeInputSlots(
1452 size_t numDiscardedInputBuffers,
1453 const char* callerTag) {
1454 int prevInput = input.fetch_add(numDiscardedInputBuffers,
1455 std::memory_order_relaxed);
1456 ALOGV("[%s] %s -- PipelineCapacity::freeInputSlots(%zu): "
1457 "pipeline availability +%zu input ==> "
1458 "input = %d, component = %d",
1459 mName, callerTag ? callerTag : "*",
1460 numDiscardedInputBuffers,
1461 numDiscardedInputBuffers,
1462 prevInput + static_cast<int>(numDiscardedInputBuffers),
1463 component.load(std::memory_order_relaxed));
1464 return prevInput + static_cast<int>(numDiscardedInputBuffers);
1465}
1466
1467int CCodecBufferChannel::PipelineCapacity::freeComponentSlot(
1468 const char* callerTag) {
1469 int prevComponent = component.fetch_add(1, std::memory_order_relaxed);
1470 ALOGV("[%s] %s -- PipelineCapacity::freeComponentSlot(): "
1471 "pipeline availability +1 component ==> "
1472 "input = %d, component = %d",
1473 mName, callerTag ? callerTag : "*",
1474 input.load(std::memory_order_relaxed),
1475 prevComponent + 1);
1476 return prevComponent + 1;
1477}
1478
1479// CCodecBufferChannel::ReorderStash
1480
1481CCodecBufferChannel::ReorderStash::ReorderStash() {
1482 clear();
1483}
1484
1485void CCodecBufferChannel::ReorderStash::clear() {
1486 mPending.clear();
1487 mStash.clear();
1488 mDepth = 0;
1489 mKey = C2Config::ORDINAL;
1490}
1491
1492void CCodecBufferChannel::ReorderStash::setDepth(uint32_t depth) {
1493 mPending.splice(mPending.end(), mStash);
1494 mDepth = depth;
1495}
1496void CCodecBufferChannel::ReorderStash::setKey(C2Config::ordinal_key_t key) {
1497 mPending.splice(mPending.end(), mStash);
1498 mKey = key;
1499}
1500
1501bool CCodecBufferChannel::ReorderStash::pop(Entry *entry) {
1502 if (mPending.empty()) {
1503 return false;
1504 }
1505 entry->buffer = mPending.front().buffer;
1506 entry->timestamp = mPending.front().timestamp;
1507 entry->flags = mPending.front().flags;
1508 entry->ordinal = mPending.front().ordinal;
1509 mPending.pop_front();
1510 return true;
1511}
1512
1513void CCodecBufferChannel::ReorderStash::emplace(
1514 const std::shared_ptr<C2Buffer> &buffer,
1515 int64_t timestamp,
1516 int32_t flags,
1517 const C2WorkOrdinalStruct &ordinal) {
1518 for (auto it = mStash.begin(); it != mStash.end(); ++it) {
1519 if (less(ordinal, it->ordinal)) {
1520 mStash.emplace(it, buffer, timestamp, flags, ordinal);
1521 return;
1522 }
1523 }
1524 mStash.emplace_back(buffer, timestamp, flags, ordinal);
1525 while (!mStash.empty() && mStash.size() > mDepth) {
1526 mPending.push_back(mStash.front());
1527 mStash.pop_front();
1528 }
1529}
1530
1531void CCodecBufferChannel::ReorderStash::defer(
1532 const CCodecBufferChannel::ReorderStash::Entry &entry) {
1533 mPending.push_front(entry);
1534}
1535
1536bool CCodecBufferChannel::ReorderStash::hasPending() const {
1537 return !mPending.empty();
1538}
1539
1540bool CCodecBufferChannel::ReorderStash::less(
1541 const C2WorkOrdinalStruct &o1, const C2WorkOrdinalStruct &o2) {
1542 switch (mKey) {
1543 case C2Config::ORDINAL: return o1.frameIndex < o2.frameIndex;
1544 case C2Config::TIMESTAMP: return o1.timestamp < o2.timestamp;
1545 case C2Config::CUSTOM: return o1.customOrdinal < o2.customOrdinal;
1546 default:
1547 ALOGD("Unrecognized key; default to timestamp");
1548 return o1.frameIndex < o2.frameIndex;
1549 }
1550}
1551
1552// CCodecBufferChannel
1553
1554CCodecBufferChannel::CCodecBufferChannel(
1555 const std::shared_ptr<CCodecCallback> &callback)
1556 : mHeapSeqNum(-1),
1557 mCCodecCallback(callback),
Wonsik Kim078b58e2019-01-09 15:08:06 -08001558 mNumInputSlots(kSmoothnessFactor),
1559 mNumOutputSlots(kSmoothnessFactor),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001560 mFrameIndex(0u),
1561 mFirstValidFrameIndex(0u),
1562 mMetaMode(MODE_NONE),
1563 mAvailablePipelineCapacity(),
1564 mInputMetEos(false) {
1565 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
1566 buffers->reset(new DummyInputBuffers(""));
1567}
1568
1569CCodecBufferChannel::~CCodecBufferChannel() {
1570 if (mCrypto != nullptr && mDealer != nullptr && mHeapSeqNum >= 0) {
1571 mCrypto->unsetHeap(mHeapSeqNum);
1572 }
1573}
1574
1575void CCodecBufferChannel::setComponent(
1576 const std::shared_ptr<Codec2Client::Component> &component) {
1577 mComponent = component;
1578 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
1579 mName = mComponentName.c_str();
1580}
1581
1582status_t CCodecBufferChannel::setInputSurface(
1583 const std::shared_ptr<InputSurfaceWrapper> &surface) {
1584 ALOGV("[%s] setInputSurface", mName);
1585 mInputSurface = surface;
1586 return mInputSurface->connect(mComponent);
1587}
1588
1589status_t CCodecBufferChannel::signalEndOfInputStream() {
1590 if (mInputSurface == nullptr) {
1591 return INVALID_OPERATION;
1592 }
1593 return mInputSurface->signalEndOfInputStream();
1594}
1595
1596status_t CCodecBufferChannel::queueInputBufferInternal(const sp<MediaCodecBuffer> &buffer) {
1597 int64_t timeUs;
1598 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1599
1600 if (mInputMetEos) {
1601 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
1602 return OK;
1603 }
1604
1605 int32_t flags = 0;
1606 int32_t tmp = 0;
1607 bool eos = false;
1608 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
1609 eos = true;
1610 mInputMetEos = true;
1611 ALOGV("[%s] input EOS", mName);
1612 }
1613 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
1614 flags |= C2FrameData::FLAG_CODEC_CONFIG;
1615 }
1616 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
1617 std::unique_ptr<C2Work> work(new C2Work);
1618 work->input.ordinal.timestamp = timeUs;
1619 work->input.ordinal.frameIndex = mFrameIndex++;
1620 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
1621 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
1622 // Keep client timestamp in customOrdinal
1623 work->input.ordinal.customOrdinal = timeUs;
1624 work->input.buffers.clear();
1625
1626 if (buffer->size() > 0u) {
1627 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
1628 std::shared_ptr<C2Buffer> c2buffer;
1629 if (!(*buffers)->releaseBuffer(buffer, &c2buffer)) {
1630 return -ENOENT;
1631 }
1632 work->input.buffers.push_back(c2buffer);
1633 } else {
1634 mAvailablePipelineCapacity.freeInputSlots(1, "queueInputBufferInternal");
1635 if (eos) {
1636 flags |= C2FrameData::FLAG_END_OF_STREAM;
1637 }
1638 }
1639 work->input.flags = (C2FrameData::flags_t)flags;
1640 // TODO: fill info's
1641
1642 work->input.configUpdate = std::move(mParamsToBeSet);
1643 work->worklets.clear();
1644 work->worklets.emplace_back(new C2Worklet);
1645
1646 std::list<std::unique_ptr<C2Work>> items;
1647 items.push_back(std::move(work));
1648 c2_status_t err = mComponent->queue(&items);
1649
1650 if (err == C2_OK && eos && buffer->size() > 0u) {
1651 mCCodecCallback->onWorkQueued(false);
1652 work.reset(new C2Work);
1653 work->input.ordinal.timestamp = timeUs;
1654 work->input.ordinal.frameIndex = mFrameIndex++;
1655 // WORKAROUND: keep client timestamp in customOrdinal
1656 work->input.ordinal.customOrdinal = timeUs;
1657 work->input.buffers.clear();
1658 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
1659
1660 items.clear();
1661 items.push_back(std::move(work));
1662 err = mComponent->queue(&items);
1663 }
1664 if (err == C2_OK) {
1665 mCCodecCallback->onWorkQueued(eos);
1666 }
1667
1668 feedInputBufferIfAvailableInternal();
1669 return err;
1670}
1671
1672status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> &params) {
1673 QueueGuard guard(mSync);
1674 if (!guard.isRunning()) {
1675 ALOGD("[%s] setParameters is only supported in the running state.", mName);
1676 return -ENOSYS;
1677 }
1678 mParamsToBeSet.insert(mParamsToBeSet.end(),
1679 std::make_move_iterator(params.begin()),
1680 std::make_move_iterator(params.end()));
1681 params.clear();
1682 return OK;
1683}
1684
1685status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
1686 QueueGuard guard(mSync);
1687 if (!guard.isRunning()) {
1688 ALOGD("[%s] No more buffers should be queued at current state.", mName);
1689 return -ENOSYS;
1690 }
1691 return queueInputBufferInternal(buffer);
1692}
1693
1694status_t CCodecBufferChannel::queueSecureInputBuffer(
1695 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
1696 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
1697 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
1698 AString *errorDetailMsg) {
1699 QueueGuard guard(mSync);
1700 if (!guard.isRunning()) {
1701 ALOGD("[%s] No more buffers should be queued at current state.", mName);
1702 return -ENOSYS;
1703 }
1704
1705 if (!hasCryptoOrDescrambler()) {
1706 return -ENOSYS;
1707 }
1708 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
1709
1710 ssize_t result = -1;
1711 ssize_t codecDataOffset = 0;
1712 if (mCrypto != nullptr) {
1713 ICrypto::DestinationBuffer destination;
1714 if (secure) {
1715 destination.mType = ICrypto::kDestinationTypeNativeHandle;
1716 destination.mHandle = encryptedBuffer->handle();
1717 } else {
1718 destination.mType = ICrypto::kDestinationTypeSharedMemory;
1719 destination.mSharedMemory = mDecryptDestination;
1720 }
1721 ICrypto::SourceBuffer source;
1722 encryptedBuffer->fillSourceBuffer(&source);
1723 result = mCrypto->decrypt(
1724 key, iv, mode, pattern, source, buffer->offset(),
1725 subSamples, numSubSamples, destination, errorDetailMsg);
1726 if (result < 0) {
1727 return result;
1728 }
1729 if (destination.mType == ICrypto::kDestinationTypeSharedMemory) {
1730 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
1731 }
1732 } else {
1733 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
1734 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
1735 hidl_vec<SubSample> hidlSubSamples;
1736 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
1737
1738 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
1739 encryptedBuffer->fillSourceBuffer(&srcBuffer);
1740
1741 DestinationBuffer dstBuffer;
1742 if (secure) {
1743 dstBuffer.type = BufferType::NATIVE_HANDLE;
1744 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
1745 } else {
1746 dstBuffer.type = BufferType::SHARED_MEMORY;
1747 dstBuffer.nonsecureMemory = srcBuffer;
1748 }
1749
1750 CasStatus status = CasStatus::OK;
1751 hidl_string detailedError;
1752 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
1753
1754 if (key != nullptr) {
1755 sctrl = (ScramblingControl)key[0];
1756 // Adjust for the PES offset
1757 codecDataOffset = key[2] | (key[3] << 8);
1758 }
1759
1760 auto returnVoid = mDescrambler->descramble(
1761 sctrl,
1762 hidlSubSamples,
1763 srcBuffer,
1764 0,
1765 dstBuffer,
1766 0,
1767 [&status, &result, &detailedError] (
1768 CasStatus _status, uint32_t _bytesWritten,
1769 const hidl_string& _detailedError) {
1770 status = _status;
1771 result = (ssize_t)_bytesWritten;
1772 detailedError = _detailedError;
1773 });
1774
1775 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
1776 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
1777 mName, returnVoid.description().c_str(), status, result);
1778 return UNKNOWN_ERROR;
1779 }
1780
1781 if (result < codecDataOffset) {
1782 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
1783 return BAD_VALUE;
1784 }
1785
1786 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
1787
1788 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
1789 encryptedBuffer->copyDecryptedContentFromMemory(result);
1790 }
1791 }
1792
1793 buffer->setRange(codecDataOffset, result - codecDataOffset);
1794 return queueInputBufferInternal(buffer);
1795}
1796
1797void CCodecBufferChannel::feedInputBufferIfAvailable() {
1798 QueueGuard guard(mSync);
1799 if (!guard.isRunning()) {
1800 ALOGV("[%s] We're not running --- no input buffer reported", mName);
1801 return;
1802 }
1803 feedInputBufferIfAvailableInternal();
1804}
1805
1806void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
1807 while (!mInputMetEos &&
1808 !mReorderStash.lock()->hasPending() &&
1809 mAvailablePipelineCapacity.allocate("feedInputBufferIfAvailable")) {
1810 sp<MediaCodecBuffer> inBuffer;
1811 size_t index;
1812 {
1813 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
1814 if (!(*buffers)->requestNewBuffer(&index, &inBuffer)) {
1815 ALOGV("[%s] no new buffer available", mName);
1816 mAvailablePipelineCapacity.free("feedInputBufferIfAvailable");
1817 break;
1818 }
1819 }
1820 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
1821 mCallback->onInputBufferAvailable(index, inBuffer);
1822 }
1823}
1824
1825status_t CCodecBufferChannel::renderOutputBuffer(
1826 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001827 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001828 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001829 bool released = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001830 {
1831 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
1832 if (*buffers) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001833 released = (*buffers)->releaseBuffer(buffer, &c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001834 }
1835 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001836 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
1837 // set to true.
1838 sendOutputBuffers();
1839 // input buffer feeding may have been gated by pending output buffers
1840 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001841 if (!c2Buffer) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001842 if (released) {
1843 ALOGD("[%s] The app is calling releaseOutputBuffer() with "
1844 "timestamp or render=true with non-video buffers. Apps should "
1845 "call releaseOutputBuffer() with render=false for those.",
1846 mName);
1847 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001848 return INVALID_OPERATION;
1849 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001850
1851#if 0
1852 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
1853 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
1854 for (const std::shared_ptr<const C2Info> &info : infoParams) {
1855 AString res;
1856 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
1857 if (ix) res.append(", ");
1858 res.append(*((int32_t*)info.get() + (ix / 4)));
1859 }
1860 ALOGV(" [%s]", res.c_str());
1861 }
1862#endif
1863 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
1864 std::static_pointer_cast<const C2StreamRotationInfo::output>(
1865 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
1866 bool flip = rotation && (rotation->flip & 1);
1867 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
1868 uint32_t transform = 0;
1869 switch (quarters) {
1870 case 0: // no rotation
1871 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
1872 break;
1873 case 1: // 90 degrees counter-clockwise
1874 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
1875 : HAL_TRANSFORM_ROT_270;
1876 break;
1877 case 2: // 180 degrees
1878 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
1879 break;
1880 case 3: // 90 degrees clockwise
1881 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
1882 : HAL_TRANSFORM_ROT_90;
1883 break;
1884 }
1885
1886 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
1887 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
1888 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
1889 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
1890 if (surfaceScaling) {
1891 videoScalingMode = surfaceScaling->value;
1892 }
1893
1894 // Use dataspace from format as it has the default aspects already applied
1895 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
1896 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
1897
1898 // HDR static info
1899 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
1900 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
1901 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
1902
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001903 // HDR10 plus info
1904 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
1905 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
1906 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
1907
Pawin Vongmasa36653902018-11-15 00:10:25 -08001908 {
1909 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1910 if (output->surface == nullptr) {
1911 ALOGI("[%s] cannot render buffer without surface", mName);
1912 return OK;
1913 }
1914 }
1915
1916 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
1917 if (blocks.size() != 1u) {
1918 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
1919 return UNKNOWN_ERROR;
1920 }
1921 const C2ConstGraphicBlock &block = blocks.front();
1922
1923 // TODO: revisit this after C2Fence implementation.
1924 android::IGraphicBufferProducer::QueueBufferInput qbi(
1925 timestampNs,
1926 false, // droppable
1927 dataSpace,
1928 Rect(blocks.front().crop().left,
1929 blocks.front().crop().top,
1930 blocks.front().crop().right(),
1931 blocks.front().crop().bottom()),
1932 videoScalingMode,
1933 transform,
1934 Fence::NO_FENCE, 0);
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001935 if (hdrStaticInfo || hdr10PlusInfo) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001936 HdrMetadata hdr;
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001937 if (hdrStaticInfo) {
1938 struct android_smpte2086_metadata smpte2086_meta = {
1939 .displayPrimaryRed = {
1940 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
1941 },
1942 .displayPrimaryGreen = {
1943 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
1944 },
1945 .displayPrimaryBlue = {
1946 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
1947 },
1948 .whitePoint = {
1949 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
1950 },
1951 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
1952 .minLuminance = hdrStaticInfo->mastering.minLuminance,
1953 };
1954
1955 struct android_cta861_3_metadata cta861_meta = {
1956 .maxContentLightLevel = hdrStaticInfo->maxCll,
1957 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
1958 };
1959
1960 hdr.validTypes = HdrMetadata::SMPTE2086 | HdrMetadata::CTA861_3;
1961 hdr.smpte2086 = smpte2086_meta;
1962 hdr.cta8613 = cta861_meta;
1963 }
1964 if (hdr10PlusInfo) {
1965 hdr.validTypes |= HdrMetadata::HDR10PLUS;
1966 hdr.hdr10plus.assign(
1967 hdr10PlusInfo->m.value,
1968 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
1969 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001970 qbi.setHdrMetadata(hdr);
1971 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -08001972 // we don't have dirty regions
1973 qbi.setSurfaceDamage(Region::INVALID_REGION);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001974 android::IGraphicBufferProducer::QueueBufferOutput qbo;
1975 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
1976 if (result != OK) {
1977 ALOGI("[%s] queueBuffer failed: %d", mName, result);
1978 return result;
1979 }
1980 ALOGV("[%s] queue buffer successful", mName);
1981
1982 int64_t mediaTimeUs = 0;
1983 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
1984 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
1985
1986 return OK;
1987}
1988
1989status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
1990 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
1991 bool released = false;
1992 {
1993 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
1994 if (*buffers && (*buffers)->releaseBuffer(buffer, nullptr)) {
1995 buffers.unlock();
1996 released = true;
1997 mAvailablePipelineCapacity.freeInputSlots(1, "discardBuffer");
1998 }
1999 }
2000 {
2001 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2002 if (*buffers && (*buffers)->releaseBuffer(buffer, nullptr)) {
2003 buffers.unlock();
2004 released = true;
2005 }
2006 }
2007 if (released) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002008 sendOutputBuffers();
Pawin Vongmasa8be93112018-12-11 14:01:42 -08002009 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -08002010 } else {
2011 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
2012 }
2013 return OK;
2014}
2015
2016void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
2017 array->clear();
2018 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
2019
2020 if (!(*buffers)->isArrayMode()) {
Wonsik Kim078b58e2019-01-09 15:08:06 -08002021 *buffers = (*buffers)->toArrayMode(mNumInputSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002022 }
2023
2024 (*buffers)->getArray(array);
2025}
2026
2027void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
2028 array->clear();
2029 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2030
2031 if (!(*buffers)->isArrayMode()) {
Wonsik Kim078b58e2019-01-09 15:08:06 -08002032 *buffers = (*buffers)->toArrayMode(mNumOutputSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002033 }
2034
2035 (*buffers)->getArray(array);
2036}
2037
2038status_t CCodecBufferChannel::start(
2039 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) {
2040 C2StreamBufferTypeSetting::input iStreamFormat(0u);
2041 C2StreamBufferTypeSetting::output oStreamFormat(0u);
2042 C2PortReorderBufferDepthTuning::output reorderDepth;
2043 C2PortReorderKeySetting::output reorderKey;
Wonsik Kim078b58e2019-01-09 15:08:06 -08002044 C2PortActualDelayTuning::input inputDelay(0);
2045 C2PortActualDelayTuning::output outputDelay(0);
2046 C2ActualPipelineDelayTuning pipelineDelay(0);
2047
Pawin Vongmasa36653902018-11-15 00:10:25 -08002048 c2_status_t err = mComponent->query(
2049 {
2050 &iStreamFormat,
2051 &oStreamFormat,
2052 &reorderDepth,
2053 &reorderKey,
Wonsik Kim078b58e2019-01-09 15:08:06 -08002054 &inputDelay,
2055 &pipelineDelay,
2056 &outputDelay,
Pawin Vongmasa36653902018-11-15 00:10:25 -08002057 },
2058 {},
2059 C2_DONT_BLOCK,
2060 nullptr);
2061 if (err == C2_BAD_INDEX) {
2062 if (!iStreamFormat || !oStreamFormat) {
2063 return UNKNOWN_ERROR;
2064 }
2065 } else if (err != C2_OK) {
2066 return UNKNOWN_ERROR;
2067 }
2068
2069 {
2070 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
2071 reorder->clear();
2072 if (reorderDepth) {
2073 reorder->setDepth(reorderDepth.value);
2074 }
2075 if (reorderKey) {
2076 reorder->setKey(reorderKey.value);
2077 }
2078 }
Wonsik Kim078b58e2019-01-09 15:08:06 -08002079
2080 mNumInputSlots =
2081 (inputDelay ? inputDelay.value : 0) +
2082 (pipelineDelay ? pipelineDelay.value : 0) +
2083 kSmoothnessFactor;
2084 mNumOutputSlots = (outputDelay ? outputDelay.value : 0) + kSmoothnessFactor;
2085
Pawin Vongmasa36653902018-11-15 00:10:25 -08002086 // TODO: get this from input format
2087 bool secure = mComponent->getName().find(".secure") != std::string::npos;
2088
2089 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
2090 int poolMask = property_get_int32(
2091 "debug.stagefright.c2-poolmask",
2092 1 << C2PlatformAllocatorStore::ION |
2093 1 << C2PlatformAllocatorStore::BUFFERQUEUE);
2094
2095 if (inputFormat != nullptr) {
2096 bool graphic = (iStreamFormat.value == C2FormatVideo);
2097 std::shared_ptr<C2BlockPool> pool;
2098 {
2099 Mutexed<BlockPools>::Locked pools(mBlockPools);
2100
2101 // set default allocator ID.
2102 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
2103 : C2PlatformAllocatorStore::ION;
2104
2105 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
2106 // from component, create the input block pool with given ID. Otherwise, use default IDs.
2107 std::vector<std::unique_ptr<C2Param>> params;
2108 err = mComponent->query({ },
2109 { C2PortAllocatorsTuning::input::PARAM_TYPE },
2110 C2_DONT_BLOCK,
2111 &params);
2112 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
2113 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
2114 mName, params.size(), asString(err), err);
2115 } else if (err == C2_OK && params.size() == 1) {
2116 C2PortAllocatorsTuning::input *inputAllocators =
2117 C2PortAllocatorsTuning::input::From(params[0].get());
2118 if (inputAllocators && inputAllocators->flexCount() > 0) {
2119 std::shared_ptr<C2Allocator> allocator;
2120 // verify allocator IDs and resolve default allocator
2121 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
2122 if (allocator) {
2123 pools->inputAllocatorId = allocator->getId();
2124 } else {
2125 ALOGD("[%s] component requested invalid input allocator ID %u",
2126 mName, inputAllocators->m.values[0]);
2127 }
2128 }
2129 }
2130
2131 // TODO: use C2Component wrapper to associate this pool with ourselves
2132 if ((poolMask >> pools->inputAllocatorId) & 1) {
2133 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
2134 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
2135 mName, pools->inputAllocatorId,
2136 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
2137 asString(err), err);
2138 } else {
2139 err = C2_NOT_FOUND;
2140 }
2141 if (err != C2_OK) {
2142 C2BlockPool::local_id_t inputPoolId =
2143 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
2144 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
2145 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
2146 mName, (unsigned long long)inputPoolId,
2147 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
2148 asString(err), err);
2149 if (err != C2_OK) {
2150 return NO_MEMORY;
2151 }
2152 }
2153 pools->inputPool = pool;
2154 }
2155
Wonsik Kim51051262018-11-28 13:59:05 -08002156 bool forceArrayMode = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002157 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
2158 if (graphic) {
2159 if (mInputSurface) {
2160 buffers->reset(new DummyInputBuffers(mName));
2161 } else if (mMetaMode == MODE_ANW) {
2162 buffers->reset(new GraphicMetadataInputBuffers(mName));
2163 } else {
Wonsik Kim078b58e2019-01-09 15:08:06 -08002164 buffers->reset(new GraphicInputBuffers(mNumInputSlots, mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08002165 }
2166 } else {
2167 if (hasCryptoOrDescrambler()) {
2168 int32_t capacity = kLinearBufferSize;
2169 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
2170 if ((size_t)capacity > kMaxLinearBufferSize) {
2171 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
2172 capacity = kMaxLinearBufferSize;
2173 }
2174 if (mDealer == nullptr) {
2175 mDealer = new MemoryDealer(
2176 align(capacity, MemoryDealer::getAllocationAlignment())
Wonsik Kim078b58e2019-01-09 15:08:06 -08002177 * (mNumInputSlots + 1),
Pawin Vongmasa36653902018-11-15 00:10:25 -08002178 "EncryptedLinearInputBuffers");
2179 mDecryptDestination = mDealer->allocate((size_t)capacity);
2180 }
2181 if (mCrypto != nullptr && mHeapSeqNum < 0) {
2182 mHeapSeqNum = mCrypto->setHeap(mDealer->getMemoryHeap());
2183 } else {
2184 mHeapSeqNum = -1;
2185 }
2186 buffers->reset(new EncryptedLinearInputBuffers(
Wonsik Kim078b58e2019-01-09 15:08:06 -08002187 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
2188 mNumInputSlots, mName));
Wonsik Kim51051262018-11-28 13:59:05 -08002189 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002190 } else {
2191 buffers->reset(new LinearInputBuffers(mName));
2192 }
2193 }
2194 (*buffers)->setFormat(inputFormat);
2195
2196 if (err == C2_OK) {
2197 (*buffers)->setPool(pool);
2198 } else {
2199 // TODO: error
2200 }
Wonsik Kim51051262018-11-28 13:59:05 -08002201
2202 if (forceArrayMode) {
Wonsik Kim078b58e2019-01-09 15:08:06 -08002203 *buffers = (*buffers)->toArrayMode(mNumInputSlots);
Wonsik Kim51051262018-11-28 13:59:05 -08002204 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002205 }
2206
2207 if (outputFormat != nullptr) {
2208 sp<IGraphicBufferProducer> outputSurface;
2209 uint32_t outputGeneration;
2210 {
2211 Mutexed<OutputSurface>::Locked output(mOutputSurface);
2212 outputSurface = output->surface ?
2213 output->surface->getIGraphicBufferProducer() : nullptr;
2214 outputGeneration = output->generation;
2215 }
2216
2217 bool graphic = (oStreamFormat.value == C2FormatVideo);
2218 C2BlockPool::local_id_t outputPoolId_;
2219
2220 {
2221 Mutexed<BlockPools>::Locked pools(mBlockPools);
2222
2223 // set default allocator ID.
2224 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
2225 : C2PlatformAllocatorStore::ION;
2226
2227 // query C2PortAllocatorsTuning::output from component, or use default allocator if
2228 // unsuccessful.
2229 std::vector<std::unique_ptr<C2Param>> params;
2230 err = mComponent->query({ },
2231 { C2PortAllocatorsTuning::output::PARAM_TYPE },
2232 C2_DONT_BLOCK,
2233 &params);
2234 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
2235 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
2236 mName, params.size(), asString(err), err);
2237 } else if (err == C2_OK && params.size() == 1) {
2238 C2PortAllocatorsTuning::output *outputAllocators =
2239 C2PortAllocatorsTuning::output::From(params[0].get());
2240 if (outputAllocators && outputAllocators->flexCount() > 0) {
2241 std::shared_ptr<C2Allocator> allocator;
2242 // verify allocator IDs and resolve default allocator
2243 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
2244 if (allocator) {
2245 pools->outputAllocatorId = allocator->getId();
2246 } else {
2247 ALOGD("[%s] component requested invalid output allocator ID %u",
2248 mName, outputAllocators->m.values[0]);
2249 }
2250 }
2251 }
2252
2253 // use bufferqueue if outputting to a surface.
2254 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
2255 // if unsuccessful.
2256 if (outputSurface) {
2257 params.clear();
2258 err = mComponent->query({ },
2259 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
2260 C2_DONT_BLOCK,
2261 &params);
2262 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
2263 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
2264 mName, params.size(), asString(err), err);
2265 } else if (err == C2_OK && params.size() == 1) {
2266 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
2267 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
2268 if (surfaceAllocator) {
2269 std::shared_ptr<C2Allocator> allocator;
2270 // verify allocator IDs and resolve default allocator
2271 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
2272 if (allocator) {
2273 pools->outputAllocatorId = allocator->getId();
2274 } else {
2275 ALOGD("[%s] component requested invalid surface output allocator ID %u",
2276 mName, surfaceAllocator->value);
2277 err = C2_BAD_VALUE;
2278 }
2279 }
2280 }
2281 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
2282 && err != C2_OK
2283 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
2284 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
2285 }
2286 }
2287
2288 if ((poolMask >> pools->outputAllocatorId) & 1) {
2289 err = mComponent->createBlockPool(
2290 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
2291 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
2292 mName, pools->outputAllocatorId,
2293 (unsigned long long)pools->outputPoolId,
2294 asString(err));
2295 } else {
2296 err = C2_NOT_FOUND;
2297 }
2298 if (err != C2_OK) {
2299 // use basic pool instead
2300 pools->outputPoolId =
2301 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
2302 }
2303
2304 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
2305 // component.
2306 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
2307 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
2308
2309 std::vector<std::unique_ptr<C2SettingResult>> failures;
2310 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
2311 ALOGD("[%s] Configured output block pool ids %llu => %s",
2312 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
2313 outputPoolId_ = pools->outputPoolId;
2314 }
2315
2316 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2317
2318 if (graphic) {
2319 if (outputSurface) {
2320 buffers->reset(new GraphicOutputBuffers(mName));
2321 } else {
Wonsik Kim078b58e2019-01-09 15:08:06 -08002322 buffers->reset(new RawGraphicOutputBuffers(mNumOutputSlots, mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08002323 }
2324 } else {
2325 buffers->reset(new LinearOutputBuffers(mName));
2326 }
2327 (*buffers)->setFormat(outputFormat->dup());
2328
2329
2330 // Try to set output surface to created block pool if given.
2331 if (outputSurface) {
2332 mComponent->setOutputSurface(
2333 outputPoolId_,
2334 outputSurface,
2335 outputGeneration);
2336 }
2337
2338 if (oStreamFormat.value == C2BufferData::LINEAR
2339 && mComponentName.find("c2.qti.") == std::string::npos) {
2340 // WORKAROUND: if we're using early CSD workaround we convert to
2341 // array mode, to appease apps assuming the output
2342 // buffers to be of the same size.
Wonsik Kim078b58e2019-01-09 15:08:06 -08002343 (*buffers) = (*buffers)->toArrayMode(mNumOutputSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002344
2345 int32_t channelCount;
2346 int32_t sampleRate;
2347 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
2348 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
2349 int32_t delay = 0;
2350 int32_t padding = 0;;
2351 if (!outputFormat->findInt32("encoder-delay", &delay)) {
2352 delay = 0;
2353 }
2354 if (!outputFormat->findInt32("encoder-padding", &padding)) {
2355 padding = 0;
2356 }
2357 if (delay || padding) {
2358 // We need write access to the buffers, and we're already in
2359 // array mode.
2360 (*buffers)->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
2361 }
2362 }
2363 }
2364 }
2365
2366 // Set up pipeline control. This has to be done after mInputBuffers and
2367 // mOutputBuffers are initialized to make sure that lingering callbacks
2368 // about buffers from the previous generation do not interfere with the
2369 // newly initialized pipeline capacity.
2370
Pawin Vongmasa36653902018-11-15 00:10:25 -08002371 mAvailablePipelineCapacity.initialize(
Wonsik Kim078b58e2019-01-09 15:08:06 -08002372 mNumInputSlots,
2373 mNumInputSlots + mNumOutputSlots,
Pawin Vongmasa36653902018-11-15 00:10:25 -08002374 mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002375
2376 mInputMetEos = false;
2377 mSync.start();
2378 return OK;
2379}
2380
2381status_t CCodecBufferChannel::requestInitialInputBuffers() {
2382 if (mInputSurface) {
2383 return OK;
2384 }
2385
2386 C2StreamFormatConfig::output oStreamFormat(0u);
2387 c2_status_t err = mComponent->query({ &oStreamFormat }, {}, C2_DONT_BLOCK, nullptr);
2388 if (err != C2_OK) {
2389 return UNKNOWN_ERROR;
2390 }
2391 std::vector<sp<MediaCodecBuffer>> toBeQueued;
2392 // TODO: use proper buffer depth instead of this random value
Wonsik Kim078b58e2019-01-09 15:08:06 -08002393 for (size_t i = 0; i < mNumInputSlots; ++i) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002394 size_t index;
2395 sp<MediaCodecBuffer> buffer;
2396 {
2397 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
2398 if (!(*buffers)->requestNewBuffer(&index, &buffer)) {
2399 if (i == 0) {
2400 ALOGW("[%s] start: cannot allocate memory at all", mName);
2401 return NO_MEMORY;
2402 } else {
2403 ALOGV("[%s] start: cannot allocate memory, only %zu buffers allocated",
2404 mName, i);
2405 }
2406 break;
2407 }
2408 }
2409 if (buffer) {
2410 Mutexed<std::list<sp<ABuffer>>>::Locked configs(mFlushedConfigs);
2411 ALOGV("[%s] input buffer %zu available", mName, index);
2412 bool post = true;
2413 if (!configs->empty()) {
2414 sp<ABuffer> config = configs->front();
2415 if (buffer->capacity() >= config->size()) {
2416 memcpy(buffer->base(), config->data(), config->size());
2417 buffer->setRange(0, config->size());
2418 buffer->meta()->clear();
2419 buffer->meta()->setInt64("timeUs", 0);
2420 buffer->meta()->setInt32("csd", 1);
2421 post = false;
2422 } else {
2423 ALOGD("[%s] buffer capacity too small for the config (%zu < %zu)",
2424 mName, buffer->capacity(), config->size());
2425 }
2426 } else if (oStreamFormat.value == C2BufferData::LINEAR && i == 0
2427 && mComponentName.find("c2.qti.") == std::string::npos) {
2428 // WORKAROUND: Some apps expect CSD available without queueing
2429 // any input. Queue an empty buffer to get the CSD.
2430 buffer->setRange(0, 0);
2431 buffer->meta()->clear();
2432 buffer->meta()->setInt64("timeUs", 0);
2433 post = false;
2434 }
2435 if (mAvailablePipelineCapacity.allocate("requestInitialInputBuffers")) {
2436 if (post) {
2437 mCallback->onInputBufferAvailable(index, buffer);
2438 } else {
2439 toBeQueued.emplace_back(buffer);
2440 }
2441 } else {
2442 ALOGD("[%s] pipeline is full while requesting %zu-th input buffer",
2443 mName, i);
2444 }
2445 }
2446 }
2447 for (const sp<MediaCodecBuffer> &buffer : toBeQueued) {
2448 if (queueInputBufferInternal(buffer) != OK) {
2449 mAvailablePipelineCapacity.freeComponentSlot("requestInitialInputBuffers");
2450 }
2451 }
2452 return OK;
2453}
2454
2455void CCodecBufferChannel::stop() {
2456 mSync.stop();
2457 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
2458 if (mInputSurface != nullptr) {
2459 mInputSurface->disconnect();
2460 mInputSurface.reset();
2461 }
2462}
2463
2464void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
2465 ALOGV("[%s] flush", mName);
2466 {
2467 Mutexed<std::list<sp<ABuffer>>>::Locked configs(mFlushedConfigs);
2468 for (const std::unique_ptr<C2Work> &work : flushedWork) {
2469 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
2470 continue;
2471 }
2472 if (work->input.buffers.empty()
2473 || work->input.buffers.front()->data().linearBlocks().empty()) {
2474 ALOGD("[%s] no linear codec config data found", mName);
2475 continue;
2476 }
2477 C2ReadView view =
2478 work->input.buffers.front()->data().linearBlocks().front().map().get();
2479 if (view.error() != C2_OK) {
2480 ALOGD("[%s] failed to map flushed codec config data: %d", mName, view.error());
2481 continue;
2482 }
2483 configs->push_back(ABuffer::CreateAsCopy(view.data(), view.capacity()));
2484 ALOGV("[%s] stashed flushed codec config data (size=%u)", mName, view.capacity());
2485 }
2486 }
2487 {
2488 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
2489 (*buffers)->flush();
2490 }
2491 {
2492 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2493 (*buffers)->flush(flushedWork);
2494 }
2495}
2496
2497void CCodecBufferChannel::onWorkDone(
2498 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
2499 const C2StreamInitDataInfo::output *initData,
2500 size_t numDiscardedInputBuffers) {
2501 if (handleWork(std::move(work), outputFormat, initData)) {
2502 mAvailablePipelineCapacity.freeInputSlots(numDiscardedInputBuffers,
2503 "onWorkDone");
2504 feedInputBufferIfAvailable();
2505 }
2506}
2507
2508void CCodecBufferChannel::onInputBufferDone(
2509 const std::shared_ptr<C2Buffer>& buffer) {
2510 bool newInputSlotAvailable;
2511 {
2512 Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
2513 newInputSlotAvailable = (*buffers)->expireComponentBuffer(buffer);
2514 if (newInputSlotAvailable) {
2515 mAvailablePipelineCapacity.freeInputSlots(1, "onInputBufferDone");
2516 }
2517 }
2518 if (newInputSlotAvailable) {
2519 feedInputBufferIfAvailable();
2520 }
2521}
2522
2523bool CCodecBufferChannel::handleWork(
2524 std::unique_ptr<C2Work> work,
2525 const sp<AMessage> &outputFormat,
2526 const C2StreamInitDataInfo::output *initData) {
2527 if ((work->input.ordinal.frameIndex - mFirstValidFrameIndex.load()).peek() < 0) {
2528 // Discard frames from previous generation.
2529 ALOGD("[%s] Discard frames from previous generation.", mName);
2530 return false;
2531 }
2532
2533 if (work->worklets.size() != 1u
2534 || !work->worklets.front()
2535 || !(work->worklets.front()->output.flags & C2FrameData::FLAG_INCOMPLETE)) {
2536 mAvailablePipelineCapacity.freeComponentSlot("handleWork");
2537 }
2538
2539 if (work->result == C2_NOT_FOUND) {
2540 ALOGD("[%s] flushed work; ignored.", mName);
2541 return true;
2542 }
2543
2544 if (work->result != C2_OK) {
2545 ALOGD("[%s] work failed to complete: %d", mName, work->result);
2546 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
2547 return false;
2548 }
2549
2550 // NOTE: MediaCodec usage supposedly have only one worklet
2551 if (work->worklets.size() != 1u) {
2552 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
2553 mName, work->worklets.size());
2554 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2555 return false;
2556 }
2557
2558 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
2559
2560 std::shared_ptr<C2Buffer> buffer;
2561 // NOTE: MediaCodec usage supposedly have only one output stream.
2562 if (worklet->output.buffers.size() > 1u) {
2563 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
2564 mName, worklet->output.buffers.size());
2565 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2566 return false;
2567 } else if (worklet->output.buffers.size() == 1u) {
2568 buffer = worklet->output.buffers[0];
2569 if (!buffer) {
2570 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
2571 }
2572 }
2573
2574 while (!worklet->output.configUpdate.empty()) {
2575 std::unique_ptr<C2Param> param;
2576 worklet->output.configUpdate.back().swap(param);
2577 worklet->output.configUpdate.pop_back();
2578 switch (param->coreIndex().coreIndex()) {
2579 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
2580 C2PortReorderBufferDepthTuning::output reorderDepth;
2581 if (reorderDepth.updateFrom(*param)) {
2582 mReorderStash.lock()->setDepth(reorderDepth.value);
2583 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
2584 mName, reorderDepth.value);
2585 } else {
2586 ALOGD("[%s] onWorkDone: failed to read reorder depth", mName);
2587 }
2588 break;
2589 }
2590 case C2PortReorderKeySetting::CORE_INDEX: {
2591 C2PortReorderKeySetting::output reorderKey;
2592 if (reorderKey.updateFrom(*param)) {
2593 mReorderStash.lock()->setKey(reorderKey.value);
2594 ALOGV("[%s] onWorkDone: updated reorder key to %u",
2595 mName, reorderKey.value);
2596 } else {
2597 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
2598 }
2599 break;
2600 }
2601 default:
2602 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
2603 mName, param->index());
2604 break;
2605 }
2606 }
2607
2608 if (outputFormat != nullptr) {
2609 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2610 ALOGD("[%s] onWorkDone: output format changed to %s",
2611 mName, outputFormat->debugString().c_str());
2612 (*buffers)->setFormat(outputFormat);
2613
2614 AString mediaType;
2615 if (outputFormat->findString(KEY_MIME, &mediaType)
2616 && mediaType == MIMETYPE_AUDIO_RAW) {
2617 int32_t channelCount;
2618 int32_t sampleRate;
2619 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
2620 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
2621 (*buffers)->updateSkipCutBuffer(sampleRate, channelCount);
2622 }
2623 }
2624 }
2625
2626 int32_t flags = 0;
2627 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
2628 flags |= MediaCodec::BUFFER_FLAG_EOS;
2629 ALOGV("[%s] onWorkDone: output EOS", mName);
2630 }
2631
2632 sp<MediaCodecBuffer> outBuffer;
2633 size_t index;
2634
2635 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
2636 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
2637 // the codec input timestamp, but client output timestamp should (reported in timeUs)
2638 // shall correspond to the client input timesamp (in customOrdinal). By using the
2639 // delta between the two, this allows for some timestamp deviation - e.g. if one input
2640 // produces multiple output.
2641 c2_cntr64_t timestamp =
2642 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
2643 - work->input.ordinal.timestamp;
2644 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
2645 mName,
2646 work->input.ordinal.customOrdinal.peekll(),
2647 work->input.ordinal.timestamp.peekll(),
2648 worklet->output.ordinal.timestamp.peekll(),
2649 timestamp.peekll());
2650
2651 if (initData != nullptr) {
2652 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2653 if ((*buffers)->registerCsd(initData, &index, &outBuffer) == OK) {
2654 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
2655 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
2656 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
2657
2658 buffers.unlock();
2659 mCallback->onOutputBufferAvailable(index, outBuffer);
2660 buffers.lock();
2661 } else {
2662 ALOGD("[%s] onWorkDone: unable to register csd", mName);
2663 buffers.unlock();
2664 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2665 buffers.lock();
2666 return false;
2667 }
2668 }
2669
2670 if (!buffer && !flags) {
2671 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
2672 mName, work->input.ordinal.frameIndex.peekull());
2673 return true;
2674 }
2675
2676 if (buffer) {
2677 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
2678 // TODO: properly translate these to metadata
2679 switch (info->coreIndex().coreIndex()) {
2680 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
2681 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2PictureTypeKeyFrame) {
2682 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
2683 }
2684 break;
2685 default:
2686 break;
2687 }
2688 }
2689 }
2690
2691 {
2692 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
2693 reorder->emplace(buffer, timestamp.peek(), flags, worklet->output.ordinal);
2694 if (flags & MediaCodec::BUFFER_FLAG_EOS) {
2695 // Flush reorder stash
2696 reorder->setDepth(0);
2697 }
2698 }
2699 sendOutputBuffers();
2700 return true;
2701}
2702
2703void CCodecBufferChannel::sendOutputBuffers() {
2704 ReorderStash::Entry entry;
2705 sp<MediaCodecBuffer> outBuffer;
2706 size_t index;
2707
2708 while (true) {
2709 {
2710 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
2711 if (!reorder->hasPending()) {
2712 break;
2713 }
2714 if (!reorder->pop(&entry)) {
2715 break;
2716 }
2717 }
2718 Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
2719 status_t err = (*buffers)->registerBuffer(entry.buffer, &index, &outBuffer);
2720 if (err != OK) {
2721 if (err != WOULD_BLOCK) {
Wonsik Kim186fdbf2019-01-29 13:30:01 -08002722 if (!(*buffers)->isArrayMode()) {
2723 *buffers = (*buffers)->toArrayMode(mNumOutputSlots);
2724 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002725 OutputBuffersArray *array = (OutputBuffersArray *)buffers->get();
2726 array->realloc(entry.buffer);
2727 mCCodecCallback->onOutputBuffersChanged();
2728 }
2729 buffers.unlock();
2730 ALOGV("[%s] sendOutputBuffers: unable to register output buffer", mName);
2731 mReorderStash.lock()->defer(entry);
2732 return;
2733 }
2734 buffers.unlock();
2735
2736 outBuffer->meta()->setInt64("timeUs", entry.timestamp);
2737 outBuffer->meta()->setInt32("flags", entry.flags);
2738 ALOGV("[%s] sendOutputBuffers: out buffer index = %zu [%p] => %p + %zu",
2739 mName, index, outBuffer.get(), outBuffer->data(), outBuffer->size());
2740 mCallback->onOutputBufferAvailable(index, outBuffer);
2741 }
2742}
2743
2744status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
2745 static std::atomic_uint32_t surfaceGeneration{0};
2746 uint32_t generation = (getpid() << 10) |
2747 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
2748 & ((1 << 10) - 1));
2749
2750 sp<IGraphicBufferProducer> producer;
2751 if (newSurface) {
2752 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
Wonsik Kim078b58e2019-01-09 15:08:06 -08002753 newSurface->setMaxDequeuedBufferCount(mNumOutputSlots + kRenderingDepth);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002754 producer = newSurface->getIGraphicBufferProducer();
2755 producer->setGenerationNumber(generation);
2756 } else {
2757 ALOGE("[%s] setting output surface to null", mName);
2758 return INVALID_OPERATION;
2759 }
2760
2761 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
2762 C2BlockPool::local_id_t outputPoolId;
2763 {
2764 Mutexed<BlockPools>::Locked pools(mBlockPools);
2765 outputPoolId = pools->outputPoolId;
2766 outputPoolIntf = pools->outputPoolIntf;
2767 }
2768
2769 if (outputPoolIntf) {
2770 if (mComponent->setOutputSurface(
2771 outputPoolId,
2772 producer,
2773 generation) != C2_OK) {
2774 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
2775 return INVALID_OPERATION;
2776 }
2777 }
2778
2779 {
2780 Mutexed<OutputSurface>::Locked output(mOutputSurface);
2781 output->surface = newSurface;
2782 output->generation = generation;
2783 }
2784
2785 return OK;
2786}
2787
2788void CCodecBufferChannel::setMetaMode(MetaMode mode) {
2789 mMetaMode = mode;
2790}
2791
2792status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
2793 // C2_OK is always translated to OK.
2794 if (c2s == C2_OK) {
2795 return OK;
2796 }
2797
2798 // Operation-dependent translation
2799 // TODO: Add as necessary
2800 switch (c2op) {
2801 case C2_OPERATION_Component_start:
2802 switch (c2s) {
2803 case C2_NO_MEMORY:
2804 return NO_MEMORY;
2805 default:
2806 return UNKNOWN_ERROR;
2807 }
2808 default:
2809 break;
2810 }
2811
2812 // Backup operation-agnostic translation
2813 switch (c2s) {
2814 case C2_BAD_INDEX:
2815 return BAD_INDEX;
2816 case C2_BAD_VALUE:
2817 return BAD_VALUE;
2818 case C2_BLOCKING:
2819 return WOULD_BLOCK;
2820 case C2_DUPLICATE:
2821 return ALREADY_EXISTS;
2822 case C2_NO_INIT:
2823 return NO_INIT;
2824 case C2_NO_MEMORY:
2825 return NO_MEMORY;
2826 case C2_NOT_FOUND:
2827 return NAME_NOT_FOUND;
2828 case C2_TIMED_OUT:
2829 return TIMED_OUT;
2830 case C2_BAD_STATE:
2831 case C2_CANCELED:
2832 case C2_CANNOT_DO:
2833 case C2_CORRUPTED:
2834 case C2_OMITTED:
2835 case C2_REFUSED:
2836 return UNKNOWN_ERROR;
2837 default:
2838 return -static_cast<status_t>(c2s);
2839 }
2840}
2841
2842} // namespace android