blob: cbef6440e06a93935d8c54c3500668e0f1baa05d [file] [log] [blame]
Wonsik Kim469c8342019-04-11 16:46:09 -07001/*
2 * Copyright 2019, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef CCODEC_BUFFERS_H_
18
19#define CCODEC_BUFFERS_H_
20
Wonsik Kim6f23cfc2021-09-24 05:45:52 -070021#include <optional>
Wonsik Kim469c8342019-04-11 16:46:09 -070022#include <string>
23
24#include <C2Config.h>
Wonsik Kim6f23cfc2021-09-24 05:45:52 -070025#include <DataConverter.h>
Wonsik Kim469c8342019-04-11 16:46:09 -070026#include <media/stagefright/foundation/AMessage.h>
27#include <media/MediaCodecBuffer.h>
28
29#include "Codec2Buffer.h"
Wonsik Kim469c8342019-04-11 16:46:09 -070030
31namespace android {
32
Wonsik Kim41d83432020-04-27 16:40:49 -070033struct ICrypto;
34class MemoryDealer;
Wonsik Kim155d5cb2019-10-09 12:49:49 -070035class SkipCutBuffer;
36
Wonsik Kim469c8342019-04-11 16:46:09 -070037constexpr size_t kLinearBufferSize = 1048576;
Sungtak Lee56925782020-11-08 00:07:27 -080038// This can fit an 8K frame.
39constexpr size_t kMaxLinearBufferSize = 7680 * 4320 * 2;
Wonsik Kim469c8342019-04-11 16:46:09 -070040
41/**
42 * Base class for representation of buffers at one port.
43 */
44class CCodecBuffers {
45public:
46 CCodecBuffers(const char *componentName, const char *name = "Buffers")
47 : mComponentName(componentName),
48 mChannelName(std::string(componentName) + ":" + name),
49 mName(mChannelName.c_str()) {
50 }
51 virtual ~CCodecBuffers() = default;
52
53 /**
54 * Set format for MediaCodec-facing buffers.
55 */
56 void setFormat(const sp<AMessage> &format);
57
58 /**
59 * Return a copy of current format.
60 */
61 sp<AMessage> dupFormat();
62
63 /**
64 * Returns true if the buffers are operating under array mode.
65 */
66 virtual bool isArrayMode() const { return false; }
67
68 /**
69 * Fills the vector with MediaCodecBuffer's if in array mode; otherwise,
70 * no-op.
71 */
72 virtual void getArray(Vector<sp<MediaCodecBuffer>> *) const {}
73
74 /**
Xiao Huang5ff11fa2022-09-20 12:04:27 +000075 * Return number of buffers owned by the client or the component.
Wonsik Kim469c8342019-04-11 16:46:09 -070076 */
Wonsik Kim0487b782020-10-28 11:45:50 -070077 virtual size_t numActiveSlots() const = 0;
Wonsik Kim469c8342019-04-11 16:46:09 -070078
79 /**
80 * Examine image data from the buffer and update the format if necessary.
81 */
82 void handleImageData(const sp<Codec2Buffer> &buffer);
83
Songyue Han1e6769b2023-08-30 18:09:27 +000084 /**
85 * Get the first pixel format of a metric session.
86 */
87 virtual uint32_t getPixelFormatIfApplicable();
88
89 /**
90 * Reset the pixel format when a new metric session started.
91 */
92 virtual bool resetPixelFormatIfApplicable();
93
Wonsik Kim469c8342019-04-11 16:46:09 -070094protected:
95 std::string mComponentName; ///< name of component for debugging
96 std::string mChannelName; ///< name of channel for debugging
97 const char *mName; ///< C-string version of channel name
98 // Format to be used for creating MediaCodec-facing buffers.
99 sp<AMessage> mFormat;
100
Wonsik Kim4a3c0462021-03-09 15:45:05 -0800101 sp<ABuffer> mLastImageData;
102 sp<AMessage> mFormatWithImageData;
103
Wonsik Kim469c8342019-04-11 16:46:09 -0700104private:
105 DISALLOW_EVIL_CONSTRUCTORS(CCodecBuffers);
106};
107
108class InputBuffers : public CCodecBuffers {
109public:
110 InputBuffers(const char *componentName, const char *name = "Input[]")
111 : CCodecBuffers(componentName, name) { }
112 virtual ~InputBuffers() = default;
113
114 /**
115 * Set a block pool to obtain input memory blocks.
116 */
117 void setPool(const std::shared_ptr<C2BlockPool> &pool) { mPool = pool; }
118
119 /**
120 * Get a new MediaCodecBuffer for input and its corresponding index.
121 * Returns false if no new buffer can be obtained at the moment.
122 */
123 virtual bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) = 0;
124
125 /**
126 * Release the buffer obtained from requestNewBuffer() and get the
127 * associated C2Buffer object back. Returns true if the buffer was on file
128 * and released successfully.
129 */
130 virtual bool releaseBuffer(
131 const sp<MediaCodecBuffer> &buffer,
132 std::shared_ptr<C2Buffer> *c2buffer,
133 bool release) = 0;
134
135 /**
136 * Release the buffer that is no longer used by the codec process. Return
137 * true if and only if the buffer was on file and released successfully.
138 */
139 virtual bool expireComponentBuffer(
140 const std::shared_ptr<C2Buffer> &c2buffer) = 0;
141
142 /**
143 * Flush internal state. After this call, no index or buffer previously
144 * returned from requestNewBuffer() is valid.
145 */
146 virtual void flush() = 0;
147
148 /**
149 * Return array-backed version of input buffers. The returned object
150 * shall retain the internal state so that it will honor index and
151 * buffer from previous calls of requestNewBuffer().
152 */
153 virtual std::unique_ptr<InputBuffers> toArrayMode(size_t size) = 0;
154
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700155 /**
156 * Release the buffer obtained from requestNewBuffer(), and create a deep
157 * copy clone of the buffer.
158 *
159 * \return the deep copy clone of the buffer; nullptr if cloning is not
160 * possible.
161 */
162 sp<Codec2Buffer> cloneAndReleaseBuffer(const sp<MediaCodecBuffer> &buffer);
163
Wonsik Kim469c8342019-04-11 16:46:09 -0700164protected:
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700165 virtual sp<Codec2Buffer> createNewBuffer() = 0;
166
Wonsik Kim469c8342019-04-11 16:46:09 -0700167 // Pool to obtain blocks for input buffers.
168 std::shared_ptr<C2BlockPool> mPool;
169
170private:
171 DISALLOW_EVIL_CONSTRUCTORS(InputBuffers);
172};
173
Pawin Vongmasa9b906982020-04-11 05:07:15 -0700174class OutputBuffersArray;
175
Wonsik Kim469c8342019-04-11 16:46:09 -0700176class OutputBuffers : public CCodecBuffers {
177public:
Wonsik Kim41d83432020-04-27 16:40:49 -0700178 OutputBuffers(const char *componentName, const char *name = "Output");
179 virtual ~OutputBuffers();
Wonsik Kim469c8342019-04-11 16:46:09 -0700180
181 /**
182 * Register output C2Buffer from the component and obtain corresponding
Pawin Vongmasa9b906982020-04-11 05:07:15 -0700183 * index and MediaCodecBuffer object.
184 *
185 * Returns:
186 * OK if registration succeeds.
187 * NO_MEMORY if all buffers are available but not compatible.
188 * WOULD_BLOCK if there are compatible buffers, but they are all in use.
Wonsik Kim469c8342019-04-11 16:46:09 -0700189 */
190 virtual status_t registerBuffer(
191 const std::shared_ptr<C2Buffer> &buffer,
192 size_t *index,
193 sp<MediaCodecBuffer> *clientBuffer) = 0;
194
195 /**
196 * Register codec specific data as a buffer to be consistent with
197 * MediaCodec behavior.
198 */
199 virtual status_t registerCsd(
200 const C2StreamInitDataInfo::output * /* csd */,
201 size_t * /* index */,
202 sp<MediaCodecBuffer> * /* clientBuffer */) = 0;
203
204 /**
205 * Release the buffer obtained from registerBuffer() and get the
206 * associated C2Buffer object back. Returns true if the buffer was on file
207 * and released successfully.
208 */
209 virtual bool releaseBuffer(
210 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) = 0;
211
212 /**
213 * Flush internal state. After this call, no index or buffer previously
214 * returned from registerBuffer() is valid.
215 */
216 virtual void flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) = 0;
217
218 /**
219 * Return array-backed version of output buffers. The returned object
220 * shall retain the internal state so that it will honor index and
221 * buffer from previous calls of registerBuffer().
222 */
Pawin Vongmasa9b906982020-04-11 05:07:15 -0700223 virtual std::unique_ptr<OutputBuffersArray> toArrayMode(size_t size) = 0;
Wonsik Kim469c8342019-04-11 16:46:09 -0700224
225 /**
226 * Initialize SkipCutBuffer object.
227 */
228 void initSkipCutBuffer(
229 int32_t delay, int32_t padding, int32_t sampleRate, int32_t channelCount);
230
231 /**
Pawin Vongmasa9b906982020-04-11 05:07:15 -0700232 * Update SkipCutBuffer from format. The @p format must not be null.
Pawin Vongmasa9b906982020-04-11 05:07:15 -0700233 */
Wonsik Kim970bf0b2020-11-10 11:54:15 -0800234 void updateSkipCutBuffer(const sp<AMessage> &format);
Pawin Vongmasa9b906982020-04-11 05:07:15 -0700235
236 /**
237 * Output Stash
238 * ============
239 *
240 * The output stash is a place to hold output buffers temporarily before
241 * they are registered to output slots. It has 2 main functions:
242 * 1. Allow reordering of output frames as the codec may produce frames in a
243 * different order.
244 * 2. Act as a "buffer" between the codec and the client because the codec
245 * may produce more buffers than available slots. This excess of codec's
246 * output buffers should be registered to slots later, after the client
247 * has released some slots.
248 *
249 * The stash consists of 2 lists of buffers: mPending and mReorderStash.
250 * mPending is a normal FIFO queue with not size limit, while mReorderStash
251 * is a sorted list with size limit mDepth.
252 *
253 * The normal flow of a non-csd output buffer is as follows:
254 *
255 * |----------------OutputBuffers---------------|
256 * |----------Output stash----------| |
257 * Codec --|-> mReorderStash --> mPending --|-> slots --|-> client
258 * | | |
259 * pushToStash() popFromStashAndRegister()
260 *
261 * The buffer that comes from the codec first enters mReorderStash. The
262 * first buffer in mReorderStash gets moved to mPending when mReorderStash
263 * overflows. Buffers in mPending are registered to slots and given to the
264 * client as soon as slots are available.
265 *
266 * Every output buffer that is not a csd buffer should be put on the stash
267 * by calling pushToStash(), then later registered to a slot by calling
268 * popFromStashAndRegister() before notifying the client with
269 * onOutputBufferAvailable().
270 *
271 * Reordering
272 * ==========
273 *
274 * mReorderStash is a sorted list with a specified size limit. The size
275 * limit can be set by calling setReorderDepth().
276 *
277 * Every buffer in mReorderStash has a C2WorkOrdinalStruct, which contains 3
278 * members, all of which are comparable. Which member of C2WorkOrdinalStruct
279 * should be used for reordering can be chosen by calling setReorderKey().
280 */
281
282 /**
283 * Return the reorder depth---the size of mReorderStash.
284 */
285 uint32_t getReorderDepth() const;
286
287 /**
288 * Set the reorder depth.
289 */
290 void setReorderDepth(uint32_t depth);
291
292 /**
293 * Set the type of "key" to use in comparisons.
294 */
295 void setReorderKey(C2Config::ordinal_key_t key);
296
297 /**
298 * Return whether the output stash has any pending buffers.
299 */
300 bool hasPending() const;
301
302 /**
303 * Flush the stash and reset the depth and the key to their default values.
304 */
305 void clearStash();
306
307 /**
308 * Flush the stash.
309 */
310 void flushStash();
311
312 /**
313 * Push a buffer to the reorder stash.
314 *
315 * @param buffer C2Buffer object from the returned work.
316 * @param notify Whether the returned work contains a buffer that should
317 * be reported to the client. This may be false if the
318 * caller wants to process the buffer without notifying the
319 * client.
320 * @param timestamp Buffer timestamp to report to the client.
321 * @param flags Buffer flags to report to the client.
322 * @param format Buffer format to report to the client.
323 * @param ordinal Ordinal used in reordering. This determines when the
324 * buffer will be popped from the output stash by
325 * `popFromStashAndRegister()`.
326 */
327 void pushToStash(
328 const std::shared_ptr<C2Buffer>& buffer,
329 bool notify,
330 int64_t timestamp,
331 int32_t flags,
332 const sp<AMessage>& format,
333 const C2WorkOrdinalStruct& ordinal);
334
335 enum BufferAction : int {
336 SKIP,
337 DISCARD,
338 NOTIFY_CLIENT,
339 REALLOCATE,
340 RETRY,
341 };
342
343 /**
344 * Try to atomically pop the first buffer from the reorder stash and
345 * register it to an output slot. The function returns a value that
346 * indicates a recommended course of action for the caller.
347 *
348 * If the stash is empty, the function will return `SKIP`.
349 *
350 * If the stash is not empty, the function will peek at the first (oldest)
351 * entry in mPending process the buffer in the entry as follows:
352 * - If the buffer should not be sent to the client, the function will
353 * return `DISCARD`. The stash entry will be removed.
354 * - If the buffer should be sent to the client, the function will attempt
355 * to register the buffer to a slot. The registration may have 3 outcomes
356 * corresponding to the following return values:
357 * - `NOTIFY_CLIENT`: The buffer is successfully registered to a slot. The
358 * output arguments @p index and @p outBuffer will contain valid values
359 * that the caller can use to call onOutputBufferAvailable(). The stash
360 * entry will be removed.
361 * - `REALLOCATE`: The buffer is not registered because it is not
362 * compatible with the current slots (which are available). The caller
363 * should reallocate the OutputBuffers with slots that can fit the
364 * returned @p c2Buffer. The stash entry will not be removed
365 * - `RETRY`: All slots are currently occupied by the client. The caller
366 * should try to call this function again after the client has released
367 * some slots.
368 *
369 * @return What the caller should do afterwards.
370 *
371 * @param[out] c2Buffer Underlying C2Buffer associated to the first buffer
372 * on the stash. This value is guaranteed to be valid
373 * unless the return value is `SKIP`.
374 * @param[out] index Slot index. This value is valid only if the return
375 * value is `NOTIFY_CLIENT`.
376 * @param[out] outBuffer Registered buffer. This value is valid only if the
377 * return valu is `NOTIFY_CLIENT`.
378 */
379 BufferAction popFromStashAndRegister(
380 std::shared_ptr<C2Buffer>* c2Buffer,
381 size_t* index,
382 sp<MediaCodecBuffer>* outBuffer);
383
384protected:
385 sp<SkipCutBuffer> mSkipCutBuffer;
386
387 /**
Wonsik Kim469c8342019-04-11 16:46:09 -0700388 * Update the SkipCutBuffer object. No-op if it's never initialized.
389 */
390 void updateSkipCutBuffer(int32_t sampleRate, int32_t channelCount);
391
392 /**
393 * Submit buffer to SkipCutBuffer object, if initialized.
394 */
395 void submit(const sp<MediaCodecBuffer> &buffer);
396
Wonsik Kim6f23cfc2021-09-24 05:45:52 -0700397 /**
Greg Kaiser92dc4542021-10-08 06:58:19 -0700398 * Apply DataConverter from |src| to |*dst| if needed. If |*dst| is nullptr,
Wonsik Kim6f23cfc2021-09-24 05:45:52 -0700399 * a new buffer is allocated.
400 *
401 * Returns true if conversion was needed and executed; false otherwise.
402 */
403 bool convert(const std::shared_ptr<C2Buffer> &src, sp<Codec2Buffer> *dst);
404
Wonsik Kim469c8342019-04-11 16:46:09 -0700405private:
Pawin Vongmasa9b906982020-04-11 05:07:15 -0700406 // SkipCutBuffer
Wonsik Kim469c8342019-04-11 16:46:09 -0700407 int32_t mDelay;
408 int32_t mPadding;
409 int32_t mSampleRate;
Wonsik Kim40b0b1d2020-03-25 15:47:35 -0700410 int32_t mChannelCount;
Wonsik Kim469c8342019-04-11 16:46:09 -0700411
Wonsik Kim40b0b1d2020-03-25 15:47:35 -0700412 void setSkipCutBuffer(int32_t skip, int32_t cut);
Wonsik Kim469c8342019-04-11 16:46:09 -0700413
Wonsik Kim6f23cfc2021-09-24 05:45:52 -0700414 // DataConverter
415 sp<DataConverter> mDataConverter;
416 sp<AMessage> mFormatWithConverter;
417 std::optional<int32_t> mSrcEncoding;
418 std::optional<int32_t> mDstEncoding;
419
Pawin Vongmasa9b906982020-04-11 05:07:15 -0700420 // Output stash
421
Pawin Vongmasa9b906982020-04-11 05:07:15 -0700422 // Struct for an entry in the output stash (mPending and mReorderStash)
423 struct StashEntry {
424 inline StashEntry()
425 : buffer(nullptr),
426 notify(false),
427 timestamp(0),
428 flags(0),
429 format(),
430 ordinal({0, 0, 0}) {}
431 inline StashEntry(
432 const std::shared_ptr<C2Buffer> &b,
433 bool n,
434 int64_t t,
435 int32_t f,
436 const sp<AMessage> &fmt,
437 const C2WorkOrdinalStruct &o)
438 : buffer(b),
439 notify(n),
440 timestamp(t),
441 flags(f),
442 format(fmt),
443 ordinal(o) {}
444 std::shared_ptr<C2Buffer> buffer;
445 bool notify;
446 int64_t timestamp;
447 int32_t flags;
448 sp<AMessage> format;
449 C2WorkOrdinalStruct ordinal;
450 };
451
452 /**
453 * FIFO queue of stash entries.
454 */
455 std::list<StashEntry> mPending;
456 /**
457 * Sorted list of stash entries.
458 */
459 std::list<StashEntry> mReorderStash;
460 /**
461 * Size limit of mReorderStash.
462 */
463 uint32_t mDepth{0};
464 /**
465 * Choice of key to use in ordering of stash entries in mReorderStash.
466 */
467 C2Config::ordinal_key_t mKey{C2Config::ORDINAL};
468
469 /**
470 * Return false if mPending is empty; otherwise, pop the first entry from
471 * mPending and return true.
472 */
473 bool popPending(StashEntry *entry);
474
475 /**
476 * Push an entry as the first entry of mPending.
477 */
478 void deferPending(const StashEntry &entry);
479
480 /**
481 * Comparison of C2WorkOrdinalStruct based on mKey.
482 */
483 bool less(const C2WorkOrdinalStruct &o1,
484 const C2WorkOrdinalStruct &o2) const;
485
Wonsik Kim469c8342019-04-11 16:46:09 -0700486 DISALLOW_EVIL_CONSTRUCTORS(OutputBuffers);
Pawin Vongmasa9b906982020-04-11 05:07:15 -0700487
488 friend OutputBuffersArray;
Wonsik Kim469c8342019-04-11 16:46:09 -0700489};
490
491/**
492 * Simple local buffer pool backed by std::vector.
493 */
494class LocalBufferPool : public std::enable_shared_from_this<LocalBufferPool> {
495public:
496 /**
497 * Create a new LocalBufferPool object.
498 *
Wonsik Kim469c8342019-04-11 16:46:09 -0700499 * \return a newly created pool object.
500 */
Wonsik Kim41d83432020-04-27 16:40:49 -0700501 static std::shared_ptr<LocalBufferPool> Create();
Wonsik Kim469c8342019-04-11 16:46:09 -0700502
503 /**
504 * Return an ABuffer object whose size is at least |capacity|.
505 *
506 * \param capacity requested capacity
507 * \return nullptr if the pool capacity is reached
508 * an ABuffer object otherwise.
509 */
510 sp<ABuffer> newBuffer(size_t capacity);
511
512private:
513 /**
514 * ABuffer backed by std::vector.
515 */
516 class VectorBuffer : public ::android::ABuffer {
517 public:
518 /**
519 * Construct a VectorBuffer by taking the ownership of supplied vector.
520 *
521 * \param vec backing vector of the buffer. this object takes
522 * ownership at construction.
523 * \param pool a LocalBufferPool object to return the vector at
524 * destruction.
525 */
526 VectorBuffer(std::vector<uint8_t> &&vec, const std::shared_ptr<LocalBufferPool> &pool);
527
528 ~VectorBuffer() override;
529
530 private:
531 std::vector<uint8_t> mVec;
532 std::weak_ptr<LocalBufferPool> mPool;
533 };
534
535 Mutex mMutex;
536 size_t mPoolCapacity;
537 size_t mUsedSize;
538 std::list<std::vector<uint8_t>> mPool;
539
540 /**
541 * Private constructor to prevent constructing non-managed LocalBufferPool.
542 */
543 explicit LocalBufferPool(size_t poolCapacity)
544 : mPoolCapacity(poolCapacity), mUsedSize(0) {
545 }
546
547 /**
548 * Take back the ownership of vec from the destructed VectorBuffer and put
549 * it in front of the pool.
550 */
551 void returnVector(std::vector<uint8_t> &&vec);
552
553 DISALLOW_EVIL_CONSTRUCTORS(LocalBufferPool);
554};
555
556class BuffersArrayImpl;
557
558/**
559 * Flexible buffer slots implementation.
560 */
561class FlexBuffersImpl {
562public:
563 FlexBuffersImpl(const char *name)
564 : mImplName(std::string(name) + ".Impl"),
565 mName(mImplName.c_str()) { }
566
567 /**
568 * Assign an empty slot for a buffer and return the index. If there's no
569 * empty slot, just add one at the end and return it.
570 *
571 * \param buffer[in] a new buffer to assign a slot.
572 * \return index of the assigned slot.
573 */
574 size_t assignSlot(const sp<Codec2Buffer> &buffer);
575
576 /**
577 * Release the slot from the client, and get the C2Buffer object back from
578 * the previously assigned buffer. Note that the slot is not completely free
579 * until the returned C2Buffer object is freed.
580 *
581 * \param buffer[in] the buffer previously assigned a slot.
582 * \param c2buffer[in,out] pointer to C2Buffer to be populated. Ignored
583 * if null.
584 * \return true if the buffer is successfully released from a slot
585 * false otherwise
586 */
587 bool releaseSlot(
588 const sp<MediaCodecBuffer> &buffer,
589 std::shared_ptr<C2Buffer> *c2buffer,
590 bool release);
591
592 /**
593 * Expire the C2Buffer object in the slot.
594 *
595 * \param c2buffer[in] C2Buffer object which the component released.
596 * \return true if the buffer is found in one of the slots and
597 * successfully released
598 * false otherwise
599 */
600 bool expireComponentBuffer(const std::shared_ptr<C2Buffer> &c2buffer);
601
602 /**
603 * The client abandoned all known buffers, so reclaim the ownership.
604 */
605 void flush();
606
607 /**
Xiao Huang5ff11fa2022-09-20 12:04:27 +0000608 * Return the number of buffers that are sent to the client or the component.
Wonsik Kim469c8342019-04-11 16:46:09 -0700609 */
Wonsik Kim0487b782020-10-28 11:45:50 -0700610 size_t numActiveSlots() const;
Wonsik Kim469c8342019-04-11 16:46:09 -0700611
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700612 /**
613 * Return the number of buffers that are sent to the component but not
614 * returned back yet.
615 */
616 size_t numComponentBuffers() const;
617
Wonsik Kim469c8342019-04-11 16:46:09 -0700618private:
619 friend class BuffersArrayImpl;
620
621 std::string mImplName; ///< name for debugging
622 const char *mName; ///< C-string version of name
623
624 struct Entry {
625 sp<Codec2Buffer> clientBuffer;
626 std::weak_ptr<C2Buffer> compBuffer;
627 };
628 std::vector<Entry> mBuffers;
629};
630
631/**
632 * Static buffer slots implementation based on a fixed-size array.
633 */
634class BuffersArrayImpl {
635public:
636 BuffersArrayImpl()
637 : mImplName("BuffersArrayImpl"),
638 mName(mImplName.c_str()) { }
639
640 /**
641 * Initialize buffer array from the original |impl|. The buffers known by
642 * the client is preserved, and the empty slots are populated so that the
643 * array size is at least |minSize|.
644 *
645 * \param impl[in] FlexBuffersImpl object used so far.
646 * \param minSize[in] minimum size of the buffer array.
647 * \param allocate[in] function to allocate a client buffer for an empty slot.
648 */
649 void initialize(
650 const FlexBuffersImpl &impl,
651 size_t minSize,
652 std::function<sp<Codec2Buffer>()> allocate);
653
654 /**
655 * Grab a buffer from the underlying array which matches the criteria.
656 *
657 * \param index[out] index of the slot.
658 * \param buffer[out] the matching buffer.
659 * \param match[in] a function to test whether the buffer matches the
660 * criteria or not.
661 * \return OK if successful,
662 * WOULD_BLOCK if slots are being used,
663 * NO_MEMORY if no slot matches the criteria, even though it's
664 * available
665 */
666 status_t grabBuffer(
667 size_t *index,
668 sp<Codec2Buffer> *buffer,
669 std::function<bool(const sp<Codec2Buffer> &)> match =
Wonsik Kim936a89c2020-05-08 16:07:50 -0700670 [](const sp<Codec2Buffer> &buffer) { return (buffer != nullptr); });
Wonsik Kim469c8342019-04-11 16:46:09 -0700671
672 /**
673 * Return the buffer from the client, and get the C2Buffer object back from
674 * the buffer. Note that the slot is not completely free until the returned
675 * C2Buffer object is freed.
676 *
677 * \param buffer[in] the buffer previously grabbed.
678 * \param c2buffer[in,out] pointer to C2Buffer to be populated. Ignored
679 * if null.
680 * \return true if the buffer is successfully returned
681 * false otherwise
682 */
683 bool returnBuffer(
684 const sp<MediaCodecBuffer> &buffer,
685 std::shared_ptr<C2Buffer> *c2buffer,
686 bool release);
687
688 /**
689 * Expire the C2Buffer object in the slot.
690 *
691 * \param c2buffer[in] C2Buffer object which the component released.
692 * \return true if the buffer is found in one of the slots and
693 * successfully released
694 * false otherwise
695 */
696 bool expireComponentBuffer(const std::shared_ptr<C2Buffer> &c2buffer);
697
698 /**
699 * Populate |array| with the underlying buffer array.
700 *
701 * \param array[out] an array to be filled with the underlying buffer array.
702 */
703 void getArray(Vector<sp<MediaCodecBuffer>> *array) const;
704
705 /**
706 * The client abandoned all known buffers, so reclaim the ownership.
707 */
708 void flush();
709
710 /**
711 * Reallocate the array with the given allocation function.
712 *
713 * \param alloc[in] the allocation function for client buffers.
714 */
715 void realloc(std::function<sp<Codec2Buffer>()> alloc);
716
717 /**
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700718 * Grow the array to the new size. It is a programming error to supply
719 * smaller size as the new size.
720 *
721 * \param newSize[in] new size of the array.
722 * \param alloc[in] the alllocation function for client buffers to fill
723 * the new empty slots.
724 */
725 void grow(size_t newSize, std::function<sp<Codec2Buffer>()> alloc);
726
727 /**
Xiao Huang5ff11fa2022-09-20 12:04:27 +0000728 * Return the number of buffers that are sent to the client or the component.
Wonsik Kim469c8342019-04-11 16:46:09 -0700729 */
Wonsik Kim0487b782020-10-28 11:45:50 -0700730 size_t numActiveSlots() const;
Wonsik Kim469c8342019-04-11 16:46:09 -0700731
Wonsik Kima39882b2019-06-20 16:13:56 -0700732 /**
733 * Return the size of the array.
734 */
735 size_t arraySize() const;
736
Wonsik Kim469c8342019-04-11 16:46:09 -0700737private:
738 std::string mImplName; ///< name for debugging
739 const char *mName; ///< C-string version of name
740
741 struct Entry {
742 const sp<Codec2Buffer> clientBuffer;
743 std::weak_ptr<C2Buffer> compBuffer;
744 bool ownedByClient;
745 };
746 std::vector<Entry> mBuffers;
747};
748
749class InputBuffersArray : public InputBuffers {
750public:
751 InputBuffersArray(const char *componentName, const char *name = "Input[N]")
752 : InputBuffers(componentName, name) { }
753 ~InputBuffersArray() override = default;
754
755 /**
756 * Initialize this object from the non-array state. We keep existing slots
757 * at the same index, and for empty slots we allocate client buffers with
758 * the given allocate function. If the number of slots is less than minSize,
759 * we fill the array to the minimum size.
760 *
761 * \param impl[in] existing non-array state
762 * \param minSize[in] minimum size of the array
763 * \param allocate[in] allocate function to fill empty slots
764 */
765 void initialize(
766 const FlexBuffersImpl &impl,
767 size_t minSize,
768 std::function<sp<Codec2Buffer>()> allocate);
769
770 bool isArrayMode() const final { return true; }
771
772 std::unique_ptr<InputBuffers> toArrayMode(size_t) final {
773 return nullptr;
774 }
775
776 void getArray(Vector<sp<MediaCodecBuffer>> *array) const final;
777
778 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override;
779
780 bool releaseBuffer(
781 const sp<MediaCodecBuffer> &buffer,
782 std::shared_ptr<C2Buffer> *c2buffer,
783 bool release) override;
784
785 bool expireComponentBuffer(
786 const std::shared_ptr<C2Buffer> &c2buffer) override;
787
788 void flush() override;
789
Wonsik Kim0487b782020-10-28 11:45:50 -0700790 size_t numActiveSlots() const final;
Wonsik Kim469c8342019-04-11 16:46:09 -0700791
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700792protected:
793 sp<Codec2Buffer> createNewBuffer() override;
794
Wonsik Kim469c8342019-04-11 16:46:09 -0700795private:
796 BuffersArrayImpl mImpl;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700797 std::function<sp<Codec2Buffer>()> mAllocate;
Wonsik Kim469c8342019-04-11 16:46:09 -0700798};
799
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800800class SlotInputBuffers : public InputBuffers {
801public:
802 SlotInputBuffers(const char *componentName, const char *name = "Slot-Input")
803 : InputBuffers(componentName, name),
804 mImpl(mName) { }
805 ~SlotInputBuffers() override = default;
806
807 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) final;
808
809 bool releaseBuffer(
810 const sp<MediaCodecBuffer> &buffer,
811 std::shared_ptr<C2Buffer> *c2buffer,
812 bool release) final;
813
814 bool expireComponentBuffer(
815 const std::shared_ptr<C2Buffer> &c2buffer) final;
816
817 void flush() final;
818
819 std::unique_ptr<InputBuffers> toArrayMode(size_t size) final;
820
Wonsik Kim0487b782020-10-28 11:45:50 -0700821 size_t numActiveSlots() const final;
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800822
823protected:
824 sp<Codec2Buffer> createNewBuffer() final;
825
826private:
827 FlexBuffersImpl mImpl;
828};
829
Wonsik Kim469c8342019-04-11 16:46:09 -0700830class LinearInputBuffers : public InputBuffers {
831public:
832 LinearInputBuffers(const char *componentName, const char *name = "1D-Input")
833 : InputBuffers(componentName, name),
834 mImpl(mName) { }
835 ~LinearInputBuffers() override = default;
836
837 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override;
838
839 bool releaseBuffer(
840 const sp<MediaCodecBuffer> &buffer,
841 std::shared_ptr<C2Buffer> *c2buffer,
842 bool release) override;
843
844 bool expireComponentBuffer(
845 const std::shared_ptr<C2Buffer> &c2buffer) override;
846
847 void flush() override;
848
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700849 std::unique_ptr<InputBuffers> toArrayMode(size_t size) override;
Wonsik Kim469c8342019-04-11 16:46:09 -0700850
Wonsik Kim0487b782020-10-28 11:45:50 -0700851 size_t numActiveSlots() const final;
Wonsik Kim469c8342019-04-11 16:46:09 -0700852
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700853protected:
854 sp<Codec2Buffer> createNewBuffer() override;
855
856 FlexBuffersImpl mImpl;
Wonsik Kim469c8342019-04-11 16:46:09 -0700857
858private:
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700859 static sp<Codec2Buffer> Alloc(
860 const std::shared_ptr<C2BlockPool> &pool, const sp<AMessage> &format);
Wonsik Kim469c8342019-04-11 16:46:09 -0700861};
862
863class EncryptedLinearInputBuffers : public LinearInputBuffers {
864public:
865 EncryptedLinearInputBuffers(
866 bool secure,
867 const sp<MemoryDealer> &dealer,
868 const sp<ICrypto> &crypto,
869 int32_t heapSeqNum,
870 size_t capacity,
871 size_t numInputSlots,
872 const char *componentName, const char *name = "EncryptedInput");
873
874 ~EncryptedLinearInputBuffers() override = default;
875
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700876 std::unique_ptr<InputBuffers> toArrayMode(size_t size) override;
877
878protected:
879 sp<Codec2Buffer> createNewBuffer() override;
Wonsik Kim469c8342019-04-11 16:46:09 -0700880
881private:
Wonsik Kim469c8342019-04-11 16:46:09 -0700882 struct Entry {
883 std::weak_ptr<C2LinearBlock> block;
884 sp<IMemory> memory;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700885 int32_t heapSeqNum;
Wonsik Kim469c8342019-04-11 16:46:09 -0700886 };
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700887
888 static sp<Codec2Buffer> Alloc(
889 const std::shared_ptr<C2BlockPool> &pool,
890 const sp<AMessage> &format,
891 C2MemoryUsage usage,
892 const std::shared_ptr<std::vector<Entry>> &memoryVector);
893
894 C2MemoryUsage mUsage;
895 sp<MemoryDealer> mDealer;
896 sp<ICrypto> mCrypto;
897 std::shared_ptr<std::vector<Entry>> mMemoryVector;
Wonsik Kim469c8342019-04-11 16:46:09 -0700898};
899
900class GraphicMetadataInputBuffers : public InputBuffers {
901public:
902 GraphicMetadataInputBuffers(const char *componentName, const char *name = "2D-MetaInput");
903 ~GraphicMetadataInputBuffers() override = default;
904
905 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override;
906
907 bool releaseBuffer(
908 const sp<MediaCodecBuffer> &buffer,
909 std::shared_ptr<C2Buffer> *c2buffer,
910 bool release) override;
911
912 bool expireComponentBuffer(
913 const std::shared_ptr<C2Buffer> &c2buffer) override;
914
915 void flush() override;
916
917 std::unique_ptr<InputBuffers> toArrayMode(size_t size) final;
918
Wonsik Kim0487b782020-10-28 11:45:50 -0700919 size_t numActiveSlots() const final;
Wonsik Kim469c8342019-04-11 16:46:09 -0700920
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700921protected:
922 sp<Codec2Buffer> createNewBuffer() override;
923
Wonsik Kim469c8342019-04-11 16:46:09 -0700924private:
925 FlexBuffersImpl mImpl;
926 std::shared_ptr<C2AllocatorStore> mStore;
927};
928
929class GraphicInputBuffers : public InputBuffers {
930public:
Wonsik Kim41d83432020-04-27 16:40:49 -0700931 GraphicInputBuffers(const char *componentName, const char *name = "2D-BB-Input");
Wonsik Kim469c8342019-04-11 16:46:09 -0700932 ~GraphicInputBuffers() override = default;
933
934 bool requestNewBuffer(size_t *index, sp<MediaCodecBuffer> *buffer) override;
935
936 bool releaseBuffer(
937 const sp<MediaCodecBuffer> &buffer,
938 std::shared_ptr<C2Buffer> *c2buffer,
939 bool release) override;
940
941 bool expireComponentBuffer(
942 const std::shared_ptr<C2Buffer> &c2buffer) override;
943
944 void flush() override;
945
946 std::unique_ptr<InputBuffers> toArrayMode(
947 size_t size) final;
948
Wonsik Kim0487b782020-10-28 11:45:50 -0700949 size_t numActiveSlots() const final;
Wonsik Kim469c8342019-04-11 16:46:09 -0700950
Songyue Han1e6769b2023-08-30 18:09:27 +0000951 uint32_t getPixelFormatIfApplicable() override;
952
953 bool resetPixelFormatIfApplicable() override;
954
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700955protected:
956 sp<Codec2Buffer> createNewBuffer() override;
957
Wonsik Kim469c8342019-04-11 16:46:09 -0700958private:
959 FlexBuffersImpl mImpl;
960 std::shared_ptr<LocalBufferPool> mLocalBufferPool;
Songyue Han1e6769b2023-08-30 18:09:27 +0000961 uint32_t mPixelFormat;
Wonsik Kim469c8342019-04-11 16:46:09 -0700962};
963
964class DummyInputBuffers : public InputBuffers {
965public:
966 DummyInputBuffers(const char *componentName, const char *name = "2D-Input")
967 : InputBuffers(componentName, name) { }
968 ~DummyInputBuffers() override = default;
969
970 bool requestNewBuffer(size_t *, sp<MediaCodecBuffer> *) override {
971 return false;
972 }
973
974 bool releaseBuffer(
975 const sp<MediaCodecBuffer> &, std::shared_ptr<C2Buffer> *, bool) override {
976 return false;
977 }
978
979 bool expireComponentBuffer(const std::shared_ptr<C2Buffer> &) override {
980 return false;
981 }
982 void flush() override {
983 }
984
985 std::unique_ptr<InputBuffers> toArrayMode(size_t) final {
986 return nullptr;
987 }
988
989 bool isArrayMode() const final { return true; }
990
991 void getArray(Vector<sp<MediaCodecBuffer>> *array) const final {
992 array->clear();
993 }
994
Wonsik Kim0487b782020-10-28 11:45:50 -0700995 size_t numActiveSlots() const final {
Wonsik Kim469c8342019-04-11 16:46:09 -0700996 return 0u;
997 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700998
999protected:
1000 sp<Codec2Buffer> createNewBuffer() override {
1001 return nullptr;
1002 }
Wonsik Kim469c8342019-04-11 16:46:09 -07001003};
1004
1005class OutputBuffersArray : public OutputBuffers {
1006public:
1007 OutputBuffersArray(const char *componentName, const char *name = "Output[N]")
1008 : OutputBuffers(componentName, name) { }
1009 ~OutputBuffersArray() override = default;
1010
1011 /**
1012 * Initialize this object from the non-array state. We keep existing slots
1013 * at the same index, and for empty slots we allocate client buffers with
1014 * the given allocate function. If the number of slots is less than minSize,
1015 * we fill the array to the minimum size.
1016 *
1017 * \param impl[in] existing non-array state
1018 * \param minSize[in] minimum size of the array
1019 * \param allocate[in] allocate function to fill empty slots
1020 */
1021 void initialize(
1022 const FlexBuffersImpl &impl,
1023 size_t minSize,
1024 std::function<sp<Codec2Buffer>()> allocate);
1025
1026 bool isArrayMode() const final { return true; }
1027
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001028 std::unique_ptr<OutputBuffersArray> toArrayMode(size_t) final {
Wonsik Kim469c8342019-04-11 16:46:09 -07001029 return nullptr;
1030 }
1031
1032 status_t registerBuffer(
1033 const std::shared_ptr<C2Buffer> &buffer,
1034 size_t *index,
1035 sp<MediaCodecBuffer> *clientBuffer) final;
1036
1037 status_t registerCsd(
1038 const C2StreamInitDataInfo::output *csd,
1039 size_t *index,
1040 sp<MediaCodecBuffer> *clientBuffer) final;
1041
1042 bool releaseBuffer(
1043 const sp<MediaCodecBuffer> &buffer, std::shared_ptr<C2Buffer> *c2buffer) override;
1044
1045 void flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) override;
1046
1047 void getArray(Vector<sp<MediaCodecBuffer>> *array) const final;
1048
Wonsik Kim0487b782020-10-28 11:45:50 -07001049 size_t numActiveSlots() const final;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001050
Wonsik Kim469c8342019-04-11 16:46:09 -07001051 /**
1052 * Reallocate the array, filled with buffers with the same size as given
1053 * buffer.
1054 *
1055 * \param c2buffer[in] the reference buffer
1056 */
1057 void realloc(const std::shared_ptr<C2Buffer> &c2buffer);
1058
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001059 /**
1060 * Grow the array to the new size. It is a programming error to supply
1061 * smaller size as the new size.
1062 *
1063 * \param newSize[in] new size of the array.
1064 */
1065 void grow(size_t newSize);
Wonsik Kim469c8342019-04-11 16:46:09 -07001066
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001067 /**
1068 * Transfer the SkipCutBuffer and the output stash from another
1069 * OutputBuffers.
1070 */
1071 void transferFrom(OutputBuffers* source);
1072
Wonsik Kim469c8342019-04-11 16:46:09 -07001073private:
1074 BuffersArrayImpl mImpl;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001075 std::function<sp<Codec2Buffer>()> mAlloc;
Wonsik Kim469c8342019-04-11 16:46:09 -07001076};
1077
1078class FlexOutputBuffers : public OutputBuffers {
1079public:
1080 FlexOutputBuffers(const char *componentName, const char *name = "Output[]")
1081 : OutputBuffers(componentName, name),
Songyue Han1e6769b2023-08-30 18:09:27 +00001082 mImpl(mName),
1083 mPixelFormat(0) { }
Wonsik Kim469c8342019-04-11 16:46:09 -07001084
1085 status_t registerBuffer(
1086 const std::shared_ptr<C2Buffer> &buffer,
1087 size_t *index,
1088 sp<MediaCodecBuffer> *clientBuffer) override;
1089
1090 status_t registerCsd(
1091 const C2StreamInitDataInfo::output *csd,
1092 size_t *index,
1093 sp<MediaCodecBuffer> *clientBuffer) final;
1094
1095 bool releaseBuffer(
1096 const sp<MediaCodecBuffer> &buffer,
1097 std::shared_ptr<C2Buffer> *c2buffer) override;
1098
1099 void flush(
1100 const std::list<std::unique_ptr<C2Work>> &flushedWork) override;
1101
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001102 std::unique_ptr<OutputBuffersArray> toArrayMode(size_t size) override;
Wonsik Kim469c8342019-04-11 16:46:09 -07001103
Wonsik Kim0487b782020-10-28 11:45:50 -07001104 size_t numActiveSlots() const final;
Wonsik Kim469c8342019-04-11 16:46:09 -07001105
1106 /**
1107 * Return an appropriate Codec2Buffer object for the type of buffers.
1108 *
1109 * \param buffer C2Buffer object to wrap.
1110 *
1111 * \return appropriate Codec2Buffer object to wrap |buffer|.
1112 */
1113 virtual sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) = 0;
1114
1115 /**
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001116 * Return a function that allocates an appropriate Codec2Buffer object for
1117 * the type of buffers, to be used as an empty array buffer. The function
1118 * must not refer to this pointer, since it may be used after this object
1119 * destructs.
Wonsik Kim469c8342019-04-11 16:46:09 -07001120 *
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001121 * \return a function that allocates appropriate Codec2Buffer object,
1122 * which can copy() from C2Buffers.
Wonsik Kim469c8342019-04-11 16:46:09 -07001123 */
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001124 virtual std::function<sp<Codec2Buffer>()> getAlloc() = 0;
Wonsik Kim469c8342019-04-11 16:46:09 -07001125
Songyue Han1e6769b2023-08-30 18:09:27 +00001126 uint32_t getPixelFormatIfApplicable() override;
1127
1128 bool resetPixelFormatIfApplicable() override;
Wonsik Kim469c8342019-04-11 16:46:09 -07001129private:
1130 FlexBuffersImpl mImpl;
Songyue Han1e6769b2023-08-30 18:09:27 +00001131
1132 uint32_t mPixelFormat;
1133
1134 /**
1135 * extract pixel format from C2Buffer when register.
1136 *
1137 * \param buffer The C2Buffer used to extract pixel format.
1138 */
1139 bool extractPixelFormatFromC2Buffer(const std::shared_ptr<C2Buffer> &buffer);
Wonsik Kim469c8342019-04-11 16:46:09 -07001140};
1141
1142class LinearOutputBuffers : public FlexOutputBuffers {
1143public:
1144 LinearOutputBuffers(const char *componentName, const char *name = "1D-Output")
1145 : FlexOutputBuffers(componentName, name) { }
1146
1147 void flush(
1148 const std::list<std::unique_ptr<C2Work>> &flushedWork) override;
1149
1150 sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) override;
1151
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001152 std::function<sp<Codec2Buffer>()> getAlloc() override;
Wonsik Kim469c8342019-04-11 16:46:09 -07001153};
1154
1155class GraphicOutputBuffers : public FlexOutputBuffers {
1156public:
1157 GraphicOutputBuffers(const char *componentName, const char *name = "2D-Output")
1158 : FlexOutputBuffers(componentName, name) { }
1159
1160 sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) override;
1161
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001162 std::function<sp<Codec2Buffer>()> getAlloc() override;
Wonsik Kim469c8342019-04-11 16:46:09 -07001163};
1164
1165class RawGraphicOutputBuffers : public FlexOutputBuffers {
1166public:
Wonsik Kim41d83432020-04-27 16:40:49 -07001167 RawGraphicOutputBuffers(const char *componentName, const char *name = "2D-BB-Output");
Wonsik Kim469c8342019-04-11 16:46:09 -07001168 ~RawGraphicOutputBuffers() override = default;
1169
1170 sp<Codec2Buffer> wrap(const std::shared_ptr<C2Buffer> &buffer) override;
1171
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001172 std::function<sp<Codec2Buffer>()> getAlloc() override;
Wonsik Kim469c8342019-04-11 16:46:09 -07001173
1174private:
1175 std::shared_ptr<LocalBufferPool> mLocalBufferPool;
1176};
1177
1178} // namespace android
1179
1180#endif // CCODEC_BUFFERS_H_