blob: 15d2989a4cfa1e74a176759e4ef45eb40f50efd8 [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodec"
19#include <utils/Log.h>
20
21#include <sstream>
22#include <thread>
23
24#include <C2Config.h>
25#include <C2Debug.h>
26#include <C2ParamInternal.h>
27#include <C2PlatformSupport.h>
28
Pawin Vongmasa36653902018-11-15 00:10:25 -080029#include <android/IOMXBufferSource.h>
Pawin Vongmasabf69de92019-10-29 06:21:27 -070030#include <android/hardware/media/c2/1.0/IInputSurface.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080031#include <android/hardware/media/omx/1.0/IGraphicBufferSource.h>
32#include <android/hardware/media/omx/1.0/IOmx.h>
33#include <android-base/stringprintf.h>
34#include <cutils/properties.h>
35#include <gui/IGraphicBufferProducer.h>
36#include <gui/Surface.h>
37#include <gui/bufferqueue/1.0/H2BGraphicBufferProducer.h>
Wonsik Kim9917d4a2019-10-24 12:56:38 -070038#include <media/omx/1.0/WOmxNode.h>
39#include <media/openmax/OMX_Core.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080040#include <media/openmax/OMX_IndexExt.h>
Wonsik Kim9917d4a2019-10-24 12:56:38 -070041#include <media/stagefright/omx/1.0/WGraphicBufferSource.h>
42#include <media/stagefright/omx/OmxGraphicBufferSource.h>
Wonsik Kim155d5cb2019-10-09 12:49:49 -070043#include <media/stagefright/CCodec.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080044#include <media/stagefright/BufferProducerWrapper.h>
45#include <media/stagefright/MediaCodecConstants.h>
46#include <media/stagefright/PersistentSurface.h>
ted.sun765db4d2020-06-23 14:03:41 +080047#include <utils/NativeHandle.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080048
49#include "C2OMXNode.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080050#include "CCodecBufferChannel.h"
Wonsik Kim155d5cb2019-10-09 12:49:49 -070051#include "CCodecConfig.h"
Wonsik Kimfb7a7672019-12-27 17:13:33 -080052#include "Codec2Mapper.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080053#include "InputSurfaceWrapper.h"
54
55extern "C" android::PersistentSurface *CreateInputSurface();
56
57namespace android {
58
59using namespace std::chrono_literals;
60using ::android::hardware::graphics::bufferqueue::V1_0::utils::H2BGraphicBufferProducer;
61using android::base::StringPrintf;
Pawin Vongmasad0f0e142018-11-15 03:36:28 -080062using ::android::hardware::media::c2::V1_0::IInputSurface;
Pawin Vongmasa36653902018-11-15 00:10:25 -080063
Wonsik Kim9917d4a2019-10-24 12:56:38 -070064typedef hardware::media::omx::V1_0::IGraphicBufferSource HGraphicBufferSource;
Wonsik Kim155d5cb2019-10-09 12:49:49 -070065typedef CCodecConfig Config;
Wonsik Kim9917d4a2019-10-24 12:56:38 -070066
Pawin Vongmasa36653902018-11-15 00:10:25 -080067namespace {
68
69class CCodecWatchdog : public AHandler {
70private:
71 enum {
72 kWhatWatch,
73 };
74 constexpr static int64_t kWatchIntervalUs = 3300000; // 3.3 secs
75
76public:
77 static sp<CCodecWatchdog> getInstance() {
78 static sp<CCodecWatchdog> instance(new CCodecWatchdog);
79 static std::once_flag flag;
80 // Call Init() only once.
81 std::call_once(flag, Init, instance);
82 return instance;
83 }
84
85 ~CCodecWatchdog() = default;
86
87 void watch(sp<CCodec> codec) {
88 bool shouldPost = false;
89 {
90 Mutexed<std::set<wp<CCodec>>>::Locked codecs(mCodecsToWatch);
91 // If a watch message is in flight, piggy-back this instance as well.
92 // Otherwise, post a new watch message.
93 shouldPost = codecs->empty();
94 codecs->emplace(codec);
95 }
96 if (shouldPost) {
97 ALOGV("posting watch message");
98 (new AMessage(kWhatWatch, this))->post(kWatchIntervalUs);
99 }
100 }
101
102protected:
103 void onMessageReceived(const sp<AMessage> &msg) {
104 switch (msg->what()) {
105 case kWhatWatch: {
106 Mutexed<std::set<wp<CCodec>>>::Locked codecs(mCodecsToWatch);
107 ALOGV("watch for %zu codecs", codecs->size());
108 for (auto it = codecs->begin(); it != codecs->end(); ++it) {
109 sp<CCodec> codec = it->promote();
110 if (codec == nullptr) {
111 continue;
112 }
113 codec->initiateReleaseIfStuck();
114 }
115 codecs->clear();
116 break;
117 }
118
119 default: {
120 TRESPASS("CCodecWatchdog: unrecognized message");
121 }
122 }
123 }
124
125private:
126 CCodecWatchdog() : mLooper(new ALooper) {}
127
128 static void Init(const sp<CCodecWatchdog> &thiz) {
129 ALOGV("Init");
130 thiz->mLooper->setName("CCodecWatchdog");
131 thiz->mLooper->registerHandler(thiz);
132 thiz->mLooper->start();
133 }
134
135 sp<ALooper> mLooper;
136
137 Mutexed<std::set<wp<CCodec>>> mCodecsToWatch;
138};
139
140class C2InputSurfaceWrapper : public InputSurfaceWrapper {
141public:
142 explicit C2InputSurfaceWrapper(
143 const std::shared_ptr<Codec2Client::InputSurface> &surface) :
144 mSurface(surface) {
145 }
146
147 ~C2InputSurfaceWrapper() override = default;
148
149 status_t connect(const std::shared_ptr<Codec2Client::Component> &comp) override {
150 if (mConnection != nullptr) {
151 return ALREADY_EXISTS;
152 }
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800153 return toStatusT(comp->connectToInputSurface(mSurface, &mConnection));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800154 }
155
156 void disconnect() override {
157 if (mConnection != nullptr) {
158 mConnection->disconnect();
159 mConnection = nullptr;
160 }
161 }
162
163 status_t start() override {
164 // InputSurface does not distinguish started state
165 return OK;
166 }
167
168 status_t signalEndOfInputStream() override {
169 C2InputSurfaceEosTuning eos(true);
170 std::vector<std::unique_ptr<C2SettingResult>> failures;
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800171 c2_status_t err = mSurface->config({&eos}, C2_MAY_BLOCK, &failures);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800172 if (err != C2_OK) {
173 return UNKNOWN_ERROR;
174 }
175 return OK;
176 }
177
178 status_t configure(Config &config __unused) {
179 // TODO
180 return OK;
181 }
182
183private:
184 std::shared_ptr<Codec2Client::InputSurface> mSurface;
185 std::shared_ptr<Codec2Client::InputSurfaceConnection> mConnection;
186};
187
188class GraphicBufferSourceWrapper : public InputSurfaceWrapper {
189public:
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700190 typedef hardware::media::omx::V1_0::Status OmxStatus;
191
Pawin Vongmasa36653902018-11-15 00:10:25 -0800192 GraphicBufferSourceWrapper(
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700193 const sp<HGraphicBufferSource> &source,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800194 uint32_t width,
Wonsik Kim9eac4d12019-05-23 12:58:48 -0700195 uint32_t height,
196 uint64_t usage)
Pawin Vongmasa36653902018-11-15 00:10:25 -0800197 : mSource(source), mWidth(width), mHeight(height) {
198 mDataSpace = HAL_DATASPACE_BT709;
Wonsik Kim9eac4d12019-05-23 12:58:48 -0700199 mConfig.mUsage = usage;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800200 }
201 ~GraphicBufferSourceWrapper() override = default;
202
203 status_t connect(const std::shared_ptr<Codec2Client::Component> &comp) override {
204 mNode = new C2OMXNode(comp);
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700205 mOmxNode = new hardware::media::omx::V1_0::utils::TWOmxNode(mNode);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800206 mNode->setFrameSize(mWidth, mHeight);
207
Wonsik Kim9eac4d12019-05-23 12:58:48 -0700208 // Usage is queried during configure(), so setting it beforehand.
209 OMX_U32 usage = mConfig.mUsage & 0xFFFFFFFF;
210 (void)mNode->setParameter(
211 (OMX_INDEXTYPE)OMX_IndexParamConsumerUsageBits,
212 &usage, sizeof(usage));
213
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700214 mSource->configure(
215 mOmxNode, static_cast<hardware::graphics::common::V1_0::Dataspace>(mDataSpace));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800216 return OK;
217 }
218
219 void disconnect() override {
220 if (mNode == nullptr) {
221 return;
222 }
223 sp<IOMXBufferSource> source = mNode->getSource();
224 if (source == nullptr) {
225 ALOGD("GBSWrapper::disconnect: node is not configured with OMXBufferSource.");
226 return;
227 }
228 source->onOmxIdle();
229 source->onOmxLoaded();
230 mNode.clear();
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700231 mOmxNode.clear();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800232 }
233
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700234 status_t GetStatus(hardware::Return<OmxStatus> &&status) {
235 if (status.isOk()) {
236 return static_cast<status_t>(status.withDefault(OmxStatus::UNKNOWN_ERROR));
237 } else if (status.isDeadObject()) {
238 return DEAD_OBJECT;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800239 }
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700240 return UNKNOWN_ERROR;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800241 }
242
243 status_t start() override {
244 sp<IOMXBufferSource> source = mNode->getSource();
245 if (source == nullptr) {
246 return NO_INIT;
247 }
Taehwan Kim8b3bcdd2020-11-26 22:40:40 +0900248
Wonsik Kim0f6b61d2021-01-05 18:55:22 -0800249 size_t numSlots = 16;
Wonsik Kim34d66012021-03-01 16:40:33 -0800250 constexpr OMX_U32 kPortIndexInput = 0;
Taehwan Kim8b3bcdd2020-11-26 22:40:40 +0900251
Wonsik Kim34d66012021-03-01 16:40:33 -0800252 OMX_PARAM_PORTDEFINITIONTYPE param;
253 param.nPortIndex = kPortIndexInput;
254 status_t err = mNode->getParameter(OMX_IndexParamPortDefinition,
255 &param, sizeof(param));
256 if (err == OK) {
257 numSlots = param.nBufferCountActual;
Taehwan Kim8b3bcdd2020-11-26 22:40:40 +0900258 }
259
260 for (size_t i = 0; i < numSlots; ++i) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800261 source->onInputBufferAdded(i);
262 }
263
264 source->onOmxExecuting();
265 return OK;
266 }
267
268 status_t signalEndOfInputStream() override {
269 return GetStatus(mSource->signalEndOfInputStream());
270 }
271
272 status_t configure(Config &config) {
273 std::stringstream status;
274 status_t err = OK;
275
276 // handle each configuration granually, in case we need to handle part of the configuration
277 // elsewhere
278
279 // TRICKY: we do not unset frame delay repeating
280 if (config.mMinFps > 0 && config.mMinFps != mConfig.mMinFps) {
281 int64_t us = 1e6 / config.mMinFps + 0.5;
282 status_t res = GetStatus(mSource->setRepeatPreviousFrameDelayUs(us));
283 status << " minFps=" << config.mMinFps << " => repeatDelayUs=" << us;
284 if (res != OK) {
285 status << " (=> " << asString(res) << ")";
286 err = res;
287 }
288 mConfig.mMinFps = config.mMinFps;
289 }
290
291 // pts gap
292 if (config.mMinAdjustedFps > 0 || config.mFixedAdjustedFps > 0) {
293 if (mNode != nullptr) {
294 OMX_PARAM_U32TYPE ptrGapParam = {};
295 ptrGapParam.nSize = sizeof(OMX_PARAM_U32TYPE);
Wonsik Kim95ba0162019-03-19 15:51:54 -0700296 float gap = (config.mMinAdjustedFps > 0)
Pawin Vongmasa36653902018-11-15 00:10:25 -0800297 ? c2_min(INT32_MAX + 0., 1e6 / config.mMinAdjustedFps + 0.5)
298 : c2_max(0. - INT32_MAX, -1e6 / config.mFixedAdjustedFps - 0.5);
Wonsik Kim95ba0162019-03-19 15:51:54 -0700299 // float -> uint32_t is undefined if the value is negative.
300 // First convert to int32_t to ensure the expected behavior.
301 ptrGapParam.nU32 = int32_t(gap);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800302 (void)mNode->setParameter(
303 (OMX_INDEXTYPE)OMX_IndexParamMaxFrameDurationForBitrateControl,
304 &ptrGapParam, sizeof(ptrGapParam));
305 }
306 }
307
308 // max fps
309 // TRICKY: we do not unset max fps to 0 unless using fixed fps
Wonsik Kim95ba0162019-03-19 15:51:54 -0700310 if ((config.mMaxFps > 0 || (config.mFixedAdjustedFps > 0 && config.mMaxFps == -1))
Pawin Vongmasa36653902018-11-15 00:10:25 -0800311 && config.mMaxFps != mConfig.mMaxFps) {
312 status_t res = GetStatus(mSource->setMaxFps(config.mMaxFps));
313 status << " maxFps=" << config.mMaxFps;
314 if (res != OK) {
315 status << " (=> " << asString(res) << ")";
316 err = res;
317 }
318 mConfig.mMaxFps = config.mMaxFps;
319 }
320
321 if (config.mTimeOffsetUs != mConfig.mTimeOffsetUs) {
322 status_t res = GetStatus(mSource->setTimeOffsetUs(config.mTimeOffsetUs));
323 status << " timeOffset " << config.mTimeOffsetUs << "us";
324 if (res != OK) {
325 status << " (=> " << asString(res) << ")";
326 err = res;
327 }
328 mConfig.mTimeOffsetUs = config.mTimeOffsetUs;
329 }
330
331 if (config.mCaptureFps != mConfig.mCaptureFps || config.mCodedFps != mConfig.mCodedFps) {
332 status_t res =
333 GetStatus(mSource->setTimeLapseConfig(config.mCodedFps, config.mCaptureFps));
334 status << " timeLapse " << config.mCaptureFps << "fps as " << config.mCodedFps << "fps";
335 if (res != OK) {
336 status << " (=> " << asString(res) << ")";
337 err = res;
338 }
339 mConfig.mCaptureFps = config.mCaptureFps;
340 mConfig.mCodedFps = config.mCodedFps;
341 }
342
343 if (config.mStartAtUs != mConfig.mStartAtUs
344 || (config.mStopped != mConfig.mStopped && !config.mStopped)) {
345 status_t res = GetStatus(mSource->setStartTimeUs(config.mStartAtUs));
346 status << " start at " << config.mStartAtUs << "us";
347 if (res != OK) {
348 status << " (=> " << asString(res) << ")";
349 err = res;
350 }
351 mConfig.mStartAtUs = config.mStartAtUs;
352 mConfig.mStopped = config.mStopped;
353 }
354
355 // suspend-resume
356 if (config.mSuspended != mConfig.mSuspended) {
357 status_t res = GetStatus(mSource->setSuspend(config.mSuspended, config.mSuspendAtUs));
358 status << " " << (config.mSuspended ? "suspend" : "resume")
359 << " at " << config.mSuspendAtUs << "us";
360 if (res != OK) {
361 status << " (=> " << asString(res) << ")";
362 err = res;
363 }
364 mConfig.mSuspended = config.mSuspended;
365 mConfig.mSuspendAtUs = config.mSuspendAtUs;
366 }
367
368 if (config.mStopped != mConfig.mStopped && config.mStopped) {
369 status_t res = GetStatus(mSource->setStopTimeUs(config.mStopAtUs));
370 status << " stop at " << config.mStopAtUs << "us";
371 if (res != OK) {
372 status << " (=> " << asString(res) << ")";
373 err = res;
374 } else {
375 status << " delayUs";
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700376 hardware::Return<void> trans = mSource->getStopTimeOffsetUs(
377 [&res, &delayUs = config.mInputDelayUs](
378 auto status, auto stopTimeOffsetUs) {
379 res = static_cast<status_t>(status);
380 delayUs = stopTimeOffsetUs;
381 });
382 if (!trans.isOk()) {
383 res = trans.isDeadObject() ? DEAD_OBJECT : UNKNOWN_ERROR;
384 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800385 if (res != OK) {
386 status << " (=> " << asString(res) << ")";
387 } else {
388 status << "=" << config.mInputDelayUs << "us";
389 }
390 mConfig.mInputDelayUs = config.mInputDelayUs;
391 }
392 mConfig.mStopAtUs = config.mStopAtUs;
393 mConfig.mStopped = config.mStopped;
394 }
395
396 // color aspects (android._color-aspects)
397
Wonsik Kim9eac4d12019-05-23 12:58:48 -0700398 // consumer usage is queried earlier.
399
Wonsik Kimbd557932019-07-02 15:51:20 -0700400 if (status.str().empty()) {
401 ALOGD("ISConfig not changed");
402 } else {
403 ALOGD("ISConfig%s", status.str().c_str());
404 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800405 return err;
406 }
407
Wonsik Kim4f3314d2019-03-26 17:00:34 -0700408 void onInputBufferDone(c2_cntr64_t index) override {
409 mNode->onInputBufferDone(index);
410 }
411
Wonsik Kim673dd192021-01-29 14:58:12 -0800412 android_dataspace getDataspace() override {
413 return mNode->getDataspace();
414 }
415
Pawin Vongmasa36653902018-11-15 00:10:25 -0800416private:
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700417 sp<HGraphicBufferSource> mSource;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800418 sp<C2OMXNode> mNode;
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700419 sp<hardware::media::omx::V1_0::IOmxNode> mOmxNode;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800420 uint32_t mWidth;
421 uint32_t mHeight;
422 Config mConfig;
423};
424
425class Codec2ClientInterfaceWrapper : public C2ComponentStore {
426 std::shared_ptr<Codec2Client> mClient;
427
428public:
429 Codec2ClientInterfaceWrapper(std::shared_ptr<Codec2Client> client)
430 : mClient(client) { }
431
432 virtual ~Codec2ClientInterfaceWrapper() = default;
433
434 virtual c2_status_t config_sm(
435 const std::vector<C2Param *> &params,
436 std::vector<std::unique_ptr<C2SettingResult>> *const failures) {
437 return mClient->config(params, C2_MAY_BLOCK, failures);
438 };
439
440 virtual c2_status_t copyBuffer(
441 std::shared_ptr<C2GraphicBuffer>,
442 std::shared_ptr<C2GraphicBuffer>) {
443 return C2_OMITTED;
444 }
445
446 virtual c2_status_t createComponent(
447 C2String, std::shared_ptr<C2Component> *const component) {
448 component->reset();
449 return C2_OMITTED;
450 }
451
452 virtual c2_status_t createInterface(
453 C2String, std::shared_ptr<C2ComponentInterface> *const interface) {
454 interface->reset();
455 return C2_OMITTED;
456 }
457
458 virtual c2_status_t query_sm(
459 const std::vector<C2Param *> &stackParams,
460 const std::vector<C2Param::Index> &heapParamIndices,
461 std::vector<std::unique_ptr<C2Param>> *const heapParams) const {
462 return mClient->query(stackParams, heapParamIndices, C2_MAY_BLOCK, heapParams);
463 }
464
465 virtual c2_status_t querySupportedParams_nb(
466 std::vector<std::shared_ptr<C2ParamDescriptor>> *const params) const {
467 return mClient->querySupportedParams(params);
468 }
469
470 virtual c2_status_t querySupportedValues_sm(
471 std::vector<C2FieldSupportedValuesQuery> &fields) const {
472 return mClient->querySupportedValues(fields, C2_MAY_BLOCK);
473 }
474
475 virtual C2String getName() const {
476 return mClient->getName();
477 }
478
479 virtual std::shared_ptr<C2ParamReflector> getParamReflector() const {
480 return mClient->getParamReflector();
481 }
482
483 virtual std::vector<std::shared_ptr<const C2Component::Traits>> listComponents() {
484 return std::vector<std::shared_ptr<const C2Component::Traits>>();
485 }
486};
487
Wonsik Kim3b4349a2020-11-10 11:54:15 -0800488void RevertOutputFormatIfNeeded(
489 const sp<AMessage> &oldFormat, sp<AMessage> &currentFormat) {
490 // We used to not report changes to these keys to the client.
491 const static std::set<std::string> sIgnoredKeys({
492 KEY_BIT_RATE,
Harish Mahendrakar8c537502021-02-23 21:20:22 -0800493 KEY_FRAME_RATE,
Wonsik Kim3b4349a2020-11-10 11:54:15 -0800494 KEY_MAX_BIT_RATE,
Harish Mahendrakar8c537502021-02-23 21:20:22 -0800495 KEY_MAX_WIDTH,
496 KEY_MAX_HEIGHT,
Wonsik Kim3b4349a2020-11-10 11:54:15 -0800497 "csd-0",
498 "csd-1",
499 "csd-2",
500 });
501 if (currentFormat == oldFormat) {
502 return;
503 }
504 sp<AMessage> diff = currentFormat->changesFrom(oldFormat);
505 AMessage::Type type;
506 for (size_t i = diff->countEntries(); i > 0; --i) {
507 if (sIgnoredKeys.count(diff->getEntryNameAt(i - 1, &type)) > 0) {
508 diff->removeEntryAt(i - 1);
509 }
510 }
511 if (diff->countEntries() == 0) {
512 currentFormat = oldFormat;
513 }
514}
515
Pawin Vongmasa36653902018-11-15 00:10:25 -0800516} // namespace
517
518// CCodec::ClientListener
519
520struct CCodec::ClientListener : public Codec2Client::Listener {
521
522 explicit ClientListener(const wp<CCodec> &codec) : mCodec(codec) {}
523
524 virtual void onWorkDone(
525 const std::weak_ptr<Codec2Client::Component>& component,
Wonsik Kimab34ed62019-01-31 15:28:46 -0800526 std::list<std::unique_ptr<C2Work>>& workItems) override {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800527 (void)component;
528 sp<CCodec> codec(mCodec.promote());
529 if (!codec) {
530 return;
531 }
Wonsik Kimab34ed62019-01-31 15:28:46 -0800532 codec->onWorkDone(workItems);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800533 }
534
535 virtual void onTripped(
536 const std::weak_ptr<Codec2Client::Component>& component,
537 const std::vector<std::shared_ptr<C2SettingResult>>& settingResult
538 ) override {
539 // TODO
540 (void)component;
541 (void)settingResult;
542 }
543
544 virtual void onError(
545 const std::weak_ptr<Codec2Client::Component>& component,
546 uint32_t errorCode) override {
Praveen Chavan72eff012020-11-20 23:20:28 -0800547 {
548 // Component is only used for reporting as we use a separate listener for each instance
549 std::shared_ptr<Codec2Client::Component> comp = component.lock();
550 if (!comp) {
551 ALOGD("Component died with error: 0x%x", errorCode);
552 } else {
553 ALOGD("Component \"%s\" returned error: 0x%x", comp->getName().c_str(), errorCode);
554 }
555 }
556
557 // Report to MediaCodec
Wonsik Kim10f33c02021-03-04 15:04:14 -0800558 // Note: for now we do not propagate the error code to MediaCodec
559 // except for C2_NO_MEMORY, as we would need to translate to a MediaCodec error.
Praveen Chavan72eff012020-11-20 23:20:28 -0800560 sp<CCodec> codec(mCodec.promote());
561 if (!codec || !codec->mCallback) {
562 return;
563 }
Wonsik Kim10f33c02021-03-04 15:04:14 -0800564 codec->mCallback->onError(
565 errorCode == C2_NO_MEMORY ? NO_MEMORY : UNKNOWN_ERROR,
566 ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800567 }
568
569 virtual void onDeath(
570 const std::weak_ptr<Codec2Client::Component>& component) override {
571 { // Log the death of the component.
572 std::shared_ptr<Codec2Client::Component> comp = component.lock();
573 if (!comp) {
574 ALOGE("Codec2 component died.");
575 } else {
576 ALOGE("Codec2 component \"%s\" died.", comp->getName().c_str());
577 }
578 }
579
580 // Report to MediaCodec.
581 sp<CCodec> codec(mCodec.promote());
582 if (!codec || !codec->mCallback) {
583 return;
584 }
585 codec->mCallback->onError(DEAD_OBJECT, ACTION_CODE_FATAL);
586 }
587
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800588 virtual void onFrameRendered(uint64_t bufferQueueId,
589 int32_t slotId,
590 int64_t timestampNs) override {
591 // TODO: implement
592 (void)bufferQueueId;
593 (void)slotId;
594 (void)timestampNs;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800595 }
596
597 virtual void onInputBufferDone(
Wonsik Kimab34ed62019-01-31 15:28:46 -0800598 uint64_t frameIndex, size_t arrayIndex) override {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800599 sp<CCodec> codec(mCodec.promote());
600 if (codec) {
Wonsik Kimab34ed62019-01-31 15:28:46 -0800601 codec->onInputBufferDone(frameIndex, arrayIndex);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800602 }
603 }
604
605private:
606 wp<CCodec> mCodec;
607};
608
609// CCodecCallbackImpl
610
611class CCodecCallbackImpl : public CCodecCallback {
612public:
613 explicit CCodecCallbackImpl(CCodec *codec) : mCodec(codec) {}
614 ~CCodecCallbackImpl() override = default;
615
616 void onError(status_t err, enum ActionCode actionCode) override {
617 mCodec->mCallback->onError(err, actionCode);
618 }
619
620 void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) override {
621 mCodec->mCallback->onOutputFramesRendered(
622 {RenderedFrameInfo(mediaTimeUs, renderTimeNs)});
623 }
624
Pawin Vongmasa36653902018-11-15 00:10:25 -0800625 void onOutputBuffersChanged() override {
626 mCodec->mCallback->onOutputBuffersChanged();
627 }
628
629private:
630 CCodec *mCodec;
631};
632
633// CCodec
634
635CCodec::CCodec()
Wonsik Kim155d5cb2019-10-09 12:49:49 -0700636 : mChannel(new CCodecBufferChannel(std::make_shared<CCodecCallbackImpl>(this))),
637 mConfig(new CCodecConfig) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800638}
639
640CCodec::~CCodec() {
641}
642
643std::shared_ptr<BufferChannelBase> CCodec::getBufferChannel() {
644 return mChannel;
645}
646
647status_t CCodec::tryAndReportOnError(std::function<status_t()> job) {
648 status_t err = job();
649 if (err != C2_OK) {
650 mCallback->onError(err, ACTION_CODE_FATAL);
651 }
652 return err;
653}
654
655void CCodec::initiateAllocateComponent(const sp<AMessage> &msg) {
656 auto setAllocating = [this] {
657 Mutexed<State>::Locked state(mState);
658 if (state->get() != RELEASED) {
659 return INVALID_OPERATION;
660 }
661 state->set(ALLOCATING);
662 return OK;
663 };
664 if (tryAndReportOnError(setAllocating) != OK) {
665 return;
666 }
667
668 sp<RefBase> codecInfo;
669 CHECK(msg->findObject("codecInfo", &codecInfo));
670 // For Codec 2.0 components, componentName == codecInfo->getCodecName().
671
672 sp<AMessage> allocMsg(new AMessage(kWhatAllocate, this));
673 allocMsg->setObject("codecInfo", codecInfo);
674 allocMsg->post();
675}
676
677void CCodec::allocate(const sp<MediaCodecInfo> &codecInfo) {
678 if (codecInfo == nullptr) {
679 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
680 return;
681 }
682 ALOGD("allocate(%s)", codecInfo->getCodecName());
683 mClientListener.reset(new ClientListener(this));
684
685 AString componentName = codecInfo->getCodecName();
686 std::shared_ptr<Codec2Client> client;
687
688 // set up preferred component store to access vendor store parameters
Pawin Vongmasa892c81d2019-03-12 00:56:50 -0700689 client = Codec2Client::CreateFromService("default");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800690 if (client) {
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800691 ALOGI("setting up '%s' as default (vendor) store", client->getServiceName().c_str());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800692 SetPreferredCodec2ComponentStore(
693 std::make_shared<Codec2ClientInterfaceWrapper>(client));
694 }
695
Chih-Yu Huangb8fe0792020-12-07 17:14:55 +0900696 std::shared_ptr<Codec2Client::Component> comp;
697 c2_status_t status = Codec2Client::CreateComponentByName(
Pawin Vongmasa36653902018-11-15 00:10:25 -0800698 componentName.c_str(),
699 mClientListener,
Chih-Yu Huangb8fe0792020-12-07 17:14:55 +0900700 &comp,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800701 &client);
Chih-Yu Huangb8fe0792020-12-07 17:14:55 +0900702 if (status != C2_OK) {
703 ALOGE("Failed Create component: %s, error=%d", componentName.c_str(), status);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800704 Mutexed<State>::Locked state(mState);
705 state->set(RELEASED);
706 state.unlock();
Chih-Yu Huangb8fe0792020-12-07 17:14:55 +0900707 mCallback->onError((status == C2_NO_MEMORY ? NO_MEMORY : UNKNOWN_ERROR), ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800708 state.lock();
709 return;
710 }
711 ALOGI("Created component [%s]", componentName.c_str());
712 mChannel->setComponent(comp);
713 auto setAllocated = [this, comp, client] {
714 Mutexed<State>::Locked state(mState);
715 if (state->get() != ALLOCATING) {
716 state->set(RELEASED);
717 return UNKNOWN_ERROR;
718 }
719 state->set(ALLOCATED);
720 state->comp = comp;
721 mClient = client;
722 return OK;
723 };
724 if (tryAndReportOnError(setAllocated) != OK) {
725 return;
726 }
727
728 // initialize config here in case setParameters is called prior to configure
Wonsik Kim155d5cb2019-10-09 12:49:49 -0700729 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
730 const std::unique_ptr<Config> &config = *configLocked;
Wonsik Kim8a6ed372019-12-03 16:05:51 -0800731 status_t err = config->initialize(mClient->getParamReflector(), comp);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800732 if (err != OK) {
733 ALOGW("Failed to initialize configuration support");
734 // TODO: report error once we complete implementation.
735 }
736 config->queryConfiguration(comp);
737
738 mCallback->onComponentAllocated(componentName.c_str());
739}
740
741void CCodec::initiateConfigureComponent(const sp<AMessage> &format) {
742 auto checkAllocated = [this] {
743 Mutexed<State>::Locked state(mState);
744 return (state->get() != ALLOCATED) ? UNKNOWN_ERROR : OK;
745 };
746 if (tryAndReportOnError(checkAllocated) != OK) {
747 return;
748 }
749
750 sp<AMessage> msg(new AMessage(kWhatConfigure, this));
751 msg->setMessage("format", format);
752 msg->post();
753}
754
755void CCodec::configure(const sp<AMessage> &msg) {
756 std::shared_ptr<Codec2Client::Component> comp;
757 auto checkAllocated = [this, &comp] {
758 Mutexed<State>::Locked state(mState);
759 if (state->get() != ALLOCATED) {
760 state->set(RELEASED);
761 return UNKNOWN_ERROR;
762 }
763 comp = state->comp;
764 return OK;
765 };
766 if (tryAndReportOnError(checkAllocated) != OK) {
767 return;
768 }
769
770 auto doConfig = [msg, comp, this]() -> status_t {
771 AString mime;
772 if (!msg->findString("mime", &mime)) {
773 return BAD_VALUE;
774 }
775
776 int32_t encoder;
777 if (!msg->findInt32("encoder", &encoder)) {
778 encoder = false;
779 }
780
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800781 int32_t flags;
782 if (!msg->findInt32("flags", &flags)) {
783 return BAD_VALUE;
784 }
785
Pawin Vongmasa36653902018-11-15 00:10:25 -0800786 // TODO: read from intf()
787 if ((!encoder) != (comp->getName().find("encoder") == std::string::npos)) {
788 return UNKNOWN_ERROR;
789 }
790
791 int32_t storeMeta;
792 if (encoder
793 && msg->findInt32("android._input-metadata-buffer-type", &storeMeta)
794 && storeMeta != kMetadataBufferTypeInvalid) {
795 if (storeMeta != kMetadataBufferTypeANWBuffer) {
796 ALOGD("Only ANW buffers are supported for legacy metadata mode");
797 return BAD_VALUE;
798 }
799 mChannel->setMetaMode(CCodecBufferChannel::MODE_ANW);
800 }
801
ted.sun765db4d2020-06-23 14:03:41 +0800802 status_t err = OK;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800803 sp<RefBase> obj;
804 sp<Surface> surface;
805 if (msg->findObject("native-window", &obj)) {
806 surface = static_cast<Surface *>(obj.get());
ted.sun765db4d2020-06-23 14:03:41 +0800807 // setup tunneled playback
808 if (surface != nullptr) {
809 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
810 const std::unique_ptr<Config> &config = *configLocked;
811 if ((config->mDomain & Config::IS_DECODER)
812 && (config->mDomain & Config::IS_VIDEO)) {
813 int32_t tunneled;
814 if (msg->findInt32("feature-tunneled-playback", &tunneled) && tunneled != 0) {
815 ALOGI("Configuring TUNNELED video playback.");
816
817 err = configureTunneledVideoPlayback(comp, &config->mSidebandHandle, msg);
818 if (err != OK) {
819 ALOGE("configureTunneledVideoPlayback failed!");
820 return err;
821 }
822 config->mTunneled = true;
823 }
824 }
825 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800826 setSurface(surface);
827 }
828
Wonsik Kim155d5cb2019-10-09 12:49:49 -0700829 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
830 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800831 config->mUsingSurface = surface != nullptr;
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800832 config->mBuffersBoundToCodec = ((flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) == 0);
833 ALOGD("[%s] buffers are %sbound to CCodec for this session",
834 comp->getName().c_str(), config->mBuffersBoundToCodec ? "" : "not ");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800835
Wonsik Kim1114eea2019-02-25 14:35:24 -0800836 // Enforce required parameters
837 int32_t i32;
838 float flt;
839 if (config->mDomain & Config::IS_AUDIO) {
840 if (!msg->findInt32(KEY_SAMPLE_RATE, &i32)) {
841 ALOGD("sample rate is missing, which is required for audio components.");
842 return BAD_VALUE;
843 }
844 if (!msg->findInt32(KEY_CHANNEL_COUNT, &i32)) {
845 ALOGD("channel count is missing, which is required for audio components.");
846 return BAD_VALUE;
847 }
848 if ((config->mDomain & Config::IS_ENCODER)
849 && !mime.equalsIgnoreCase(MEDIA_MIMETYPE_AUDIO_FLAC)
850 && !msg->findInt32(KEY_BIT_RATE, &i32)
851 && !msg->findFloat(KEY_BIT_RATE, &flt)) {
852 ALOGD("bitrate is missing, which is required for audio encoders.");
853 return BAD_VALUE;
854 }
855 }
Wonsik Kimd91f3fb2021-02-24 12:35:31 -0800856 int32_t width = 0;
857 int32_t height = 0;
Wonsik Kim1114eea2019-02-25 14:35:24 -0800858 if (config->mDomain & (Config::IS_IMAGE | Config::IS_VIDEO)) {
Wonsik Kimd91f3fb2021-02-24 12:35:31 -0800859 if (!msg->findInt32(KEY_WIDTH, &width)) {
Wonsik Kim1114eea2019-02-25 14:35:24 -0800860 ALOGD("width is missing, which is required for image/video components.");
861 return BAD_VALUE;
862 }
Wonsik Kimd91f3fb2021-02-24 12:35:31 -0800863 if (!msg->findInt32(KEY_HEIGHT, &height)) {
Wonsik Kim1114eea2019-02-25 14:35:24 -0800864 ALOGD("height is missing, which is required for image/video components.");
865 return BAD_VALUE;
866 }
867 if ((config->mDomain & Config::IS_ENCODER) && (config->mDomain & Config::IS_VIDEO)) {
Harish Mahendrakar71cbb9d2019-05-21 11:21:27 -0700868 int32_t mode = BITRATE_MODE_VBR;
869 if (msg->findInt32(KEY_BITRATE_MODE, &mode) && mode == BITRATE_MODE_CQ) {
Harish Mahendrakar817d3182019-03-11 16:37:47 -0700870 if (!msg->findInt32(KEY_QUALITY, &i32)) {
871 ALOGD("quality is missing, which is required for video encoders in CQ.");
872 return BAD_VALUE;
873 }
874 } else {
875 if (!msg->findInt32(KEY_BIT_RATE, &i32)
876 && !msg->findFloat(KEY_BIT_RATE, &flt)) {
877 ALOGD("bitrate is missing, which is required for video encoders.");
878 return BAD_VALUE;
879 }
Wonsik Kim1114eea2019-02-25 14:35:24 -0800880 }
881 if (!msg->findInt32(KEY_I_FRAME_INTERVAL, &i32)
882 && !msg->findFloat(KEY_I_FRAME_INTERVAL, &flt)) {
883 ALOGD("I frame interval is missing, which is required for video encoders.");
884 return BAD_VALUE;
885 }
Wonsik Kimaab2eea2019-05-22 10:37:58 -0700886 if (!msg->findInt32(KEY_FRAME_RATE, &i32)
887 && !msg->findFloat(KEY_FRAME_RATE, &flt)) {
888 ALOGD("frame rate is missing, which is required for video encoders.");
889 return BAD_VALUE;
890 }
Wonsik Kim1114eea2019-02-25 14:35:24 -0800891 }
892 }
893
Pawin Vongmasa36653902018-11-15 00:10:25 -0800894 /*
895 * Handle input surface configuration
896 */
897 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))
898 && (config->mDomain & Config::IS_ENCODER)) {
899 config->mISConfig.reset(new InputSurfaceWrapper::Config{});
900 {
901 config->mISConfig->mMinFps = 0;
902 int64_t value;
Chong Zhang038e8f82019-02-06 19:05:14 -0800903 if (msg->findInt64(KEY_REPEAT_PREVIOUS_FRAME_AFTER, &value) && value > 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800904 config->mISConfig->mMinFps = 1e6 / value;
905 }
Wonsik Kim95ba0162019-03-19 15:51:54 -0700906 if (!msg->findFloat(
907 KEY_MAX_FPS_TO_ENCODER, &config->mISConfig->mMaxFps)) {
908 config->mISConfig->mMaxFps = -1;
909 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800910 config->mISConfig->mMinAdjustedFps = 0;
911 config->mISConfig->mFixedAdjustedFps = 0;
Chong Zhang038e8f82019-02-06 19:05:14 -0800912 if (msg->findInt64(KEY_MAX_PTS_GAP_TO_ENCODER, &value)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800913 if (value < 0 && value >= INT32_MIN) {
914 config->mISConfig->mFixedAdjustedFps = -1e6 / value;
Wonsik Kim95ba0162019-03-19 15:51:54 -0700915 config->mISConfig->mMaxFps = -1;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800916 } else if (value > 0 && value <= INT32_MAX) {
917 config->mISConfig->mMinAdjustedFps = 1e6 / value;
918 }
919 }
920 }
921
922 {
Wonsik Kim8e55f3a2019-09-03 14:10:37 -0700923 bool captureFpsFound = false;
924 double timeLapseFps;
925 float captureRate;
926 if (msg->findDouble("time-lapse-fps", &timeLapseFps)) {
927 config->mISConfig->mCaptureFps = timeLapseFps;
928 captureFpsFound = true;
929 } else if (msg->findAsFloat(KEY_CAPTURE_RATE, &captureRate)) {
930 config->mISConfig->mCaptureFps = captureRate;
931 captureFpsFound = true;
932 }
933 if (captureFpsFound) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800934 (void)msg->findAsFloat(KEY_FRAME_RATE, &config->mISConfig->mCodedFps);
935 }
936 }
937
938 {
939 config->mISConfig->mSuspended = false;
940 config->mISConfig->mSuspendAtUs = -1;
941 int32_t value;
Chong Zhang038e8f82019-02-06 19:05:14 -0800942 if (msg->findInt32(KEY_CREATE_INPUT_SURFACE_SUSPENDED, &value) && value) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800943 config->mISConfig->mSuspended = true;
944 }
945 }
Wonsik Kim9eac4d12019-05-23 12:58:48 -0700946 config->mISConfig->mUsage = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800947 }
948
949 /*
950 * Handle desired color format.
951 */
Wonsik Kimd79ee1f2020-08-27 17:41:56 -0700952 int32_t defaultColorFormat = COLOR_FormatYUV420Flexible;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800953 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))) {
Wonsik Kimd79ee1f2020-08-27 17:41:56 -0700954 int32_t format = 0;
955 // Query vendor format for Flexible YUV
956 std::vector<std::unique_ptr<C2Param>> heapParams;
957 C2StoreFlexiblePixelFormatDescriptorsInfo *pixelFormatInfo = nullptr;
958 if (mClient->query(
959 {},
960 {C2StoreFlexiblePixelFormatDescriptorsInfo::PARAM_TYPE},
961 C2_MAY_BLOCK,
962 &heapParams) == C2_OK
963 && heapParams.size() == 1u) {
964 pixelFormatInfo = C2StoreFlexiblePixelFormatDescriptorsInfo::From(
965 heapParams[0].get());
966 } else {
967 pixelFormatInfo = nullptr;
968 }
969 std::optional<uint32_t> flexPixelFormat{};
970 std::optional<uint32_t> flexPlanarPixelFormat{};
971 std::optional<uint32_t> flexSemiPlanarPixelFormat{};
972 if (pixelFormatInfo && *pixelFormatInfo) {
973 for (size_t i = 0; i < pixelFormatInfo->flexCount(); ++i) {
974 const C2FlexiblePixelFormatDescriptorStruct &desc =
975 pixelFormatInfo->m.values[i];
976 if (desc.bitDepth != 8
977 || desc.subsampling != C2Color::YUV_420
978 // TODO(b/180076105): some device report wrong layout
979 // || desc.layout == C2Color::INTERLEAVED_PACKED
980 // || desc.layout == C2Color::INTERLEAVED_ALIGNED
981 || desc.layout == C2Color::UNKNOWN_LAYOUT) {
982 continue;
983 }
984 if (!flexPixelFormat) {
985 flexPixelFormat = desc.pixelFormat;
986 }
987 if (desc.layout == C2Color::PLANAR_PACKED && !flexPlanarPixelFormat) {
988 flexPlanarPixelFormat = desc.pixelFormat;
989 }
990 if (desc.layout == C2Color::SEMIPLANAR_PACKED && !flexSemiPlanarPixelFormat) {
991 flexSemiPlanarPixelFormat = desc.pixelFormat;
992 }
993 }
994 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800995 if (!msg->findInt32(KEY_COLOR_FORMAT, &format)) {
Wonsik Kimd79ee1f2020-08-27 17:41:56 -0700996 // Also handle default color format (encoders require color format, so this is only
997 // needed for decoders.
Pawin Vongmasa36653902018-11-15 00:10:25 -0800998 if (!(config->mDomain & Config::IS_ENCODER)) {
Wonsik Kimd79ee1f2020-08-27 17:41:56 -0700999 if (surface == nullptr) {
Wonsik Kim1eb88a92021-03-29 20:44:04 -07001000 const char *prefix = "";
1001 if (flexSemiPlanarPixelFormat) {
1002 format = COLOR_FormatYUV420SemiPlanar;
1003 prefix = "semi-";
1004 } else {
1005 format = COLOR_FormatYUV420Planar;
1006 }
1007 ALOGD("Client requested ByteBuffer mode decoder w/o color format set: "
1008 "using default %splanar color format", prefix);
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07001009 } else {
1010 format = COLOR_FormatSurface;
1011 }
1012 defaultColorFormat = format;
1013 }
1014 } else {
1015 if ((config->mDomain & Config::IS_ENCODER) || !surface) {
1016 switch (format) {
1017 case COLOR_FormatYUV420Flexible:
1018 format = flexPixelFormat.value_or(COLOR_FormatYUV420Planar);
1019 break;
1020 case COLOR_FormatYUV420Planar:
1021 case COLOR_FormatYUV420PackedPlanar:
1022 format = flexPlanarPixelFormat.value_or(
1023 flexPixelFormat.value_or(format));
1024 break;
1025 case COLOR_FormatYUV420SemiPlanar:
1026 case COLOR_FormatYUV420PackedSemiPlanar:
1027 format = flexSemiPlanarPixelFormat.value_or(
1028 flexPixelFormat.value_or(format));
1029 break;
1030 default:
1031 // No-op
1032 break;
1033 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001034 }
1035 }
1036
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07001037 if (format != 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001038 msg->setInt32("android._color-format", format);
1039 }
1040 }
1041
Wonsik Kim77e97c72021-01-20 10:33:22 -08001042 /*
1043 * Handle dataspace
1044 */
1045 int32_t usingRecorder;
1046 if (msg->findInt32("android._using-recorder", &usingRecorder) && usingRecorder) {
1047 android_dataspace dataSpace = HAL_DATASPACE_BT709;
1048 int32_t width, height;
1049 if (msg->findInt32("width", &width)
1050 && msg->findInt32("height", &height)) {
Wonsik Kim4f13d112021-03-17 04:37:46 +00001051 ColorAspects aspects;
1052 getColorAspectsFromFormat(msg, aspects);
1053 setDefaultCodecColorAspectsIfNeeded(aspects, width, height);
Wonsik Kim77e97c72021-01-20 10:33:22 -08001054 // TODO: read dataspace / color aspect from the component
Wonsik Kim4f13d112021-03-17 04:37:46 +00001055 setColorAspectsIntoFormat(aspects, const_cast<sp<AMessage> &>(msg));
1056 dataSpace = getDataSpaceForColorAspects(aspects, true /* mayexpand */);
Wonsik Kim77e97c72021-01-20 10:33:22 -08001057 }
1058 msg->setInt32("android._dataspace", (int32_t)dataSpace);
1059 ALOGD("setting dataspace to %x", dataSpace);
1060 }
1061
Wonsik Kim8a6ed372019-12-03 16:05:51 -08001062 int32_t subscribeToAllVendorParams;
1063 if (msg->findInt32("x-*", &subscribeToAllVendorParams) && subscribeToAllVendorParams) {
1064 if (config->subscribeToAllVendorParams(comp, C2_MAY_BLOCK) != OK) {
1065 ALOGD("[%s] Failed to subscribe to all vendor params", comp->getName().c_str());
1066 }
1067 }
1068
Pawin Vongmasa36653902018-11-15 00:10:25 -08001069 std::vector<std::unique_ptr<C2Param>> configUpdate;
Wonsik Kimaa484ac2019-02-13 16:54:02 -08001070 // NOTE: We used to ignore "video-bitrate" at configure; replicate
1071 // the behavior here.
1072 sp<AMessage> sdkParams = msg;
1073 int32_t videoBitrate;
1074 if (sdkParams->findInt32(PARAMETER_KEY_VIDEO_BITRATE, &videoBitrate)) {
1075 sdkParams = msg->dup();
1076 sdkParams->removeEntryAt(sdkParams->findEntryByName(PARAMETER_KEY_VIDEO_BITRATE));
1077 }
ted.sun765db4d2020-06-23 14:03:41 +08001078 err = config->getConfigUpdateFromSdkParams(
Wonsik Kimaa484ac2019-02-13 16:54:02 -08001079 comp, sdkParams, Config::IS_CONFIG, C2_DONT_BLOCK, &configUpdate);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001080 if (err != OK) {
1081 ALOGW("failed to convert configuration to c2 params");
1082 }
Wonsik Kimaab2eea2019-05-22 10:37:58 -07001083
1084 int32_t maxBframes = 0;
1085 if ((config->mDomain & Config::IS_ENCODER)
1086 && (config->mDomain & Config::IS_VIDEO)
1087 && sdkParams->findInt32(KEY_MAX_B_FRAMES, &maxBframes)
1088 && maxBframes > 0) {
1089 std::unique_ptr<C2StreamGopTuning::output> gop =
1090 C2StreamGopTuning::output::AllocUnique(2 /* flexCount */, 0u /* stream */);
1091 gop->m.values[0] = { P_FRAME, UINT32_MAX };
1092 gop->m.values[1] = {
1093 C2Config::picture_type_t(P_FRAME | B_FRAME),
1094 uint32_t(maxBframes)
1095 };
1096 configUpdate.push_back(std::move(gop));
1097 }
1098
Ray Essicka0ae6972021-03-10 19:40:01 -08001099 if ((config->mDomain & Config::IS_ENCODER)
1100 && (config->mDomain & Config::IS_VIDEO)) {
1101 // we may not use all 3 of these entries
1102 std::unique_ptr<C2StreamPictureQuantizationTuning::output> qp =
1103 C2StreamPictureQuantizationTuning::output::AllocUnique(3 /* flexCount */,
1104 0u /* stream */);
1105
1106 int ix = 0;
1107
1108 int32_t iMax = INT32_MAX;
1109 int32_t iMin = INT32_MIN;
1110 (void) sdkParams->findInt32(KEY_VIDEO_QP_I_MAX, &iMax);
1111 (void) sdkParams->findInt32(KEY_VIDEO_QP_I_MIN, &iMin);
1112 if (iMax != INT32_MAX || iMin != INT32_MIN) {
1113 qp->m.values[ix++] = {I_FRAME, iMin, iMax};
1114 }
1115
1116 int32_t pMax = INT32_MAX;
1117 int32_t pMin = INT32_MIN;
1118 (void) sdkParams->findInt32(KEY_VIDEO_QP_P_MAX, &pMax);
1119 (void) sdkParams->findInt32(KEY_VIDEO_QP_P_MIN, &pMin);
1120 if (pMax != INT32_MAX || pMin != INT32_MIN) {
1121 qp->m.values[ix++] = {P_FRAME, pMin, pMax};
1122 }
1123
1124 int32_t bMax = INT32_MAX;
1125 int32_t bMin = INT32_MIN;
1126 (void) sdkParams->findInt32(KEY_VIDEO_QP_B_MAX, &bMax);
1127 (void) sdkParams->findInt32(KEY_VIDEO_QP_B_MIN, &bMin);
1128 if (bMax != INT32_MAX || bMin != INT32_MIN) {
1129 qp->m.values[ix++] = {B_FRAME, bMin, bMax};
1130 }
1131
1132 // adjust to reflect actual use.
1133 qp->setFlexCount(ix);
1134
1135 configUpdate.push_back(std::move(qp));
1136 }
1137
Pawin Vongmasa36653902018-11-15 00:10:25 -08001138 err = config->setParameters(comp, configUpdate, C2_DONT_BLOCK);
1139 if (err != OK) {
1140 ALOGW("failed to configure c2 params");
1141 return err;
1142 }
1143
1144 std::vector<std::unique_ptr<C2Param>> params;
1145 C2StreamUsageTuning::input usage(0u, 0u);
1146 C2StreamMaxBufferSizeInfo::input maxInputSize(0u, 0u);
Wonsik Kim9ca01d32019-04-01 14:45:47 -07001147 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001148
Wonsik Kim3baecda2021-02-07 22:19:56 -08001149 C2Param::Index colorAspectsRequestIndex =
1150 C2StreamColorAspectsInfo::output::PARAM_TYPE | C2Param::CoreIndex::IS_REQUEST_FLAG;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001151 std::initializer_list<C2Param::Index> indices {
Wonsik Kim3baecda2021-02-07 22:19:56 -08001152 colorAspectsRequestIndex.withStream(0u),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001153 };
1154 c2_status_t c2err = comp->query(
Wonsik Kim9ca01d32019-04-01 14:45:47 -07001155 { &usage, &maxInputSize, &prepend },
Pawin Vongmasa36653902018-11-15 00:10:25 -08001156 indices,
1157 C2_DONT_BLOCK,
1158 &params);
1159 if (c2err != C2_OK && c2err != C2_BAD_INDEX) {
1160 ALOGE("Failed to query component interface: %d", c2err);
1161 return UNKNOWN_ERROR;
1162 }
Wonsik Kim9eac4d12019-05-23 12:58:48 -07001163 if (usage) {
1164 if (usage.value & C2MemoryUsage::CPU_READ) {
1165 config->mInputFormat->setInt32("using-sw-read-often", true);
1166 }
1167 if (config->mISConfig) {
1168 C2AndroidMemoryUsage androidUsage(C2MemoryUsage(usage.value));
1169 config->mISConfig->mUsage = androidUsage.asGrallocUsage();
1170 }
Wonsik Kim666604a2020-05-14 16:57:49 -07001171 config->mInputFormat->setInt64("android._C2MemoryUsage", usage.value);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001172 }
1173
1174 // NOTE: we don't blindly use client specified input size if specified as clients
1175 // at times specify too small size. Instead, mimic the behavior from OMX, where the
1176 // client specified size is only used to ask for bigger buffers than component suggested
1177 // size.
1178 int32_t clientInputSize = 0;
1179 bool clientSpecifiedInputSize =
1180 msg->findInt32(KEY_MAX_INPUT_SIZE, &clientInputSize) && clientInputSize > 0;
1181 // TEMP: enforce minimum buffer size of 1MB for video decoders
1182 // and 16K / 4K for audio encoders/decoders
1183 if (maxInputSize.value == 0) {
1184 if (config->mDomain & Config::IS_AUDIO) {
1185 maxInputSize.value = encoder ? 16384 : 4096;
1186 } else if (!encoder) {
1187 maxInputSize.value = 1048576u;
1188 }
1189 }
1190
1191 // verify that CSD fits into this size (if defined)
1192 if ((config->mDomain & Config::IS_DECODER) && maxInputSize.value > 0) {
1193 sp<ABuffer> csd;
1194 for (size_t ix = 0; msg->findBuffer(StringPrintf("csd-%zu", ix).c_str(), &csd); ++ix) {
1195 if (csd && csd->size() > maxInputSize.value) {
1196 maxInputSize.value = csd->size();
1197 }
1198 }
1199 }
1200
1201 // TODO: do this based on component requiring linear allocator for input
1202 if ((config->mDomain & Config::IS_DECODER) || (config->mDomain & Config::IS_AUDIO)) {
1203 if (clientSpecifiedInputSize) {
1204 // Warn that we're overriding client's max input size if necessary.
1205 if ((uint32_t)clientInputSize < maxInputSize.value) {
1206 ALOGD("client requested max input size %d, which is smaller than "
1207 "what component recommended (%u); overriding with component "
1208 "recommendation.", clientInputSize, maxInputSize.value);
1209 ALOGW("This behavior is subject to change. It is recommended that "
1210 "app developers double check whether the requested "
1211 "max input size is in reasonable range.");
1212 } else {
1213 maxInputSize.value = clientInputSize;
1214 }
1215 }
1216 // Pass max input size on input format to the buffer channel (if supplied by the
1217 // component or by a default)
1218 if (maxInputSize.value) {
1219 config->mInputFormat->setInt32(
1220 KEY_MAX_INPUT_SIZE,
1221 (int32_t)(c2_min(maxInputSize.value, uint32_t(INT32_MAX))));
1222 }
1223 }
1224
Wonsik Kim9ca01d32019-04-01 14:45:47 -07001225 int32_t clientPrepend;
1226 if ((config->mDomain & Config::IS_VIDEO)
1227 && (config->mDomain & Config::IS_ENCODER)
Lajos Molnara2b5f5a2020-10-14 16:36:18 -07001228 && msg->findInt32(KEY_PREPEND_HEADER_TO_SYNC_FRAMES, &clientPrepend)
Wonsik Kim9ca01d32019-04-01 14:45:47 -07001229 && clientPrepend
1230 && (!prepend || prepend.value != PREPEND_HEADER_TO_ALL_SYNC)) {
Lajos Molnara2b5f5a2020-10-14 16:36:18 -07001231 ALOGE("Failed to set KEY_PREPEND_HEADER_TO_SYNC_FRAMES");
Wonsik Kim9ca01d32019-04-01 14:45:47 -07001232 return BAD_VALUE;
1233 }
1234
Wonsik Kimd91f3fb2021-02-24 12:35:31 -08001235 int32_t componentColorFormat = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001236 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))) {
1237 // propagate HDR static info to output format for both encoders and decoders
1238 // if component supports this info, we will update from component, but only the raw port,
1239 // so don't propagate if component already filled it in.
1240 sp<ABuffer> hdrInfo;
1241 if (msg->findBuffer(KEY_HDR_STATIC_INFO, &hdrInfo)
1242 && !config->mOutputFormat->findBuffer(KEY_HDR_STATIC_INFO, &hdrInfo)) {
1243 config->mOutputFormat->setBuffer(KEY_HDR_STATIC_INFO, hdrInfo);
1244 }
1245
1246 // Set desired color format from configuration parameter
1247 int32_t format;
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07001248 if (!msg->findInt32(KEY_COLOR_FORMAT, &format)) {
1249 format = defaultColorFormat;
1250 }
1251 if (config->mDomain & Config::IS_ENCODER) {
1252 config->mInputFormat->setInt32(KEY_COLOR_FORMAT, format);
Wonsik Kimd91f3fb2021-02-24 12:35:31 -08001253 if (msg->findInt32("android._color-format", &componentColorFormat)) {
1254 config->mInputFormat->setInt32("android._color-format", componentColorFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001255 }
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07001256 } else {
1257 config->mOutputFormat->setInt32(KEY_COLOR_FORMAT, format);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001258 }
1259 }
1260
1261 // propagate encoder delay and padding to output format
1262 if ((config->mDomain & Config::IS_DECODER) && (config->mDomain & Config::IS_AUDIO)) {
1263 int delay = 0;
1264 if (msg->findInt32("encoder-delay", &delay)) {
1265 config->mOutputFormat->setInt32("encoder-delay", delay);
1266 }
1267 int padding = 0;
1268 if (msg->findInt32("encoder-padding", &padding)) {
1269 config->mOutputFormat->setInt32("encoder-padding", padding);
1270 }
1271 }
1272
1273 // set channel-mask
1274 if (config->mDomain & Config::IS_AUDIO) {
1275 int32_t mask;
1276 if (msg->findInt32(KEY_CHANNEL_MASK, &mask)) {
1277 if (config->mDomain & Config::IS_ENCODER) {
1278 config->mInputFormat->setInt32(KEY_CHANNEL_MASK, mask);
1279 } else {
1280 config->mOutputFormat->setInt32(KEY_CHANNEL_MASK, mask);
1281 }
1282 }
1283 }
1284
Wonsik Kim3baecda2021-02-07 22:19:56 -08001285 std::unique_ptr<C2Param> colorTransferRequestParam;
1286 for (std::unique_ptr<C2Param> &param : params) {
1287 if (param->index() == colorAspectsRequestIndex.withStream(0u)) {
1288 ALOGI("found color transfer request param");
1289 colorTransferRequestParam = std::move(param);
1290 }
1291 }
1292 int32_t colorTransferRequest = 0;
1293 if (config->mDomain & (Config::IS_IMAGE | Config::IS_VIDEO)
1294 && !sdkParams->findInt32("color-transfer-request", &colorTransferRequest)) {
1295 colorTransferRequest = 0;
1296 }
1297
1298 if (colorTransferRequest != 0) {
1299 if (colorTransferRequestParam && *colorTransferRequestParam) {
1300 C2StreamColorAspectsInfo::output *info =
1301 static_cast<C2StreamColorAspectsInfo::output *>(
1302 colorTransferRequestParam.get());
1303 if (!C2Mapper::map(info->transfer, &colorTransferRequest)) {
1304 colorTransferRequest = 0;
1305 }
1306 } else {
1307 colorTransferRequest = 0;
1308 }
1309 config->mInputFormat->setInt32("color-transfer-request", colorTransferRequest);
1310 }
1311
Wonsik Kimd91f3fb2021-02-24 12:35:31 -08001312 if (componentColorFormat != 0 && componentColorFormat != COLOR_FormatSurface) {
1313 // Need to get stride/vstride
1314 uint32_t pixelFormat = PIXEL_FORMAT_UNKNOWN;
1315 if (C2Mapper::mapPixelFormatFrameworkToCodec(componentColorFormat, &pixelFormat)) {
1316 // TODO: retrieve these values without allocating a buffer.
1317 // Currently allocating a buffer is necessary to retrieve the layout.
1318 int64_t blockUsage =
1319 usage.value | C2MemoryUsage::CPU_READ | C2MemoryUsage::CPU_WRITE;
1320 std::shared_ptr<C2GraphicBlock> block = FetchGraphicBlock(
1321 width, height, pixelFormat, blockUsage, {comp->getName()});
1322 sp<GraphicBlockBuffer> buffer;
1323 if (block) {
1324 buffer = GraphicBlockBuffer::Allocate(
1325 config->mInputFormat,
1326 block,
1327 [](size_t size) -> sp<ABuffer> { return new ABuffer(size); });
1328 } else {
1329 ALOGD("Failed to allocate a graphic block "
1330 "(width=%d height=%d pixelFormat=%u usage=%llx)",
1331 width, height, pixelFormat, (long long)blockUsage);
1332 // This means that byte buffer mode is not supported in this configuration
1333 // anyway. Skip setting stride/vstride to input format.
1334 }
1335 if (buffer) {
1336 sp<ABuffer> imageData = buffer->getImageData();
1337 MediaImage2 *img = nullptr;
1338 if (imageData && imageData->data()
1339 && imageData->size() >= sizeof(MediaImage2)) {
1340 img = (MediaImage2*)imageData->data();
1341 }
1342 if (img && img->mNumPlanes > 0 && img->mType != img->MEDIA_IMAGE_TYPE_UNKNOWN) {
1343 int32_t stride = img->mPlane[0].mRowInc;
1344 config->mInputFormat->setInt32(KEY_STRIDE, stride);
1345 if (img->mNumPlanes > 1 && stride > 0) {
1346 int64_t offsetDelta =
1347 (int64_t)img->mPlane[1].mOffset - (int64_t)img->mPlane[0].mOffset;
1348 if (offsetDelta % stride == 0) {
1349 int32_t vstride = int32_t(offsetDelta / stride);
1350 config->mInputFormat->setInt32(KEY_SLICE_HEIGHT, vstride);
1351 } else {
1352 ALOGD("Cannot report accurate slice height: "
1353 "offsetDelta = %lld stride = %d",
1354 (long long)offsetDelta, stride);
1355 }
1356 }
1357 }
1358 }
1359 }
1360 }
1361
1362 ALOGD("setup formats input: %s",
1363 config->mInputFormat->debugString().c_str());
1364 ALOGD("setup formats output: %s",
Pawin Vongmasa36653902018-11-15 00:10:25 -08001365 config->mOutputFormat->debugString().c_str());
1366 return OK;
1367 };
1368 if (tryAndReportOnError(doConfig) != OK) {
1369 return;
1370 }
1371
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001372 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1373 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001374
Houxiang Dai183ca3d2021-02-04 14:20:11 +08001375 config->queryConfiguration(comp);
1376
Pawin Vongmasa36653902018-11-15 00:10:25 -08001377 mCallback->onComponentConfigured(config->mInputFormat, config->mOutputFormat);
1378}
1379
1380void CCodec::initiateCreateInputSurface() {
1381 status_t err = [this] {
1382 Mutexed<State>::Locked state(mState);
1383 if (state->get() != ALLOCATED) {
1384 return UNKNOWN_ERROR;
1385 }
1386 // TODO: read it from intf() properly.
1387 if (state->comp->getName().find("encoder") == std::string::npos) {
1388 return INVALID_OPERATION;
1389 }
1390 return OK;
1391 }();
1392 if (err != OK) {
1393 mCallback->onInputSurfaceCreationFailed(err);
1394 return;
1395 }
1396
1397 (new AMessage(kWhatCreateInputSurface, this))->post();
1398}
1399
Lajos Molnar47118272019-01-31 16:28:04 -08001400sp<PersistentSurface> CCodec::CreateOmxInputSurface() {
1401 using namespace android::hardware::media::omx::V1_0;
1402 using namespace android::hardware::media::omx::V1_0::utils;
1403 using namespace android::hardware::graphics::bufferqueue::V1_0::utils;
1404 typedef android::hardware::media::omx::V1_0::Status OmxStatus;
1405 android::sp<IOmx> omx = IOmx::getService();
1406 typedef android::hardware::graphics::bufferqueue::V1_0::
1407 IGraphicBufferProducer HGraphicBufferProducer;
1408 typedef android::hardware::media::omx::V1_0::
1409 IGraphicBufferSource HGraphicBufferSource;
1410 OmxStatus s;
1411 android::sp<HGraphicBufferProducer> gbp;
1412 android::sp<HGraphicBufferSource> gbs;
Pawin Vongmasa18588322019-05-18 01:52:13 -07001413
Chong Zhangc8ce1d82019-03-27 10:18:38 -07001414 using ::android::hardware::Return;
1415 Return<void> transStatus = omx->createInputSurface(
Lajos Molnar47118272019-01-31 16:28:04 -08001416 [&s, &gbp, &gbs](
1417 OmxStatus status,
1418 const android::sp<HGraphicBufferProducer>& producer,
1419 const android::sp<HGraphicBufferSource>& source) {
1420 s = status;
1421 gbp = producer;
1422 gbs = source;
1423 });
1424 if (transStatus.isOk() && s == OmxStatus::OK) {
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001425 return new PersistentSurface(new H2BGraphicBufferProducer(gbp), gbs);
Lajos Molnar47118272019-01-31 16:28:04 -08001426 }
1427
1428 return nullptr;
1429}
1430
1431sp<PersistentSurface> CCodec::CreateCompatibleInputSurface() {
1432 sp<PersistentSurface> surface(CreateInputSurface());
1433
1434 if (surface == nullptr) {
1435 surface = CreateOmxInputSurface();
1436 }
1437
1438 return surface;
1439}
1440
Pawin Vongmasa36653902018-11-15 00:10:25 -08001441void CCodec::createInputSurface() {
1442 status_t err;
1443 sp<IGraphicBufferProducer> bufferProducer;
1444
1445 sp<AMessage> inputFormat;
1446 sp<AMessage> outputFormat;
Wonsik Kim9eac4d12019-05-23 12:58:48 -07001447 uint64_t usage = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001448 {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001449 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1450 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001451 inputFormat = config->mInputFormat;
1452 outputFormat = config->mOutputFormat;
Wonsik Kim9eac4d12019-05-23 12:58:48 -07001453 usage = config->mISConfig ? config->mISConfig->mUsage : 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001454 }
1455
Lajos Molnar47118272019-01-31 16:28:04 -08001456 sp<PersistentSurface> persistentSurface = CreateCompatibleInputSurface();
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001457 sp<hidl::base::V1_0::IBase> hidlTarget = persistentSurface->getHidlTarget();
1458 sp<IInputSurface> hidlInputSurface = IInputSurface::castFrom(hidlTarget);
1459 sp<HGraphicBufferSource> gbs = HGraphicBufferSource::castFrom(hidlTarget);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001460
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001461 if (hidlInputSurface) {
Pawin Vongmasa1c75a232019-01-09 04:41:52 -08001462 std::shared_ptr<Codec2Client::InputSurface> inputSurface =
1463 std::make_shared<Codec2Client::InputSurface>(hidlInputSurface);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001464 err = setupInputSurface(std::make_shared<C2InputSurfaceWrapper>(
Pawin Vongmasa1c75a232019-01-09 04:41:52 -08001465 inputSurface));
1466 bufferProducer = inputSurface->getGraphicBufferProducer();
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001467 } else if (gbs) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001468 int32_t width = 0;
1469 (void)outputFormat->findInt32("width", &width);
1470 int32_t height = 0;
1471 (void)outputFormat->findInt32("height", &height);
1472 err = setupInputSurface(std::make_shared<GraphicBufferSourceWrapper>(
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001473 gbs, width, height, usage));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001474 bufferProducer = persistentSurface->getBufferProducer();
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001475 } else {
1476 ALOGE("Corrupted input surface");
1477 mCallback->onInputSurfaceCreationFailed(UNKNOWN_ERROR);
1478 return;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001479 }
1480
1481 if (err != OK) {
1482 ALOGE("Failed to set up input surface: %d", err);
1483 mCallback->onInputSurfaceCreationFailed(err);
1484 return;
1485 }
1486
1487 mCallback->onInputSurfaceCreated(
1488 inputFormat,
1489 outputFormat,
1490 new BufferProducerWrapper(bufferProducer));
1491}
1492
1493status_t CCodec::setupInputSurface(const std::shared_ptr<InputSurfaceWrapper> &surface) {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001494 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1495 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001496 config->mUsingSurface = true;
1497
1498 // we are now using surface - apply default color aspects to input format - as well as
1499 // get dataspace
Wonsik Kim8a6ed372019-12-03 16:05:51 -08001500 bool inputFormatChanged = config->updateFormats(Config::IS_INPUT);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001501 ALOGD("input format %s to %s",
1502 inputFormatChanged ? "changed" : "unchanged",
1503 config->mInputFormat->debugString().c_str());
1504
1505 // configure dataspace
1506 static_assert(sizeof(int32_t) == sizeof(android_dataspace), "dataspace size mismatch");
1507 android_dataspace dataSpace = HAL_DATASPACE_UNKNOWN;
1508 (void)config->mInputFormat->findInt32("android._dataspace", (int32_t*)&dataSpace);
1509 surface->setDataSpace(dataSpace);
1510
1511 status_t err = mChannel->setInputSurface(surface);
1512 if (err != OK) {
1513 // undo input format update
1514 config->mUsingSurface = false;
Wonsik Kim8a6ed372019-12-03 16:05:51 -08001515 (void)config->updateFormats(Config::IS_INPUT);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001516 return err;
1517 }
1518 config->mInputSurface = surface;
1519
1520 if (config->mISConfig) {
1521 surface->configure(*config->mISConfig);
1522 } else {
1523 ALOGD("ISConfig: no configuration");
1524 }
1525
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001526 return OK;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001527}
1528
1529void CCodec::initiateSetInputSurface(const sp<PersistentSurface> &surface) {
1530 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
1531 msg->setObject("surface", surface);
1532 msg->post();
1533}
1534
1535void CCodec::setInputSurface(const sp<PersistentSurface> &surface) {
1536 sp<AMessage> inputFormat;
1537 sp<AMessage> outputFormat;
Wonsik Kim9eac4d12019-05-23 12:58:48 -07001538 uint64_t usage = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001539 {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001540 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1541 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001542 inputFormat = config->mInputFormat;
1543 outputFormat = config->mOutputFormat;
Wonsik Kim9eac4d12019-05-23 12:58:48 -07001544 usage = config->mISConfig ? config->mISConfig->mUsage : 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001545 }
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001546 sp<hidl::base::V1_0::IBase> hidlTarget = surface->getHidlTarget();
1547 sp<IInputSurface> inputSurface = IInputSurface::castFrom(hidlTarget);
1548 sp<HGraphicBufferSource> gbs = HGraphicBufferSource::castFrom(hidlTarget);
1549 if (inputSurface) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001550 status_t err = setupInputSurface(std::make_shared<C2InputSurfaceWrapper>(
1551 std::make_shared<Codec2Client::InputSurface>(inputSurface)));
1552 if (err != OK) {
1553 ALOGE("Failed to set up input surface: %d", err);
1554 mCallback->onInputSurfaceDeclined(err);
1555 return;
1556 }
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001557 } else if (gbs) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001558 int32_t width = 0;
1559 (void)outputFormat->findInt32("width", &width);
1560 int32_t height = 0;
1561 (void)outputFormat->findInt32("height", &height);
1562 status_t err = setupInputSurface(std::make_shared<GraphicBufferSourceWrapper>(
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001563 gbs, width, height, usage));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001564 if (err != OK) {
1565 ALOGE("Failed to set up input surface: %d", err);
1566 mCallback->onInputSurfaceDeclined(err);
1567 return;
1568 }
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001569 } else {
1570 ALOGE("Failed to set input surface: Corrupted surface.");
1571 mCallback->onInputSurfaceDeclined(UNKNOWN_ERROR);
1572 return;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001573 }
1574 mCallback->onInputSurfaceAccepted(inputFormat, outputFormat);
1575}
1576
1577void CCodec::initiateStart() {
1578 auto setStarting = [this] {
1579 Mutexed<State>::Locked state(mState);
1580 if (state->get() != ALLOCATED) {
1581 return UNKNOWN_ERROR;
1582 }
1583 state->set(STARTING);
1584 return OK;
1585 };
1586 if (tryAndReportOnError(setStarting) != OK) {
1587 return;
1588 }
1589
1590 (new AMessage(kWhatStart, this))->post();
1591}
1592
1593void CCodec::start() {
1594 std::shared_ptr<Codec2Client::Component> comp;
1595 auto checkStarting = [this, &comp] {
1596 Mutexed<State>::Locked state(mState);
1597 if (state->get() != STARTING) {
1598 return UNKNOWN_ERROR;
1599 }
1600 comp = state->comp;
1601 return OK;
1602 };
1603 if (tryAndReportOnError(checkStarting) != OK) {
1604 return;
1605 }
1606
1607 c2_status_t err = comp->start();
1608 if (err != C2_OK) {
1609 mCallback->onError(toStatusT(err, C2_OPERATION_Component_start),
1610 ACTION_CODE_FATAL);
1611 return;
1612 }
1613 sp<AMessage> inputFormat;
1614 sp<AMessage> outputFormat;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001615 status_t err2 = OK;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001616 bool buffersBoundToCodec = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001617 {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001618 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1619 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001620 inputFormat = config->mInputFormat;
Wonsik Kim274c8322020-02-28 10:42:21 -08001621 // start triggers format dup
1622 outputFormat = config->mOutputFormat = config->mOutputFormat->dup();
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001623 if (config->mInputSurface) {
1624 err2 = config->mInputSurface->start();
Wonsik Kim673dd192021-01-29 14:58:12 -08001625 config->mInputSurfaceDataspace = config->mInputSurface->getDataspace();
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001626 }
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001627 buffersBoundToCodec = config->mBuffersBoundToCodec;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001628 }
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001629 if (err2 != OK) {
1630 mCallback->onError(err2, ACTION_CODE_FATAL);
1631 return;
1632 }
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001633 err2 = mChannel->start(inputFormat, outputFormat, buffersBoundToCodec);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001634 if (err2 != OK) {
1635 mCallback->onError(err2, ACTION_CODE_FATAL);
1636 return;
1637 }
1638
1639 auto setRunning = [this] {
1640 Mutexed<State>::Locked state(mState);
1641 if (state->get() != STARTING) {
1642 return UNKNOWN_ERROR;
1643 }
1644 state->set(RUNNING);
1645 return OK;
1646 };
1647 if (tryAndReportOnError(setRunning) != OK) {
1648 return;
1649 }
1650 mCallback->onStartCompleted();
1651
1652 (void)mChannel->requestInitialInputBuffers();
1653}
1654
1655void CCodec::initiateShutdown(bool keepComponentAllocated) {
1656 if (keepComponentAllocated) {
1657 initiateStop();
1658 } else {
1659 initiateRelease();
1660 }
1661}
1662
1663void CCodec::initiateStop() {
1664 {
1665 Mutexed<State>::Locked state(mState);
1666 if (state->get() == ALLOCATED
1667 || state->get() == RELEASED
1668 || state->get() == STOPPING
1669 || state->get() == RELEASING) {
1670 // We're already stopped, released, or doing it right now.
1671 state.unlock();
1672 mCallback->onStopCompleted();
1673 state.lock();
1674 return;
1675 }
1676 state->set(STOPPING);
1677 }
1678
Wonsik Kim936a89c2020-05-08 16:07:50 -07001679 mChannel->reset();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001680 (new AMessage(kWhatStop, this))->post();
1681}
1682
1683void CCodec::stop() {
1684 std::shared_ptr<Codec2Client::Component> comp;
1685 {
1686 Mutexed<State>::Locked state(mState);
1687 if (state->get() == RELEASING) {
1688 state.unlock();
1689 // We're already stopped or release is in progress.
1690 mCallback->onStopCompleted();
1691 state.lock();
1692 return;
1693 } else if (state->get() != STOPPING) {
1694 state.unlock();
1695 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1696 state.lock();
1697 return;
1698 }
1699 comp = state->comp;
1700 }
1701 status_t err = comp->stop();
1702 if (err != C2_OK) {
1703 // TODO: convert err into status_t
1704 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1705 }
1706
1707 {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001708 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1709 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001710 if (config->mInputSurface) {
1711 config->mInputSurface->disconnect();
1712 config->mInputSurface = nullptr;
Wonsik Kim673dd192021-01-29 14:58:12 -08001713 config->mInputSurfaceDataspace = HAL_DATASPACE_UNKNOWN;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001714 }
1715 }
1716 {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001717 Mutexed<State>::Locked state(mState);
1718 if (state->get() == STOPPING) {
1719 state->set(ALLOCATED);
1720 }
1721 }
1722 mCallback->onStopCompleted();
1723}
1724
1725void CCodec::initiateRelease(bool sendCallback /* = true */) {
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001726 bool clearInputSurfaceIfNeeded = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001727 {
1728 Mutexed<State>::Locked state(mState);
1729 if (state->get() == RELEASED || state->get() == RELEASING) {
1730 // We're already released or doing it right now.
1731 if (sendCallback) {
1732 state.unlock();
1733 mCallback->onReleaseCompleted();
1734 state.lock();
1735 }
1736 return;
1737 }
1738 if (state->get() == ALLOCATING) {
1739 state->set(RELEASING);
1740 // With the altered state allocate() would fail and clean up.
1741 if (sendCallback) {
1742 state.unlock();
1743 mCallback->onReleaseCompleted();
1744 state.lock();
1745 }
1746 return;
1747 }
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001748 if (state->get() == STARTING
1749 || state->get() == RUNNING
1750 || state->get() == STOPPING) {
1751 // Input surface may have been started, so clean up is needed.
1752 clearInputSurfaceIfNeeded = true;
1753 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001754 state->set(RELEASING);
1755 }
1756
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001757 if (clearInputSurfaceIfNeeded) {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001758 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1759 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001760 if (config->mInputSurface) {
1761 config->mInputSurface->disconnect();
1762 config->mInputSurface = nullptr;
Wonsik Kim673dd192021-01-29 14:58:12 -08001763 config->mInputSurfaceDataspace = HAL_DATASPACE_UNKNOWN;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001764 }
1765 }
1766
Wonsik Kim936a89c2020-05-08 16:07:50 -07001767 mChannel->reset();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001768 // thiz holds strong ref to this while the thread is running.
1769 sp<CCodec> thiz(this);
1770 std::thread([thiz, sendCallback] { thiz->release(sendCallback); }).detach();
1771}
1772
1773void CCodec::release(bool sendCallback) {
1774 std::shared_ptr<Codec2Client::Component> comp;
1775 {
1776 Mutexed<State>::Locked state(mState);
1777 if (state->get() == RELEASED) {
1778 if (sendCallback) {
1779 state.unlock();
1780 mCallback->onReleaseCompleted();
1781 state.lock();
1782 }
1783 return;
1784 }
1785 comp = state->comp;
1786 }
1787 comp->release();
1788
1789 {
1790 Mutexed<State>::Locked state(mState);
1791 state->set(RELEASED);
1792 state->comp.reset();
1793 }
Wonsik Kim936a89c2020-05-08 16:07:50 -07001794 (new AMessage(kWhatRelease, this))->post();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001795 if (sendCallback) {
1796 mCallback->onReleaseCompleted();
1797 }
1798}
1799
1800status_t CCodec::setSurface(const sp<Surface> &surface) {
Wonsik Kim75e22f42021-04-14 23:34:51 -07001801 {
1802 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1803 const std::unique_ptr<Config> &config = *configLocked;
1804 if (config->mTunneled && config->mSidebandHandle != nullptr) {
1805 sp<ANativeWindow> nativeWindow = static_cast<ANativeWindow *>(surface.get());
1806 status_t err = native_window_set_sideband_stream(
1807 nativeWindow.get(),
1808 const_cast<native_handle_t *>(config->mSidebandHandle->handle()));
1809 if (err != OK) {
1810 ALOGE("NativeWindow(%p) native_window_set_sideband_stream(%p) failed! (err %d).",
1811 nativeWindow.get(), config->mSidebandHandle->handle(), err);
1812 return err;
1813 }
ted.sun765db4d2020-06-23 14:03:41 +08001814 }
1815 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001816 return mChannel->setSurface(surface);
1817}
1818
1819void CCodec::signalFlush() {
1820 status_t err = [this] {
1821 Mutexed<State>::Locked state(mState);
1822 if (state->get() == FLUSHED) {
1823 return ALREADY_EXISTS;
1824 }
1825 if (state->get() != RUNNING) {
1826 return UNKNOWN_ERROR;
1827 }
1828 state->set(FLUSHING);
1829 return OK;
1830 }();
1831 switch (err) {
1832 case ALREADY_EXISTS:
1833 mCallback->onFlushCompleted();
1834 return;
1835 case OK:
1836 break;
1837 default:
1838 mCallback->onError(err, ACTION_CODE_FATAL);
1839 return;
1840 }
1841
1842 mChannel->stop();
1843 (new AMessage(kWhatFlush, this))->post();
1844}
1845
1846void CCodec::flush() {
1847 std::shared_ptr<Codec2Client::Component> comp;
1848 auto checkFlushing = [this, &comp] {
1849 Mutexed<State>::Locked state(mState);
1850 if (state->get() != FLUSHING) {
1851 return UNKNOWN_ERROR;
1852 }
1853 comp = state->comp;
1854 return OK;
1855 };
1856 if (tryAndReportOnError(checkFlushing) != OK) {
1857 return;
1858 }
1859
1860 std::list<std::unique_ptr<C2Work>> flushedWork;
1861 c2_status_t err = comp->flush(C2Component::FLUSH_COMPONENT, &flushedWork);
1862 {
1863 Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
1864 flushedWork.splice(flushedWork.end(), *queue);
1865 }
1866 if (err != C2_OK) {
1867 // TODO: convert err into status_t
1868 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1869 }
1870
1871 mChannel->flush(flushedWork);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001872
1873 {
1874 Mutexed<State>::Locked state(mState);
Iris Changce521ee2019-07-05 16:18:54 +08001875 if (state->get() == FLUSHING) {
1876 state->set(FLUSHED);
1877 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001878 }
1879 mCallback->onFlushCompleted();
1880}
1881
1882void CCodec::signalResume() {
Wonsik Kime75a5da2020-02-14 17:29:03 -08001883 std::shared_ptr<Codec2Client::Component> comp;
1884 auto setResuming = [this, &comp] {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001885 Mutexed<State>::Locked state(mState);
1886 if (state->get() != FLUSHED) {
1887 return UNKNOWN_ERROR;
1888 }
1889 state->set(RESUMING);
Wonsik Kime75a5da2020-02-14 17:29:03 -08001890 comp = state->comp;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001891 return OK;
1892 };
1893 if (tryAndReportOnError(setResuming) != OK) {
1894 return;
1895 }
1896
Wonsik Kime75a5da2020-02-14 17:29:03 -08001897 {
1898 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1899 const std::unique_ptr<Config> &config = *configLocked;
Harish Mahendrakar8c537502021-02-23 21:20:22 -08001900 sp<AMessage> outputFormat = config->mOutputFormat;
Wonsik Kime75a5da2020-02-14 17:29:03 -08001901 config->queryConfiguration(comp);
Harish Mahendrakar8c537502021-02-23 21:20:22 -08001902 RevertOutputFormatIfNeeded(outputFormat, config->mOutputFormat);
Wonsik Kime75a5da2020-02-14 17:29:03 -08001903 }
1904
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001905 (void)mChannel->start(nullptr, nullptr, [&]{
1906 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1907 const std::unique_ptr<Config> &config = *configLocked;
1908 return config->mBuffersBoundToCodec;
1909 }());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001910
1911 {
1912 Mutexed<State>::Locked state(mState);
1913 if (state->get() != RESUMING) {
1914 state.unlock();
1915 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1916 state.lock();
1917 return;
1918 }
1919 state->set(RUNNING);
1920 }
1921
1922 (void)mChannel->requestInitialInputBuffers();
1923}
1924
Wonsik Kimaa484ac2019-02-13 16:54:02 -08001925void CCodec::signalSetParameters(const sp<AMessage> &msg) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001926 std::shared_ptr<Codec2Client::Component> comp;
1927 auto checkState = [this, &comp] {
1928 Mutexed<State>::Locked state(mState);
1929 if (state->get() == RELEASED) {
1930 return INVALID_OPERATION;
1931 }
1932 comp = state->comp;
1933 return OK;
1934 };
1935 if (tryAndReportOnError(checkState) != OK) {
1936 return;
1937 }
1938
Wonsik Kimaa484ac2019-02-13 16:54:02 -08001939 // NOTE: We used to ignore "bitrate" at setParameters; replicate
1940 // the behavior here.
1941 sp<AMessage> params = msg;
1942 int32_t bitrate;
1943 if (params->findInt32(KEY_BIT_RATE, &bitrate)) {
1944 params = msg->dup();
1945 params->removeEntryAt(params->findEntryByName(KEY_BIT_RATE));
1946 }
1947
Houxiang Dai5a97b472021-03-22 17:56:04 +08001948 int32_t syncId = 0;
1949 if (params->findInt32("audio-hw-sync", &syncId)
1950 || params->findInt32("hw-av-sync-id", &syncId)) {
1951 configureTunneledVideoPlayback(comp, nullptr, params);
1952 }
1953
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001954 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1955 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001956
1957 /**
1958 * Handle input surface parameters
1959 */
1960 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))
Wonsik Kim8a6ed372019-12-03 16:05:51 -08001961 && (config->mDomain & Config::IS_ENCODER)
1962 && config->mInputSurface && config->mISConfig) {
Chong Zhang038e8f82019-02-06 19:05:14 -08001963 (void)params->findInt64(PARAMETER_KEY_OFFSET_TIME, &config->mISConfig->mTimeOffsetUs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001964
1965 if (params->findInt64("skip-frames-before", &config->mISConfig->mStartAtUs)) {
1966 config->mISConfig->mStopped = false;
1967 } else if (params->findInt64("stop-time-us", &config->mISConfig->mStopAtUs)) {
1968 config->mISConfig->mStopped = true;
1969 }
1970
1971 int32_t value;
Chong Zhang038e8f82019-02-06 19:05:14 -08001972 if (params->findInt32(PARAMETER_KEY_SUSPEND, &value)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001973 config->mISConfig->mSuspended = value;
1974 config->mISConfig->mSuspendAtUs = -1;
Chong Zhang038e8f82019-02-06 19:05:14 -08001975 (void)params->findInt64(PARAMETER_KEY_SUSPEND_TIME, &config->mISConfig->mSuspendAtUs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001976 }
1977
1978 (void)config->mInputSurface->configure(*config->mISConfig);
1979 if (config->mISConfig->mStopped) {
1980 config->mInputFormat->setInt64(
1981 "android._stop-time-offset-us", config->mISConfig->mInputDelayUs);
1982 }
1983 }
1984
1985 std::vector<std::unique_ptr<C2Param>> configUpdate;
1986 (void)config->getConfigUpdateFromSdkParams(
1987 comp, params, Config::IS_PARAM, C2_MAY_BLOCK, &configUpdate);
1988 // Prefer to pass parameters to the buffer channel, so they can be synchronized with the frames.
1989 // Parameter synchronization is not defined when using input surface. For now, route
1990 // these directly to the component.
1991 if (config->mInputSurface == nullptr
1992 && (property_get_bool("debug.stagefright.ccodec_delayed_params", false)
1993 || comp->getName().find("c2.android.") == 0)) {
1994 mChannel->setParameters(configUpdate);
1995 } else {
Wonsik Kim3b4349a2020-11-10 11:54:15 -08001996 sp<AMessage> outputFormat = config->mOutputFormat;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001997 (void)config->setParameters(comp, configUpdate, C2_MAY_BLOCK);
Wonsik Kim3b4349a2020-11-10 11:54:15 -08001998 RevertOutputFormatIfNeeded(outputFormat, config->mOutputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001999 }
2000}
2001
2002void CCodec::signalEndOfInputStream() {
2003 mCallback->onSignaledInputEOS(mChannel->signalEndOfInputStream());
2004}
2005
2006void CCodec::signalRequestIDRFrame() {
2007 std::shared_ptr<Codec2Client::Component> comp;
2008 {
2009 Mutexed<State>::Locked state(mState);
2010 if (state->get() == RELEASED) {
2011 ALOGD("no IDR request sent since component is released");
2012 return;
2013 }
2014 comp = state->comp;
2015 }
2016 ALOGV("request IDR");
Wonsik Kim155d5cb2019-10-09 12:49:49 -07002017 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2018 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002019 std::vector<std::unique_ptr<C2Param>> params;
2020 params.push_back(
2021 std::make_unique<C2StreamRequestSyncFrameTuning::output>(0u, true));
2022 config->setParameters(comp, params, C2_MAY_BLOCK);
2023}
2024
Wonsik Kim874ad382021-03-12 09:59:36 -08002025status_t CCodec::querySupportedParameters(std::vector<std::string> *names) {
2026 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2027 const std::unique_ptr<Config> &config = *configLocked;
2028 return config->querySupportedParameters(names);
2029}
2030
2031status_t CCodec::describeParameter(
2032 const std::string &name, CodecParameterDescriptor *desc) {
2033 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2034 const std::unique_ptr<Config> &config = *configLocked;
2035 return config->describe(name, desc);
2036}
2037
2038status_t CCodec::subscribeToParameters(const std::vector<std::string> &names) {
2039 std::shared_ptr<Codec2Client::Component> comp = mState.lock()->comp;
2040 if (!comp) {
2041 return INVALID_OPERATION;
2042 }
2043 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2044 const std::unique_ptr<Config> &config = *configLocked;
2045 return config->subscribeToVendorConfigUpdate(comp, names);
2046}
2047
2048status_t CCodec::unsubscribeFromParameters(const std::vector<std::string> &names) {
2049 std::shared_ptr<Codec2Client::Component> comp = mState.lock()->comp;
2050 if (!comp) {
2051 return INVALID_OPERATION;
2052 }
2053 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2054 const std::unique_ptr<Config> &config = *configLocked;
2055 return config->unsubscribeFromVendorConfigUpdate(comp, names);
2056}
2057
Wonsik Kimab34ed62019-01-31 15:28:46 -08002058void CCodec::onWorkDone(std::list<std::unique_ptr<C2Work>> &workItems) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002059 if (!workItems.empty()) {
Wonsik Kimab34ed62019-01-31 15:28:46 -08002060 Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
2061 queue->splice(queue->end(), workItems);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002062 }
2063 (new AMessage(kWhatWorkDone, this))->post();
2064}
2065
Wonsik Kimab34ed62019-01-31 15:28:46 -08002066void CCodec::onInputBufferDone(uint64_t frameIndex, size_t arrayIndex) {
2067 mChannel->onInputBufferDone(frameIndex, arrayIndex);
Wonsik Kim4f3314d2019-03-26 17:00:34 -07002068 if (arrayIndex == 0) {
2069 // We always put no more than one buffer per work, if we use an input surface.
Wonsik Kim155d5cb2019-10-09 12:49:49 -07002070 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2071 const std::unique_ptr<Config> &config = *configLocked;
Wonsik Kim4f3314d2019-03-26 17:00:34 -07002072 if (config->mInputSurface) {
2073 config->mInputSurface->onInputBufferDone(frameIndex);
2074 }
2075 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002076}
2077
2078void CCodec::onMessageReceived(const sp<AMessage> &msg) {
2079 TimePoint now = std::chrono::steady_clock::now();
2080 CCodecWatchdog::getInstance()->watch(this);
2081 switch (msg->what()) {
2082 case kWhatAllocate: {
2083 // C2ComponentStore::createComponent() should return within 100ms.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002084 setDeadline(now, 1500ms, "allocate");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002085 sp<RefBase> obj;
2086 CHECK(msg->findObject("codecInfo", &obj));
2087 allocate((MediaCodecInfo *)obj.get());
2088 break;
2089 }
2090 case kWhatConfigure: {
2091 // C2Component::commit_sm() should return within 5ms.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002092 setDeadline(now, 1500ms, "configure");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002093 sp<AMessage> format;
2094 CHECK(msg->findMessage("format", &format));
2095 configure(format);
2096 break;
2097 }
2098 case kWhatStart: {
2099 // C2Component::start() should return within 500ms.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002100 setDeadline(now, 1500ms, "start");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002101 start();
2102 break;
2103 }
2104 case kWhatStop: {
2105 // C2Component::stop() should return within 500ms.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002106 setDeadline(now, 1500ms, "stop");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002107 stop();
Pawin Vongmasa36653902018-11-15 00:10:25 -08002108 break;
2109 }
2110 case kWhatFlush: {
2111 // C2Component::flush_sm() should return within 5ms.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002112 setDeadline(now, 1500ms, "flush");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002113 flush();
2114 break;
2115 }
Wonsik Kim936a89c2020-05-08 16:07:50 -07002116 case kWhatRelease: {
2117 mChannel->release();
2118 mClient.reset();
2119 mClientListener.reset();
2120 break;
2121 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002122 case kWhatCreateInputSurface: {
2123 // Surface operations may be briefly blocking.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002124 setDeadline(now, 1500ms, "createInputSurface");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002125 createInputSurface();
2126 break;
2127 }
2128 case kWhatSetInputSurface: {
2129 // Surface operations may be briefly blocking.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002130 setDeadline(now, 1500ms, "setInputSurface");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002131 sp<RefBase> obj;
2132 CHECK(msg->findObject("surface", &obj));
2133 sp<PersistentSurface> surface(static_cast<PersistentSurface *>(obj.get()));
2134 setInputSurface(surface);
2135 break;
2136 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002137 case kWhatWorkDone: {
2138 std::unique_ptr<C2Work> work;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002139 bool shouldPost = false;
2140 {
2141 Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
2142 if (queue->empty()) {
2143 break;
2144 }
2145 work.swap(queue->front());
2146 queue->pop_front();
2147 shouldPost = !queue->empty();
2148 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002149 if (shouldPost) {
2150 (new AMessage(kWhatWorkDone, this))->post();
2151 }
2152
Pawin Vongmasa36653902018-11-15 00:10:25 -08002153 // handle configuration changes in work done
Wonsik Kim75e22f42021-04-14 23:34:51 -07002154 std::unique_ptr<C2Param> initData;
2155 sp<AMessage> outputFormat = nullptr;
2156 {
2157 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2158 const std::unique_ptr<Config> &config = *configLocked;
2159 Config::Watcher<C2StreamInitDataInfo::output> initDataWatcher =
2160 config->watch<C2StreamInitDataInfo::output>();
2161 if (!work->worklets.empty()
2162 && (work->worklets.front()->output.flags
2163 & C2FrameData::FLAG_DISCARD_FRAME) == 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002164
Wonsik Kim75e22f42021-04-14 23:34:51 -07002165 // copy buffer info to config
2166 std::vector<std::unique_ptr<C2Param>> updates;
2167 for (const std::unique_ptr<C2Param> &param
2168 : work->worklets.front()->output.configUpdate) {
2169 updates.push_back(C2Param::Copy(*param));
2170 }
2171 unsigned stream = 0;
2172 std::vector<std::shared_ptr<C2Buffer>> &outputBuffers =
2173 work->worklets.front()->output.buffers;
2174 for (const std::shared_ptr<C2Buffer> &buf : outputBuffers) {
2175 for (const std::shared_ptr<const C2Info> &info : buf->info()) {
2176 // move all info into output-stream #0 domain
2177 updates.emplace_back(
2178 C2Param::CopyAsStream(*info, true /* output */, stream));
2179 }
2180
2181 const std::vector<C2ConstGraphicBlock> blocks = buf->data().graphicBlocks();
2182 // for now only do the first block
2183 if (!blocks.empty()) {
2184 // ALOGV("got output buffer with crop %u,%u+%u,%u and size %u,%u",
2185 // block.crop().left, block.crop().top,
2186 // block.crop().width, block.crop().height,
2187 // block.width(), block.height());
2188 const C2ConstGraphicBlock &block = blocks[0];
2189 updates.emplace_back(new C2StreamCropRectInfo::output(
2190 stream, block.crop()));
2191 updates.emplace_back(new C2StreamPictureSizeInfo::output(
2192 stream, block.crop().width, block.crop().height));
2193 }
2194 ++stream;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002195 }
George Burgess IVc813a592020-02-22 22:54:44 -08002196
Wonsik Kim75e22f42021-04-14 23:34:51 -07002197 sp<AMessage> oldFormat = config->mOutputFormat;
2198 config->updateConfiguration(updates, config->mOutputDomain);
2199 RevertOutputFormatIfNeeded(oldFormat, config->mOutputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002200
Wonsik Kim75e22f42021-04-14 23:34:51 -07002201 // copy standard infos to graphic buffers if not already present (otherwise, we
2202 // may overwrite the actual intermediate value with a final value)
2203 stream = 0;
2204 const static C2Param::Index stdGfxInfos[] = {
2205 C2StreamRotationInfo::output::PARAM_TYPE,
2206 C2StreamColorAspectsInfo::output::PARAM_TYPE,
2207 C2StreamDataSpaceInfo::output::PARAM_TYPE,
2208 C2StreamHdrStaticInfo::output::PARAM_TYPE,
2209 C2StreamHdr10PlusInfo::output::PARAM_TYPE,
2210 C2StreamPixelAspectRatioInfo::output::PARAM_TYPE,
2211 C2StreamSurfaceScalingInfo::output::PARAM_TYPE
2212 };
2213 for (const std::shared_ptr<C2Buffer> &buf : outputBuffers) {
2214 if (buf->data().graphicBlocks().size()) {
2215 for (C2Param::Index ix : stdGfxInfos) {
2216 if (!buf->hasInfo(ix)) {
2217 const C2Param *param =
2218 config->getConfigParameterValue(ix.withStream(stream));
2219 if (param) {
2220 std::shared_ptr<C2Param> info(C2Param::Copy(*param));
2221 buf->setInfo(std::static_pointer_cast<C2Info>(info));
2222 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002223 }
2224 }
2225 }
Wonsik Kim75e22f42021-04-14 23:34:51 -07002226 ++stream;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002227 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002228 }
Wonsik Kim75e22f42021-04-14 23:34:51 -07002229 if (config->mInputSurface) {
2230 config->mInputSurface->onInputBufferDone(work->input.ordinal.frameIndex);
2231 }
2232 if (initDataWatcher.hasChanged()) {
2233 initData = C2Param::Copy(*initDataWatcher.update().get());
2234 }
2235 outputFormat = config->mOutputFormat;
Wonsik Kim9c387412021-04-19 21:03:53 +00002236 }
2237 mChannel->onWorkDone(
Wonsik Kim75e22f42021-04-14 23:34:51 -07002238 std::move(work), outputFormat,
2239 initData ? (C2StreamInitDataInfo::output *)initData.get() : nullptr);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002240 break;
2241 }
2242 case kWhatWatch: {
2243 // watch message already posted; no-op.
2244 break;
2245 }
2246 default: {
2247 ALOGE("unrecognized message");
2248 break;
2249 }
2250 }
2251 setDeadline(TimePoint::max(), 0ms, "none");
2252}
2253
2254void CCodec::setDeadline(
2255 const TimePoint &now,
2256 const std::chrono::milliseconds &timeout,
2257 const char *name) {
2258 int32_t mult = std::max(1, property_get_int32("debug.stagefright.ccodec_timeout_mult", 1));
2259 Mutexed<NamedTimePoint>::Locked deadline(mDeadline);
2260 deadline->set(now + (timeout * mult), name);
2261}
2262
ted.sun765db4d2020-06-23 14:03:41 +08002263status_t CCodec::configureTunneledVideoPlayback(
2264 std::shared_ptr<Codec2Client::Component> comp,
2265 sp<NativeHandle> *sidebandHandle,
2266 const sp<AMessage> &msg) {
2267 std::vector<std::unique_ptr<C2SettingResult>> failures;
2268
2269 std::unique_ptr<C2PortTunneledModeTuning::output> tunneledPlayback =
2270 C2PortTunneledModeTuning::output::AllocUnique(
2271 1,
2272 C2PortTunneledModeTuning::Struct::SIDEBAND,
2273 C2PortTunneledModeTuning::Struct::REALTIME,
2274 0);
2275 // TODO: use KEY_AUDIO_HW_SYNC, KEY_HARDWARE_AV_SYNC_ID when they are in MediaCodecConstants.h
2276 if (msg->findInt32("audio-hw-sync", &tunneledPlayback->m.syncId[0])) {
2277 tunneledPlayback->m.syncType = C2PortTunneledModeTuning::Struct::sync_type_t::AUDIO_HW_SYNC;
2278 } else if (msg->findInt32("hw-av-sync-id", &tunneledPlayback->m.syncId[0])) {
2279 tunneledPlayback->m.syncType = C2PortTunneledModeTuning::Struct::sync_type_t::HW_AV_SYNC;
2280 } else {
2281 tunneledPlayback->m.syncType = C2PortTunneledModeTuning::Struct::sync_type_t::REALTIME;
2282 tunneledPlayback->setFlexCount(0);
2283 }
2284 c2_status_t c2err = comp->config({ tunneledPlayback.get() }, C2_MAY_BLOCK, &failures);
2285 if (c2err != C2_OK) {
2286 return UNKNOWN_ERROR;
2287 }
2288
Houxiang Dai5a97b472021-03-22 17:56:04 +08002289 if (sidebandHandle == nullptr) {
2290 return OK;
2291 }
2292
ted.sun765db4d2020-06-23 14:03:41 +08002293 std::vector<std::unique_ptr<C2Param>> params;
2294 c2err = comp->query({}, {C2PortTunnelHandleTuning::output::PARAM_TYPE}, C2_DONT_BLOCK, &params);
2295 if (c2err == C2_OK && params.size() == 1u) {
2296 C2PortTunnelHandleTuning::output *videoTunnelSideband =
2297 C2PortTunnelHandleTuning::output::From(params[0].get());
2298 // Currently, Codec2 only supports non-fd case for sideband native_handle.
2299 native_handle_t *handle = native_handle_create(0, videoTunnelSideband->flexCount());
2300 *sidebandHandle = NativeHandle::create(handle, true /* ownsHandle */);
2301 if (handle != nullptr && videoTunnelSideband->flexCount()) {
2302 memcpy(handle->data, videoTunnelSideband->m.values,
2303 sizeof(int32_t) * videoTunnelSideband->flexCount());
2304 return OK;
2305 } else {
2306 return NO_MEMORY;
2307 }
2308 }
2309 return UNKNOWN_ERROR;
2310}
2311
Pawin Vongmasa36653902018-11-15 00:10:25 -08002312void CCodec::initiateReleaseIfStuck() {
2313 std::string name;
2314 bool pendingDeadline = false;
Wonsik Kimab34ed62019-01-31 15:28:46 -08002315 {
2316 Mutexed<NamedTimePoint>::Locked deadline(mDeadline);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002317 if (deadline->get() < std::chrono::steady_clock::now()) {
2318 name = deadline->getName();
Pawin Vongmasa36653902018-11-15 00:10:25 -08002319 }
2320 if (deadline->get() != TimePoint::max()) {
2321 pendingDeadline = true;
2322 }
2323 }
Wonsik Kim75e22f42021-04-14 23:34:51 -07002324 bool tunneled = false;
2325 {
2326 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2327 const std::unique_ptr<Config> &config = *configLocked;
2328 tunneled = config->mTunneled;
2329 }
2330 if (!tunneled && name.empty()) {
Wonsik Kimab34ed62019-01-31 15:28:46 -08002331 constexpr std::chrono::steady_clock::duration kWorkDurationThreshold = 3s;
2332 std::chrono::steady_clock::duration elapsed = mChannel->elapsed();
2333 if (elapsed >= kWorkDurationThreshold) {
2334 name = "queue";
2335 }
2336 if (elapsed > 0s) {
2337 pendingDeadline = true;
2338 }
2339 }
2340 if (name.empty()) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002341 // We're not stuck.
2342 if (pendingDeadline) {
2343 // If we are not stuck yet but still has deadline coming up,
2344 // post watch message to check back later.
2345 (new AMessage(kWhatWatch, this))->post();
2346 }
2347 return;
2348 }
2349
Chih-Yu Huang82e5ab32021-02-17 16:27:08 +09002350 C2String compName;
2351 {
2352 Mutexed<State>::Locked state(mState);
2353 compName = state->comp->getName();
2354 }
2355 ALOGW("[%s] previous call to %s exceeded timeout", compName.c_str(), name.c_str());
2356
Pawin Vongmasa36653902018-11-15 00:10:25 -08002357 initiateRelease(false);
2358 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2359}
2360
Wonsik Kim155d5cb2019-10-09 12:49:49 -07002361// static
2362PersistentSurface *CCodec::CreateInputSurface() {
Pawin Vongmasa18588322019-05-18 01:52:13 -07002363 using namespace android;
Wonsik Kim9917d4a2019-10-24 12:56:38 -07002364 using ::android::hardware::media::omx::V1_0::implementation::TWGraphicBufferSource;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002365 // Attempt to create a Codec2's input surface.
Pawin Vongmasa18588322019-05-18 01:52:13 -07002366 std::shared_ptr<Codec2Client::InputSurface> inputSurface =
2367 Codec2Client::CreateInputSurface();
Lajos Molnar47118272019-01-31 16:28:04 -08002368 if (!inputSurface) {
Pawin Vongmasa18588322019-05-18 01:52:13 -07002369 if (property_get_int32("debug.stagefright.c2inputsurface", 0) == -1) {
2370 sp<IGraphicBufferProducer> gbp;
2371 sp<OmxGraphicBufferSource> gbs = new OmxGraphicBufferSource();
2372 status_t err = gbs->initCheck();
2373 if (err != OK) {
2374 ALOGE("Failed to create persistent input surface: error %d", err);
2375 return nullptr;
2376 }
2377 return new PersistentSurface(
Wonsik Kim9917d4a2019-10-24 12:56:38 -07002378 gbs->getIGraphicBufferProducer(), new TWGraphicBufferSource(gbs));
Pawin Vongmasa18588322019-05-18 01:52:13 -07002379 } else {
2380 return nullptr;
2381 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002382 }
Pawin Vongmasa18588322019-05-18 01:52:13 -07002383 return new PersistentSurface(
Lajos Molnar47118272019-01-31 16:28:04 -08002384 inputSurface->getGraphicBufferProducer(),
Pawin Vongmasa18588322019-05-18 01:52:13 -07002385 static_cast<sp<android::hidl::base::V1_0::IBase>>(
Lajos Molnar47118272019-01-31 16:28:04 -08002386 inputSurface->getHalInterface()));
Pawin Vongmasa36653902018-11-15 00:10:25 -08002387}
2388
Wonsik Kimffb889a2020-05-28 11:32:25 -07002389class IntfCache {
2390public:
2391 IntfCache() = default;
2392
2393 status_t init(const std::string &name) {
2394 std::shared_ptr<Codec2Client::Interface> intf{
2395 Codec2Client::CreateInterfaceByName(name.c_str())};
2396 if (!intf) {
2397 ALOGW("IntfCache [%s]: Unrecognized interface name", name.c_str());
2398 mInitStatus = NO_INIT;
2399 return NO_INIT;
2400 }
2401 const static C2StreamUsageTuning::input sUsage{0u /* stream id */};
2402 mFields.push_back(C2FieldSupportedValuesQuery::Possible(
2403 C2ParamField{&sUsage, &sUsage.value}));
2404 c2_status_t err = intf->querySupportedValues(mFields, C2_MAY_BLOCK);
2405 if (err != C2_OK) {
2406 ALOGW("IntfCache [%s]: failed to query usage supported value (err=%d)",
2407 name.c_str(), err);
2408 mFields[0].status = err;
2409 }
2410 std::vector<std::unique_ptr<C2Param>> params;
2411 err = intf->query(
2412 {&mApiFeatures},
2413 {C2PortAllocatorsTuning::input::PARAM_TYPE},
2414 C2_MAY_BLOCK,
2415 &params);
2416 if (err != C2_OK && err != C2_BAD_INDEX) {
2417 ALOGW("IntfCache [%s]: failed to query api features (err=%d)",
2418 name.c_str(), err);
2419 }
2420 while (!params.empty()) {
2421 C2Param *param = params.back().release();
2422 params.pop_back();
2423 if (!param) {
2424 continue;
2425 }
2426 if (param->type() == C2PortAllocatorsTuning::input::PARAM_TYPE) {
2427 mInputAllocators.reset(
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07002428 C2PortAllocatorsTuning::input::From(param));
Wonsik Kimffb889a2020-05-28 11:32:25 -07002429 }
2430 }
2431 mInitStatus = OK;
2432 return OK;
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002433 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07002434
2435 status_t initCheck() const { return mInitStatus; }
2436
2437 const C2FieldSupportedValuesQuery &getUsageSupportedValues() const {
2438 CHECK_EQ(1u, mFields.size());
2439 return mFields[0];
2440 }
2441
2442 const C2ApiFeaturesSetting &getApiFeatures() const {
2443 return mApiFeatures;
2444 }
2445
2446 const C2PortAllocatorsTuning::input &getInputAllocators() const {
2447 static std::unique_ptr<C2PortAllocatorsTuning::input> sInvalidated = []{
2448 std::unique_ptr<C2PortAllocatorsTuning::input> param =
2449 C2PortAllocatorsTuning::input::AllocUnique(0);
2450 param->invalidate();
2451 return param;
2452 }();
2453 return mInputAllocators ? *mInputAllocators : *sInvalidated;
2454 }
2455
2456private:
2457 status_t mInitStatus{NO_INIT};
2458
2459 std::vector<C2FieldSupportedValuesQuery> mFields;
2460 C2ApiFeaturesSetting mApiFeatures;
2461 std::unique_ptr<C2PortAllocatorsTuning::input> mInputAllocators;
2462};
2463
2464static const IntfCache &GetIntfCache(const std::string &name) {
2465 static IntfCache sNullIntfCache;
2466 static std::mutex sMutex;
2467 static std::map<std::string, IntfCache> sCache;
2468 std::unique_lock<std::mutex> lock{sMutex};
2469 auto it = sCache.find(name);
2470 if (it == sCache.end()) {
2471 lock.unlock();
2472 IntfCache intfCache;
2473 status_t err = intfCache.init(name);
2474 if (err != OK) {
2475 return sNullIntfCache;
2476 }
2477 lock.lock();
2478 it = sCache.insert({name, std::move(intfCache)}).first;
2479 }
2480 return it->second;
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002481}
2482
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002483static status_t GetCommonAllocatorIds(
2484 const std::vector<std::string> &names,
2485 C2Allocator::type_t type,
2486 std::set<C2Allocator::id_t> *ids) {
2487 int poolMask = GetCodec2PoolMask();
2488 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
2489 C2Allocator::id_t defaultAllocatorId =
2490 (type == C2Allocator::LINEAR) ? preferredLinearId : C2PlatformAllocatorStore::GRALLOC;
2491
2492 ids->clear();
2493 if (names.empty()) {
2494 return OK;
2495 }
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002496 bool firstIteration = true;
2497 for (const std::string &name : names) {
Wonsik Kimffb889a2020-05-28 11:32:25 -07002498 const IntfCache &intfCache = GetIntfCache(name);
2499 if (intfCache.initCheck() != OK) {
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002500 continue;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002501 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07002502 const C2PortAllocatorsTuning::input &allocators = intfCache.getInputAllocators();
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002503 if (firstIteration) {
2504 firstIteration = false;
Wonsik Kimffb889a2020-05-28 11:32:25 -07002505 if (allocators && allocators.flexCount() > 0) {
2506 ids->insert(allocators.m.values,
2507 allocators.m.values + allocators.flexCount());
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002508 }
2509 if (ids->empty()) {
2510 // The component does not advertise allocators. Use default.
2511 ids->insert(defaultAllocatorId);
2512 }
2513 continue;
2514 }
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002515 bool filtered = false;
Wonsik Kimffb889a2020-05-28 11:32:25 -07002516 if (allocators && allocators.flexCount() > 0) {
2517 filtered = true;
2518 for (auto it = ids->begin(); it != ids->end(); ) {
2519 bool found = false;
2520 for (size_t j = 0; j < allocators.flexCount(); ++j) {
2521 if (allocators.m.values[j] == *it) {
2522 found = true;
2523 break;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002524 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07002525 }
2526 if (found) {
2527 ++it;
2528 } else {
2529 it = ids->erase(it);
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002530 }
2531 }
2532 }
2533 if (!filtered) {
2534 // The component does not advertise supported allocators. Use default.
2535 bool containsDefault = (ids->count(defaultAllocatorId) > 0u);
2536 if (ids->size() != (containsDefault ? 1 : 0)) {
2537 ids->clear();
2538 if (containsDefault) {
2539 ids->insert(defaultAllocatorId);
2540 }
2541 }
2542 }
2543 }
2544 // Finally, filter with pool masks
2545 for (auto it = ids->begin(); it != ids->end(); ) {
2546 if ((poolMask >> *it) & 1) {
2547 ++it;
2548 } else {
2549 it = ids->erase(it);
2550 }
2551 }
2552 return OK;
2553}
2554
2555static status_t CalculateMinMaxUsage(
2556 const std::vector<std::string> &names, uint64_t *minUsage, uint64_t *maxUsage) {
2557 static C2StreamUsageTuning::input sUsage{0u /* stream id */};
2558 *minUsage = 0;
2559 *maxUsage = ~0ull;
2560 for (const std::string &name : names) {
Wonsik Kimffb889a2020-05-28 11:32:25 -07002561 const IntfCache &intfCache = GetIntfCache(name);
2562 if (intfCache.initCheck() != OK) {
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002563 continue;
2564 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07002565 const C2FieldSupportedValuesQuery &usageSupportedValues =
2566 intfCache.getUsageSupportedValues();
2567 if (usageSupportedValues.status != C2_OK) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002568 continue;
2569 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07002570 const C2FieldSupportedValues &supported = usageSupportedValues.values;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002571 if (supported.type != C2FieldSupportedValues::FLAGS) {
2572 continue;
2573 }
2574 if (supported.values.empty()) {
2575 *maxUsage = 0;
2576 continue;
2577 }
2578 *minUsage |= supported.values[0].u64;
2579 int64_t currentMaxUsage = 0;
2580 for (const C2Value::Primitive &flags : supported.values) {
2581 currentMaxUsage |= flags.u64;
2582 }
2583 *maxUsage &= currentMaxUsage;
2584 }
2585 return OK;
2586}
2587
2588// static
2589status_t CCodec::CanFetchLinearBlock(
2590 const std::vector<std::string> &names, const C2MemoryUsage &usage, bool *isCompatible) {
Wonsik Kimffb889a2020-05-28 11:32:25 -07002591 for (const std::string &name : names) {
2592 const IntfCache &intfCache = GetIntfCache(name);
2593 if (intfCache.initCheck() != OK) {
2594 continue;
2595 }
2596 const C2ApiFeaturesSetting &features = intfCache.getApiFeatures();
2597 if (features && !(features.value & API_SAME_INPUT_BUFFER)) {
2598 *isCompatible = false;
2599 return OK;
2600 }
2601 }
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002602 std::set<C2Allocator::id_t> allocators;
2603 GetCommonAllocatorIds(names, C2Allocator::LINEAR, &allocators);
2604 if (allocators.empty()) {
2605 *isCompatible = false;
2606 return OK;
2607 }
Chih-Yu Huang990b5622020-10-13 14:53:37 +09002608
2609 uint64_t minUsage = 0;
2610 uint64_t maxUsage = ~0ull;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002611 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
Chih-Yu Huang990b5622020-10-13 14:53:37 +09002612 minUsage |= usage.expected;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002613 *isCompatible = ((maxUsage & minUsage) == minUsage);
2614 return OK;
2615}
2616
2617static std::shared_ptr<C2BlockPool> GetPool(C2Allocator::id_t allocId) {
2618 static std::mutex sMutex{};
2619 static std::map<C2Allocator::id_t, std::shared_ptr<C2BlockPool>> sPools;
2620 std::unique_lock<std::mutex> lock{sMutex};
2621 std::shared_ptr<C2BlockPool> pool;
2622 auto it = sPools.find(allocId);
2623 if (it == sPools.end()) {
2624 c2_status_t err = CreateCodec2BlockPool(allocId, nullptr, &pool);
2625 if (err == OK) {
2626 sPools.emplace(allocId, pool);
2627 } else {
2628 pool.reset();
2629 }
2630 } else {
2631 pool = it->second;
2632 }
2633 return pool;
2634}
2635
2636// static
2637std::shared_ptr<C2LinearBlock> CCodec::FetchLinearBlock(
2638 size_t capacity, const C2MemoryUsage &usage, const std::vector<std::string> &names) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002639 std::set<C2Allocator::id_t> allocators;
2640 GetCommonAllocatorIds(names, C2Allocator::LINEAR, &allocators);
2641 if (allocators.empty()) {
2642 allocators.insert(C2PlatformAllocatorStore::DEFAULT_LINEAR);
2643 }
Chih-Yu Huang990b5622020-10-13 14:53:37 +09002644
2645 uint64_t minUsage = 0;
2646 uint64_t maxUsage = ~0ull;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002647 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
Chih-Yu Huang990b5622020-10-13 14:53:37 +09002648 minUsage |= usage.expected;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002649 if ((maxUsage & minUsage) != minUsage) {
2650 allocators.clear();
2651 allocators.insert(C2PlatformAllocatorStore::DEFAULT_LINEAR);
2652 }
2653 std::shared_ptr<C2LinearBlock> block;
2654 for (C2Allocator::id_t allocId : allocators) {
2655 std::shared_ptr<C2BlockPool> pool = GetPool(allocId);
2656 if (!pool) {
2657 continue;
2658 }
2659 c2_status_t err = pool->fetchLinearBlock(capacity, C2MemoryUsage{minUsage}, &block);
2660 if (err != C2_OK || !block) {
2661 block.reset();
2662 continue;
2663 }
2664 break;
2665 }
2666 return block;
2667}
2668
2669// static
2670status_t CCodec::CanFetchGraphicBlock(
2671 const std::vector<std::string> &names, bool *isCompatible) {
2672 uint64_t minUsage = 0;
2673 uint64_t maxUsage = ~0ull;
2674 std::set<C2Allocator::id_t> allocators;
2675 GetCommonAllocatorIds(names, C2Allocator::GRAPHIC, &allocators);
2676 if (allocators.empty()) {
2677 *isCompatible = false;
2678 return OK;
2679 }
2680 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
2681 *isCompatible = ((maxUsage & minUsage) == minUsage);
2682 return OK;
2683}
2684
2685// static
2686std::shared_ptr<C2GraphicBlock> CCodec::FetchGraphicBlock(
2687 int32_t width,
2688 int32_t height,
2689 int32_t format,
2690 uint64_t usage,
2691 const std::vector<std::string> &names) {
2692 uint32_t halPixelFormat = HAL_PIXEL_FORMAT_YCBCR_420_888;
2693 if (!C2Mapper::mapPixelFormatFrameworkToCodec(format, &halPixelFormat)) {
2694 ALOGD("Unrecognized pixel format: %d", format);
2695 return nullptr;
2696 }
2697 uint64_t minUsage = 0;
2698 uint64_t maxUsage = ~0ull;
2699 std::set<C2Allocator::id_t> allocators;
2700 GetCommonAllocatorIds(names, C2Allocator::GRAPHIC, &allocators);
2701 if (allocators.empty()) {
2702 allocators.insert(C2PlatformAllocatorStore::DEFAULT_GRAPHIC);
2703 }
2704 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
2705 minUsage |= usage;
2706 if ((maxUsage & minUsage) != minUsage) {
2707 allocators.clear();
2708 allocators.insert(C2PlatformAllocatorStore::DEFAULT_GRAPHIC);
2709 }
2710 std::shared_ptr<C2GraphicBlock> block;
2711 for (C2Allocator::id_t allocId : allocators) {
2712 std::shared_ptr<C2BlockPool> pool;
2713 c2_status_t err = CreateCodec2BlockPool(allocId, nullptr, &pool);
2714 if (err != C2_OK || !pool) {
2715 continue;
2716 }
2717 err = pool->fetchGraphicBlock(
2718 width, height, halPixelFormat, C2MemoryUsage{minUsage}, &block);
2719 if (err != C2_OK || !block) {
2720 block.reset();
2721 continue;
2722 }
2723 break;
2724 }
2725 return block;
2726}
2727
Wonsik Kim155d5cb2019-10-09 12:49:49 -07002728} // namespace android