blob: cdb860c984545afc4acccf64cc9a6a0bf0d9b771 [file] [log] [blame]
Andreas Huberf9334412010-12-15 15:17:42 -08001/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerDecoder"
19#include <utils/Log.h>
Lajos Molnar1cd13982014-01-17 15:12:51 -080020#include <inttypes.h>
Andreas Huberf9334412010-12-15 15:17:42 -080021
22#include "NuPlayerDecoder.h"
23
Lajos Molnar1cd13982014-01-17 15:12:51 -080024#include <media/ICrypto.h>
Chong Zhanga7fa1d92014-06-11 14:49:23 -070025#include <media/stagefright/foundation/ABitReader.h>
Andreas Huberf9334412010-12-15 15:17:42 -080026#include <media/stagefright/foundation/ABuffer.h>
27#include <media/stagefright/foundation/ADebug.h>
Andreas Huber5bc087c2010-12-23 10:27:40 -080028#include <media/stagefright/foundation/AMessage.h>
Lajos Molnar09524832014-07-17 14:29:51 -070029#include <media/stagefright/MediaBuffer.h>
Lajos Molnar1cd13982014-01-17 15:12:51 -080030#include <media/stagefright/MediaCodec.h>
Andreas Huberf9334412010-12-15 15:17:42 -080031#include <media/stagefright/MediaDefs.h>
Lajos Molnar1cd13982014-01-17 15:12:51 -080032#include <media/stagefright/MediaErrors.h>
Andreas Huberf9334412010-12-15 15:17:42 -080033
34namespace android {
35
36NuPlayer::Decoder::Decoder(
Glenn Kasten11731182011-02-08 17:26:17 -080037 const sp<AMessage> &notify,
38 const sp<NativeWindowWrapper> &nativeWindow)
Andreas Huberf9334412010-12-15 15:17:42 -080039 : mNotify(notify),
Lajos Molnar1cd13982014-01-17 15:12:51 -080040 mNativeWindow(nativeWindow),
41 mBufferGeneration(0),
Wei Jia704e7262014-06-04 16:21:56 -070042 mPaused(true),
Lajos Molnar1cd13982014-01-17 15:12:51 -080043 mComponentName("decoder") {
44 // Every decoder has its own looper because MediaCodec operations
45 // are blocking, but NuPlayer needs asynchronous operations.
46 mDecoderLooper = new ALooper;
Marco Nelissen9e2b7912014-08-18 16:13:03 -070047 mDecoderLooper->setName("NPDecoder");
Lajos Molnar1cd13982014-01-17 15:12:51 -080048 mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
49
50 mCodecLooper = new ALooper;
Marco Nelissen9e2b7912014-08-18 16:13:03 -070051 mCodecLooper->setName("NPDecoder-CL");
Lajos Molnar1cd13982014-01-17 15:12:51 -080052 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
Andreas Huberf9334412010-12-15 15:17:42 -080053}
54
55NuPlayer::Decoder::~Decoder() {
56}
57
Lajos Molnar09524832014-07-17 14:29:51 -070058static
59status_t PostAndAwaitResponse(
60 const sp<AMessage> &msg, sp<AMessage> *response) {
61 status_t err = msg->postAndAwaitResponse(response);
62
63 if (err != OK) {
64 return err;
65 }
66
67 if (!(*response)->findInt32("err", &err)) {
68 err = OK;
69 }
70
71 return err;
72}
73
Lajos Molnar87603c02014-08-20 19:25:30 -070074void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
75 mCSDsForCurrentFormat.clear();
76 for (int32_t i = 0; ; ++i) {
77 AString tag = "csd-";
78 tag.append(i);
79 sp<ABuffer> buffer;
80 if (!format->findBuffer(tag.c_str(), &buffer)) {
81 break;
82 }
83 mCSDsForCurrentFormat.push(buffer);
84 }
85}
86
Lajos Molnar1cd13982014-01-17 15:12:51 -080087void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
Andreas Huberf9334412010-12-15 15:17:42 -080088 CHECK(mCodec == NULL);
Andreas Huberf9334412010-12-15 15:17:42 -080089
Lajos Molnar1cd13982014-01-17 15:12:51 -080090 ++mBufferGeneration;
91
Andreas Huber84066782011-08-16 09:34:26 -070092 AString mime;
93 CHECK(format->findString("mime", &mime));
Andreas Huberf9334412010-12-15 15:17:42 -080094
Lajos Molnar1cd13982014-01-17 15:12:51 -080095 sp<Surface> surface = NULL;
96 if (mNativeWindow != NULL) {
97 surface = mNativeWindow->getSurfaceTextureClient();
Andreas Huber84066782011-08-16 09:34:26 -070098 }
Andreas Huberf9334412010-12-15 15:17:42 -080099
Lajos Molnar1cd13982014-01-17 15:12:51 -0800100 mComponentName = mime;
101 mComponentName.append(" decoder");
102 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), surface.get());
103
104 mCodec = MediaCodec::CreateByType(mCodecLooper, mime.c_str(), false /* encoder */);
Lajos Molnar09524832014-07-17 14:29:51 -0700105 int32_t secure = 0;
106 if (format->findInt32("secure", &secure) && secure != 0) {
107 if (mCodec != NULL) {
108 mCodec->getName(&mComponentName);
109 mComponentName.append(".secure");
110 mCodec->release();
111 ALOGI("[%s] creating", mComponentName.c_str());
112 mCodec = MediaCodec::CreateByComponentName(
113 mCodecLooper, mComponentName.c_str());
114 }
115 }
Lajos Molnar1cd13982014-01-17 15:12:51 -0800116 if (mCodec == NULL) {
Lajos Molnar09524832014-07-17 14:29:51 -0700117 ALOGE("Failed to create %s%s decoder",
118 (secure ? "secure " : ""), mime.c_str());
Lajos Molnar1cd13982014-01-17 15:12:51 -0800119 handleError(UNKNOWN_ERROR);
120 return;
121 }
122
123 mCodec->getName(&mComponentName);
124
Lajos Molnar14986f62014-09-15 11:04:44 -0700125 status_t err;
Glenn Kasten11731182011-02-08 17:26:17 -0800126 if (mNativeWindow != NULL) {
Lajos Molnar1cd13982014-01-17 15:12:51 -0800127 // disconnect from surface as MediaCodec will reconnect
Lajos Molnar14986f62014-09-15 11:04:44 -0700128 err = native_window_api_disconnect(
129 surface.get(), NATIVE_WINDOW_API_MEDIA);
130 // We treat this as a warning, as this is a preparatory step.
131 // Codec will try to connect to the surface, which is where
132 // any error signaling will occur.
133 ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
Lajos Molnar1cd13982014-01-17 15:12:51 -0800134 }
Lajos Molnar14986f62014-09-15 11:04:44 -0700135 err = mCodec->configure(
Lajos Molnar1cd13982014-01-17 15:12:51 -0800136 format, surface, NULL /* crypto */, 0 /* flags */);
137 if (err != OK) {
138 ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
139 handleError(err);
140 return;
141 }
Lajos Molnar87603c02014-08-20 19:25:30 -0700142 rememberCodecSpecificData(format);
143
Lajos Molnar1cd13982014-01-17 15:12:51 -0800144 // the following should work in configured state
145 CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
146 CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
147
148 err = mCodec->start();
149 if (err != OK) {
150 ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
151 handleError(err);
152 return;
Andreas Huberf9334412010-12-15 15:17:42 -0800153 }
154
Lajos Molnar1cd13982014-01-17 15:12:51 -0800155 // the following should work after start
156 CHECK_EQ((status_t)OK, mCodec->getInputBuffers(&mInputBuffers));
Lajos Molnar09524832014-07-17 14:29:51 -0700157 releaseAndResetMediaBuffers();
Lajos Molnar1cd13982014-01-17 15:12:51 -0800158 CHECK_EQ((status_t)OK, mCodec->getOutputBuffers(&mOutputBuffers));
159 ALOGV("[%s] got %zu input and %zu output buffers",
160 mComponentName.c_str(),
161 mInputBuffers.size(),
162 mOutputBuffers.size());
Andreas Huber078cfcf2011-09-15 12:25:04 -0700163
Lajos Molnar1cd13982014-01-17 15:12:51 -0800164 requestCodecNotification();
Wei Jia704e7262014-06-04 16:21:56 -0700165 mPaused = false;
Lajos Molnar1cd13982014-01-17 15:12:51 -0800166}
Andreas Huber078cfcf2011-09-15 12:25:04 -0700167
Lajos Molnar09524832014-07-17 14:29:51 -0700168void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
169 for (size_t i = 0; i < mMediaBuffers.size(); i++) {
170 if (mMediaBuffers[i] != NULL) {
171 mMediaBuffers[i]->release();
172 mMediaBuffers.editItemAt(i) = NULL;
173 }
174 }
175 mMediaBuffers.resize(mInputBuffers.size());
Wei Jia81e50d02014-07-24 10:28:47 -0700176 for (size_t i = 0; i < mMediaBuffers.size(); i++) {
177 mMediaBuffers.editItemAt(i) = NULL;
178 }
Lajos Molnar09524832014-07-17 14:29:51 -0700179 mInputBufferIsDequeued.clear();
180 mInputBufferIsDequeued.resize(mInputBuffers.size());
Wei Jia81e50d02014-07-24 10:28:47 -0700181 for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
182 mInputBufferIsDequeued.editItemAt(i) = false;
183 }
Lajos Molnar09524832014-07-17 14:29:51 -0700184}
185
Lajos Molnar1cd13982014-01-17 15:12:51 -0800186void NuPlayer::Decoder::requestCodecNotification() {
187 if (mCodec != NULL) {
188 sp<AMessage> reply = new AMessage(kWhatCodecNotify, id());
189 reply->setInt32("generation", mBufferGeneration);
190 mCodec->requestActivityNotification(reply);
191 }
192}
193
194bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
195 int32_t generation;
196 CHECK(msg->findInt32("generation", &generation));
197 return generation != mBufferGeneration;
198}
199
200void NuPlayer::Decoder::init() {
201 mDecoderLooper->registerHandler(this);
202}
203
204void NuPlayer::Decoder::configure(const sp<AMessage> &format) {
205 sp<AMessage> msg = new AMessage(kWhatConfigure, id());
206 msg->setMessage("format", format);
207 msg->post();
208}
209
Lajos Molnar87603c02014-08-20 19:25:30 -0700210void NuPlayer::Decoder::signalUpdateFormat(const sp<AMessage> &format) {
211 sp<AMessage> msg = new AMessage(kWhatUpdateFormat, id());
212 msg->setMessage("format", format);
213 msg->post();
214}
215
Lajos Molnar09524832014-07-17 14:29:51 -0700216status_t NuPlayer::Decoder::getInputBuffers(Vector<sp<ABuffer> > *buffers) const {
217 sp<AMessage> msg = new AMessage(kWhatGetInputBuffers, id());
218 msg->setPointer("buffers", buffers);
219
220 sp<AMessage> response;
221 return PostAndAwaitResponse(msg, &response);
222}
223
Lajos Molnar1cd13982014-01-17 15:12:51 -0800224void NuPlayer::Decoder::handleError(int32_t err)
225{
Wei Jiac22c6952014-08-29 14:47:50 -0700226 mCodec->release();
227
Lajos Molnar1cd13982014-01-17 15:12:51 -0800228 sp<AMessage> notify = mNotify->dup();
229 notify->setInt32("what", kWhatError);
230 notify->setInt32("err", err);
231 notify->post();
232}
233
234bool NuPlayer::Decoder::handleAnInputBuffer() {
235 size_t bufferIx = -1;
236 status_t res = mCodec->dequeueInputBuffer(&bufferIx);
237 ALOGV("[%s] dequeued input: %d",
238 mComponentName.c_str(), res == OK ? (int)bufferIx : res);
239 if (res != OK) {
240 if (res != -EAGAIN) {
241 handleError(res);
242 }
243 return false;
Andreas Huber078cfcf2011-09-15 12:25:04 -0700244 }
245
Lajos Molnar1cd13982014-01-17 15:12:51 -0800246 CHECK_LT(bufferIx, mInputBuffers.size());
Andreas Huberf9334412010-12-15 15:17:42 -0800247
Lajos Molnar09524832014-07-17 14:29:51 -0700248 if (mMediaBuffers[bufferIx] != NULL) {
249 mMediaBuffers[bufferIx]->release();
250 mMediaBuffers.editItemAt(bufferIx) = NULL;
251 }
252 mInputBufferIsDequeued.editItemAt(bufferIx) = true;
253
Lajos Molnar1cd13982014-01-17 15:12:51 -0800254 sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, id());
255 reply->setSize("buffer-ix", bufferIx);
256 reply->setInt32("generation", mBufferGeneration);
257
Lajos Molnar87603c02014-08-20 19:25:30 -0700258 if (!mCSDsToSubmit.isEmpty()) {
259 sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
260 ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
261 reply->setBuffer("buffer", buffer);
262 mCSDsToSubmit.removeAt(0);
263 reply->post();
264 return true;
265 }
266
Lajos Molnar1cd13982014-01-17 15:12:51 -0800267 sp<AMessage> notify = mNotify->dup();
268 notify->setInt32("what", kWhatFillThisBuffer);
269 notify->setBuffer("buffer", mInputBuffers[bufferIx]);
270 notify->setMessage("reply", reply);
271 notify->post();
272 return true;
273}
274
275void android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) {
276 size_t bufferIx;
277 CHECK(msg->findSize("buffer-ix", &bufferIx));
278 CHECK_LT(bufferIx, mInputBuffers.size());
279 sp<ABuffer> codecBuffer = mInputBuffers[bufferIx];
280
281 sp<ABuffer> buffer;
282 bool hasBuffer = msg->findBuffer("buffer", &buffer);
Lajos Molnar09524832014-07-17 14:29:51 -0700283
284 // handle widevine classic source - that fills an arbitrary input buffer
285 MediaBuffer *mediaBuffer = NULL;
Wei Jia96e92b52014-09-18 17:36:20 -0700286 if (hasBuffer) {
287 mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
288 if (mediaBuffer != NULL) {
Lajos Molnar09524832014-07-17 14:29:51 -0700289 // likely filled another buffer than we requested: adjust buffer index
290 size_t ix;
291 for (ix = 0; ix < mInputBuffers.size(); ix++) {
292 const sp<ABuffer> &buf = mInputBuffers[ix];
293 if (buf->data() == mediaBuffer->data()) {
294 // all input buffers are dequeued on start, hence the check
295 CHECK(mInputBufferIsDequeued[ix]);
296 ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
297 mComponentName.c_str(), ix, bufferIx);
298
299 // TRICKY: need buffer for the metadata, so instead, set
300 // codecBuffer to the same (though incorrect) buffer to
301 // avoid a memcpy into the codecBuffer
302 codecBuffer = buffer;
303 codecBuffer->setRange(
304 mediaBuffer->range_offset(),
305 mediaBuffer->range_length());
306 bufferIx = ix;
307 break;
308 }
309 }
310 CHECK(ix < mInputBuffers.size());
311 }
312 }
313
314 mInputBufferIsDequeued.editItemAt(bufferIx) = false;
315
Lajos Molnar1cd13982014-01-17 15:12:51 -0800316 if (buffer == NULL /* includes !hasBuffer */) {
317 int32_t streamErr = ERROR_END_OF_STREAM;
318 CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
319
320 if (streamErr == OK) {
321 /* buffers are returned to hold on to */
322 return;
323 }
324
325 // attempt to queue EOS
326 status_t err = mCodec->queueInputBuffer(
327 bufferIx,
328 0,
329 0,
330 0,
331 MediaCodec::BUFFER_FLAG_EOS);
332 if (streamErr == ERROR_END_OF_STREAM && err != OK) {
333 streamErr = err;
334 // err will not be ERROR_END_OF_STREAM
335 }
336
337 if (streamErr != ERROR_END_OF_STREAM) {
338 handleError(streamErr);
339 }
340 } else {
341 int64_t timeUs = 0;
342 uint32_t flags = 0;
343 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
344
Lajos Molnar87603c02014-08-20 19:25:30 -0700345 int32_t eos, csd;
346 // we do not expect SYNCFRAME for decoder
Lajos Molnar1cd13982014-01-17 15:12:51 -0800347 if (buffer->meta()->findInt32("eos", &eos) && eos) {
348 flags |= MediaCodec::BUFFER_FLAG_EOS;
Lajos Molnar87603c02014-08-20 19:25:30 -0700349 } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
350 flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
Lajos Molnar1cd13982014-01-17 15:12:51 -0800351 }
352
353 // copy into codec buffer
354 if (buffer != codecBuffer) {
355 CHECK_LE(buffer->size(), codecBuffer->capacity());
356 codecBuffer->setRange(0, buffer->size());
357 memcpy(codecBuffer->data(), buffer->data(), buffer->size());
358 }
359
360 status_t err = mCodec->queueInputBuffer(
361 bufferIx,
362 codecBuffer->offset(),
363 codecBuffer->size(),
364 timeUs,
365 flags);
366 if (err != OK) {
367 ALOGE("Failed to queue input buffer for %s (err=%d)",
368 mComponentName.c_str(), err);
369 handleError(err);
370 }
Lajos Molnar09524832014-07-17 14:29:51 -0700371
372 if (mediaBuffer != NULL) {
373 CHECK(mMediaBuffers[bufferIx] == NULL);
374 mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
375 }
Lajos Molnar1cd13982014-01-17 15:12:51 -0800376 }
377}
378
379bool NuPlayer::Decoder::handleAnOutputBuffer() {
380 size_t bufferIx = -1;
381 size_t offset;
382 size_t size;
383 int64_t timeUs;
384 uint32_t flags;
385 status_t res = mCodec->dequeueOutputBuffer(
386 &bufferIx, &offset, &size, &timeUs, &flags);
387
388 if (res != OK) {
389 ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res);
390 } else {
391 ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")",
392 mComponentName.c_str(), (int)bufferIx, timeUs, flags);
393 }
394
395 if (res == INFO_OUTPUT_BUFFERS_CHANGED) {
396 res = mCodec->getOutputBuffers(&mOutputBuffers);
397 if (res != OK) {
398 ALOGE("Failed to get output buffers for %s after INFO event (err=%d)",
399 mComponentName.c_str(), res);
400 handleError(res);
401 return false;
402 }
403 // NuPlayer ignores this
404 return true;
405 } else if (res == INFO_FORMAT_CHANGED) {
406 sp<AMessage> format = new AMessage();
407 res = mCodec->getOutputFormat(&format);
408 if (res != OK) {
409 ALOGE("Failed to get output format for %s after INFO event (err=%d)",
410 mComponentName.c_str(), res);
411 handleError(res);
412 return false;
413 }
414
415 sp<AMessage> notify = mNotify->dup();
416 notify->setInt32("what", kWhatOutputFormatChanged);
417 notify->setMessage("format", format);
418 notify->post();
419 return true;
420 } else if (res == INFO_DISCONTINUITY) {
421 // nothing to do
422 return true;
423 } else if (res != OK) {
424 if (res != -EAGAIN) {
425 handleError(res);
426 }
427 return false;
428 }
429
430 CHECK_LT(bufferIx, mOutputBuffers.size());
431 sp<ABuffer> buffer = mOutputBuffers[bufferIx];
432 buffer->setRange(offset, size);
433 buffer->meta()->clear();
434 buffer->meta()->setInt64("timeUs", timeUs);
435 if (flags & MediaCodec::BUFFER_FLAG_EOS) {
436 buffer->meta()->setInt32("eos", true);
437 }
438 // we do not expect CODECCONFIG or SYNCFRAME for decoder
439
440 sp<AMessage> reply = new AMessage(kWhatRenderBuffer, id());
441 reply->setSize("buffer-ix", bufferIx);
442 reply->setInt32("generation", mBufferGeneration);
443
444 sp<AMessage> notify = mNotify->dup();
445 notify->setInt32("what", kWhatDrainThisBuffer);
446 notify->setBuffer("buffer", buffer);
447 notify->setMessage("reply", reply);
448 notify->post();
449
450 // FIXME: This should be handled after rendering is complete,
451 // but Renderer needs it now
452 if (flags & MediaCodec::BUFFER_FLAG_EOS) {
453 ALOGV("queueing eos [%s]", mComponentName.c_str());
454 sp<AMessage> notify = mNotify->dup();
455 notify->setInt32("what", kWhatEOS);
456 notify->setInt32("err", ERROR_END_OF_STREAM);
457 notify->post();
458 }
459 return true;
460}
461
462void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
463 status_t err;
464 int32_t render;
465 size_t bufferIx;
466 CHECK(msg->findSize("buffer-ix", &bufferIx));
467 if (msg->findInt32("render", &render) && render) {
Lajos Molnardc43dfa2014-05-07 15:33:04 -0700468 int64_t timestampNs;
469 CHECK(msg->findInt64("timestampNs", &timestampNs));
470 err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
Lajos Molnar1cd13982014-01-17 15:12:51 -0800471 } else {
472 err = mCodec->releaseOutputBuffer(bufferIx);
473 }
474 if (err != OK) {
475 ALOGE("failed to release output buffer for %s (err=%d)",
476 mComponentName.c_str(), err);
477 handleError(err);
478 }
479}
480
481void NuPlayer::Decoder::onFlush() {
482 status_t err = OK;
483 if (mCodec != NULL) {
484 err = mCodec->flush();
Lajos Molnar87603c02014-08-20 19:25:30 -0700485 mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
Lajos Molnar1cd13982014-01-17 15:12:51 -0800486 ++mBufferGeneration;
487 }
488
489 if (err != OK) {
490 ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
491 handleError(err);
492 return;
493 }
494
Lajos Molnar09524832014-07-17 14:29:51 -0700495 releaseAndResetMediaBuffers();
496
Lajos Molnar1cd13982014-01-17 15:12:51 -0800497 sp<AMessage> notify = mNotify->dup();
498 notify->setInt32("what", kWhatFlushCompleted);
499 notify->post();
Wei Jia704e7262014-06-04 16:21:56 -0700500 mPaused = true;
501}
502
503void NuPlayer::Decoder::onResume() {
504 mPaused = false;
Lajos Molnar1cd13982014-01-17 15:12:51 -0800505}
506
507void NuPlayer::Decoder::onShutdown() {
508 status_t err = OK;
509 if (mCodec != NULL) {
510 err = mCodec->release();
511 mCodec = NULL;
512 ++mBufferGeneration;
513
514 if (mNativeWindow != NULL) {
515 // reconnect to surface as MediaCodec disconnected from it
Wei Jia3fb9f682014-08-20 14:30:09 -0700516 status_t error =
Lajos Molnar1cd13982014-01-17 15:12:51 -0800517 native_window_api_connect(
518 mNativeWindow->getNativeWindow().get(),
Wei Jia3fb9f682014-08-20 14:30:09 -0700519 NATIVE_WINDOW_API_MEDIA);
520 ALOGW_IF(error != NO_ERROR,
521 "[%s] failed to connect to native window, error=%d",
522 mComponentName.c_str(), error);
Lajos Molnar1cd13982014-01-17 15:12:51 -0800523 }
524 mComponentName = "decoder";
525 }
526
Lajos Molnar09524832014-07-17 14:29:51 -0700527 releaseAndResetMediaBuffers();
528
Lajos Molnar1cd13982014-01-17 15:12:51 -0800529 if (err != OK) {
530 ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
531 handleError(err);
532 return;
533 }
534
535 sp<AMessage> notify = mNotify->dup();
536 notify->setInt32("what", kWhatShutdownCompleted);
537 notify->post();
Wei Jia704e7262014-06-04 16:21:56 -0700538 mPaused = true;
Andreas Huberf9334412010-12-15 15:17:42 -0800539}
540
541void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
Lajos Molnar1cd13982014-01-17 15:12:51 -0800542 ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
543
Andreas Huberf9334412010-12-15 15:17:42 -0800544 switch (msg->what()) {
Lajos Molnar1cd13982014-01-17 15:12:51 -0800545 case kWhatConfigure:
546 {
547 sp<AMessage> format;
548 CHECK(msg->findMessage("format", &format));
549 onConfigure(format);
550 break;
551 }
552
Lajos Molnar87603c02014-08-20 19:25:30 -0700553 case kWhatUpdateFormat:
554 {
555 sp<AMessage> format;
556 CHECK(msg->findMessage("format", &format));
557 rememberCodecSpecificData(format);
558 break;
559 }
560
Lajos Molnar09524832014-07-17 14:29:51 -0700561 case kWhatGetInputBuffers:
562 {
563 uint32_t replyID;
564 CHECK(msg->senderAwaitsResponse(&replyID));
565
566 Vector<sp<ABuffer> > *dstBuffers;
567 CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
568
569 dstBuffers->clear();
570 for (size_t i = 0; i < mInputBuffers.size(); i++) {
571 dstBuffers->push(mInputBuffers[i]);
572 }
573
574 (new AMessage)->postReply(replyID);
575 break;
576 }
577
Andreas Huberf9334412010-12-15 15:17:42 -0800578 case kWhatCodecNotify:
579 {
Lajos Molnar1cd13982014-01-17 15:12:51 -0800580 if (!isStaleReply(msg)) {
Wei Jia704e7262014-06-04 16:21:56 -0700581 if (!mPaused) {
582 while (handleAnInputBuffer()) {
583 }
Lajos Molnar1cd13982014-01-17 15:12:51 -0800584 }
Andreas Huberf9334412010-12-15 15:17:42 -0800585
Lajos Molnar1cd13982014-01-17 15:12:51 -0800586 while (handleAnOutputBuffer()) {
587 }
Andreas Huberf9334412010-12-15 15:17:42 -0800588 }
Lajos Molnar1cd13982014-01-17 15:12:51 -0800589
590 requestCodecNotification();
591 break;
592 }
593
594 case kWhatInputBufferFilled:
595 {
596 if (!isStaleReply(msg)) {
597 onInputBufferFilled(msg);
598 }
Lajos Molnarb9b87fe2014-09-10 13:53:21 -0700599
Lajos Molnar1cd13982014-01-17 15:12:51 -0800600 break;
601 }
602
603 case kWhatRenderBuffer:
604 {
605 if (!isStaleReply(msg)) {
606 onRenderBuffer(msg);
607 }
608 break;
609 }
610
611 case kWhatFlush:
612 {
Lajos Molnar87603c02014-08-20 19:25:30 -0700613 sp<AMessage> format;
614 if (msg->findMessage("new-format", &format)) {
615 rememberCodecSpecificData(format);
616 }
Lajos Molnar1cd13982014-01-17 15:12:51 -0800617 onFlush();
618 break;
619 }
620
Wei Jia704e7262014-06-04 16:21:56 -0700621 case kWhatResume:
622 {
623 onResume();
624 break;
625 }
626
Lajos Molnar1cd13982014-01-17 15:12:51 -0800627 case kWhatShutdown:
628 {
629 onShutdown();
Andreas Huberf9334412010-12-15 15:17:42 -0800630 break;
631 }
632
633 default:
634 TRESPASS();
635 break;
636 }
637}
638
Lajos Molnar87603c02014-08-20 19:25:30 -0700639void NuPlayer::Decoder::signalFlush(const sp<AMessage> &format) {
640 sp<AMessage> msg = new AMessage(kWhatFlush, id());
641 if (format != NULL) {
642 msg->setMessage("new-format", format);
643 }
644 msg->post();
Andreas Huberf9334412010-12-15 15:17:42 -0800645}
646
647void NuPlayer::Decoder::signalResume() {
Wei Jia704e7262014-06-04 16:21:56 -0700648 (new AMessage(kWhatResume, id()))->post();
Andreas Huberf9334412010-12-15 15:17:42 -0800649}
650
Andreas Huber3831a062010-12-21 10:22:33 -0800651void NuPlayer::Decoder::initiateShutdown() {
Lajos Molnar1cd13982014-01-17 15:12:51 -0800652 (new AMessage(kWhatShutdown, id()))->post();
Andreas Huber3831a062010-12-21 10:22:33 -0800653}
654
Robert Shih6d0a94e2014-01-23 16:18:22 -0800655bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(const sp<AMessage> &targetFormat) const {
656 if (targetFormat == NULL) {
657 return true;
658 }
659
660 AString mime;
661 if (!targetFormat->findString("mime", &mime)) {
662 return false;
663 }
664
665 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
666 // field-by-field comparison
667 const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
668 for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
669 int32_t oldVal, newVal;
Lajos Molnar1cd13982014-01-17 15:12:51 -0800670 if (!mOutputFormat->findInt32(keys[i], &oldVal) ||
671 !targetFormat->findInt32(keys[i], &newVal) ||
672 oldVal != newVal) {
Robert Shih6d0a94e2014-01-23 16:18:22 -0800673 return false;
674 }
675 }
676
677 sp<ABuffer> oldBuf, newBuf;
Lajos Molnar1cd13982014-01-17 15:12:51 -0800678 if (mOutputFormat->findBuffer("csd-0", &oldBuf) &&
679 targetFormat->findBuffer("csd-0", &newBuf)) {
Robert Shih6d0a94e2014-01-23 16:18:22 -0800680 if (oldBuf->size() != newBuf->size()) {
681 return false;
682 }
683 return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
684 }
685 }
686 return false;
687}
688
689bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
Lajos Molnar1cd13982014-01-17 15:12:51 -0800690 if (mOutputFormat == NULL) {
Robert Shih6d0a94e2014-01-23 16:18:22 -0800691 return false;
692 }
693
694 if (targetFormat == NULL) {
695 return true;
696 }
697
698 AString oldMime, newMime;
Lajos Molnar1cd13982014-01-17 15:12:51 -0800699 if (!mOutputFormat->findString("mime", &oldMime)
Robert Shih6d0a94e2014-01-23 16:18:22 -0800700 || !targetFormat->findString("mime", &newMime)
701 || !(oldMime == newMime)) {
702 return false;
703 }
704
705 bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
706 bool seamless;
707 if (audio) {
708 seamless = supportsSeamlessAudioFormatChange(targetFormat);
709 } else {
Lajos Molnar1cd13982014-01-17 15:12:51 -0800710 int32_t isAdaptive;
711 seamless = (mCodec != NULL &&
712 mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
713 isAdaptive);
Robert Shih6d0a94e2014-01-23 16:18:22 -0800714 }
715
716 ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
717 return seamless;
718}
719
Chong Zhangb86e68f2014-08-01 13:46:53 -0700720struct CCData {
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700721 CCData(uint8_t type, uint8_t data1, uint8_t data2)
722 : mType(type), mData1(data1), mData2(data2) {
723 }
Chong Zhangb86e68f2014-08-01 13:46:53 -0700724 bool getChannel(size_t *channel) const {
725 if (mData1 >= 0x10 && mData1 <= 0x1f) {
726 *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0);
727 return true;
728 }
729 return false;
730 }
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700731
732 uint8_t mType;
733 uint8_t mData1;
734 uint8_t mData2;
735};
736
Chong Zhangb86e68f2014-08-01 13:46:53 -0700737static bool isNullPad(CCData *cc) {
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700738 return cc->mData1 < 0x10 && cc->mData2 < 0x10;
739}
740
Chong Zhangb86e68f2014-08-01 13:46:53 -0700741static void dumpBytePair(const sp<ABuffer> &ccBuf) {
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700742 size_t offset = 0;
743 AString out;
744
745 while (offset < ccBuf->size()) {
746 char tmp[128];
747
748 CCData *cc = (CCData *) (ccBuf->data() + offset);
749
750 if (isNullPad(cc)) {
751 // 1 null pad or XDS metadata, ignore
752 offset += sizeof(CCData);
753 continue;
754 }
755
756 if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
757 // 2 basic chars
758 sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
759 } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
760 && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
761 // 1 special char
762 sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
763 } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
764 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
765 // 1 Spanish/French char
766 sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
767 } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
768 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
769 // 1 Portuguese/German/Danish char
770 sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
771 } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
772 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
773 // Mid-Row Codes (Table 69)
774 sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
775 } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
776 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
777 ||
778 ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
779 && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
780 // Misc Control Codes (Table 70)
781 sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
782 } else if ((cc->mData1 & 0x70) == 0x10
783 && (cc->mData2 & 0x40) == 0x40
784 && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
785 // Preamble Address Codes (Table 71)
786 sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
787 } else {
788 sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
789 }
790
791 if (out.size() > 0) {
792 out.append(", ");
793 }
794
795 out.append(tmp);
796
797 offset += sizeof(CCData);
798 }
799
800 ALOGI("%s", out.c_str());
801}
802
Chong Zhangb86e68f2014-08-01 13:46:53 -0700803NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> &notify)
804 : mNotify(notify),
805 mCurrentChannel(0),
806 mSelectedTrack(-1) {
807 for (size_t i = 0; i < sizeof(mTrackIndices)/sizeof(mTrackIndices[0]); ++i) {
808 mTrackIndices[i] = -1;
809 }
810}
811
812size_t NuPlayer::CCDecoder::getTrackCount() const {
813 return mFoundChannels.size();
814}
815
816sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
817 if (!isTrackValid(index)) {
818 return NULL;
819 }
820
821 sp<AMessage> format = new AMessage();
822
823 format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
824 format->setString("language", "und");
825 format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
826 //CC1, field 0 channel 0
827 bool isDefaultAuto = (mFoundChannels[index] == 0);
828 format->setInt32("auto", isDefaultAuto);
829 format->setInt32("default", isDefaultAuto);
830 format->setInt32("forced", 0);
831
832 return format;
833}
834
835status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
836 if (!isTrackValid(index)) {
837 return BAD_VALUE;
838 }
839
840 if (select) {
841 if (mSelectedTrack == (ssize_t)index) {
842 ALOGE("track %zu already selected", index);
843 return BAD_VALUE;
844 }
845 ALOGV("selected track %zu", index);
846 mSelectedTrack = index;
847 } else {
848 if (mSelectedTrack != (ssize_t)index) {
849 ALOGE("track %zu is not selected", index);
850 return BAD_VALUE;
851 }
852 ALOGV("unselected track %zu", index);
853 mSelectedTrack = -1;
854 }
855
856 return OK;
857}
858
859bool NuPlayer::CCDecoder::isSelected() const {
860 return mSelectedTrack >= 0 && mSelectedTrack < (int32_t) getTrackCount();
861}
862
863bool NuPlayer::CCDecoder::isTrackValid(size_t index) const {
864 return index < getTrackCount();
865}
866
867int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const {
868 if (channel < sizeof(mTrackIndices)/sizeof(mTrackIndices[0])) {
869 return mTrackIndices[channel];
870 }
871 return -1;
872}
873
874// returns true if a new CC track is found
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700875bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
876 int64_t timeUs;
877 CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
878
879 sp<ABuffer> sei;
880 if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
881 return false;
882 }
883
Chong Zhangb86e68f2014-08-01 13:46:53 -0700884 bool trackAdded = false;
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700885
Chong Zhang862f8452014-06-26 19:55:23 -0700886 NALBitReader br(sei->data() + 1, sei->size() - 1);
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700887 // sei_message()
Chong Zhang862f8452014-06-26 19:55:23 -0700888 while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message()
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700889 uint32_t payload_type = 0;
890 size_t payload_size = 0;
891 uint8_t last_byte;
892
893 do {
894 last_byte = br.getBits(8);
895 payload_type += last_byte;
896 } while (last_byte == 0xFF);
897
898 do {
899 last_byte = br.getBits(8);
900 payload_size += last_byte;
901 } while (last_byte == 0xFF);
902
903 // sei_payload()
904 if (payload_type == 4) {
905 // user_data_registered_itu_t_t35()
906
907 // ATSC A/72: 6.4.2
908 uint8_t itu_t_t35_country_code = br.getBits(8);
909 uint16_t itu_t_t35_provider_code = br.getBits(16);
910 uint32_t user_identifier = br.getBits(32);
911 uint8_t user_data_type_code = br.getBits(8);
912
913 payload_size -= 1 + 2 + 4 + 1;
914
915 if (itu_t_t35_country_code == 0xB5
916 && itu_t_t35_provider_code == 0x0031
917 && user_identifier == 'GA94'
918 && user_data_type_code == 0x3) {
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700919 // MPEG_cc_data()
920 // ATSC A/53 Part 4: 6.2.3.1
921 br.skipBits(1); //process_em_data_flag
922 bool process_cc_data_flag = br.getBits(1);
923 br.skipBits(1); //additional_data_flag
924 size_t cc_count = br.getBits(5);
925 br.skipBits(8); // em_data;
926 payload_size -= 2;
927
928 if (process_cc_data_flag) {
929 AString out;
930
931 sp<ABuffer> ccBuf = new ABuffer(cc_count * sizeof(CCData));
932 ccBuf->setRange(0, 0);
933
934 for (size_t i = 0; i < cc_count; i++) {
935 uint8_t marker = br.getBits(5);
936 CHECK_EQ(marker, 0x1f);
937
938 bool cc_valid = br.getBits(1);
939 uint8_t cc_type = br.getBits(2);
940 // remove odd parity bit
941 uint8_t cc_data_1 = br.getBits(8) & 0x7f;
942 uint8_t cc_data_2 = br.getBits(8) & 0x7f;
943
944 if (cc_valid
945 && (cc_type == 0 || cc_type == 1)) {
946 CCData cc(cc_type, cc_data_1, cc_data_2);
947 if (!isNullPad(&cc)) {
Chong Zhangb86e68f2014-08-01 13:46:53 -0700948 size_t channel;
949 if (cc.getChannel(&channel) && getTrackIndex(channel) < 0) {
950 mTrackIndices[channel] = mFoundChannels.size();
951 mFoundChannels.push_back(channel);
952 trackAdded = true;
953 }
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700954 memcpy(ccBuf->data() + ccBuf->size(),
955 (void *)&cc, sizeof(cc));
956 ccBuf->setRange(0, ccBuf->size() + sizeof(CCData));
957 }
958 }
959 }
960 payload_size -= cc_count * 3;
961
962 mCCMap.add(timeUs, ccBuf);
963 break;
964 }
965 } else {
966 ALOGV("Malformed SEI payload type 4");
967 }
968 } else {
969 ALOGV("Unsupported SEI payload type %d", payload_type);
970 }
971
972 // skipping remaining bits of this payload
973 br.skipBits(payload_size * 8);
974 }
975
Chong Zhangb86e68f2014-08-01 13:46:53 -0700976 return trackAdded;
977}
978
979sp<ABuffer> NuPlayer::CCDecoder::filterCCBuf(
980 const sp<ABuffer> &ccBuf, size_t index) {
981 sp<ABuffer> filteredCCBuf = new ABuffer(ccBuf->size());
982 filteredCCBuf->setRange(0, 0);
983
984 size_t cc_count = ccBuf->size() / sizeof(CCData);
985 const CCData* cc_data = (const CCData*)ccBuf->data();
986 for (size_t i = 0; i < cc_count; ++i) {
987 size_t channel;
988 if (cc_data[i].getChannel(&channel)) {
989 mCurrentChannel = channel;
990 }
991 if (mCurrentChannel == mFoundChannels[index]) {
992 memcpy(filteredCCBuf->data() + filteredCCBuf->size(),
993 (void *)&cc_data[i], sizeof(CCData));
994 filteredCCBuf->setRange(0, filteredCCBuf->size() + sizeof(CCData));
995 }
996 }
997
998 return filteredCCBuf;
Chong Zhanga7fa1d92014-06-11 14:49:23 -0700999}
1000
1001void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
Chong Zhangb86e68f2014-08-01 13:46:53 -07001002 if (extractFromSEI(accessUnit)) {
Chong Zhanga7fa1d92014-06-11 14:49:23 -07001003 ALOGI("Found CEA-608 track");
1004 sp<AMessage> msg = mNotify->dup();
1005 msg->setInt32("what", kWhatTrackAdded);
1006 msg->post();
1007 }
1008 // TODO: extract CC from other sources
1009}
1010
1011void NuPlayer::CCDecoder::display(int64_t timeUs) {
Chong Zhangb86e68f2014-08-01 13:46:53 -07001012 if (!isTrackValid(mSelectedTrack)) {
1013 ALOGE("Could not find current track(index=%d)", mSelectedTrack);
1014 return;
1015 }
1016
Chong Zhanga7fa1d92014-06-11 14:49:23 -07001017 ssize_t index = mCCMap.indexOfKey(timeUs);
1018 if (index < 0) {
1019 ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
1020 return;
1021 }
1022
Chong Zhangb86e68f2014-08-01 13:46:53 -07001023 sp<ABuffer> ccBuf = filterCCBuf(mCCMap.valueAt(index), mSelectedTrack);
Chong Zhanga7fa1d92014-06-11 14:49:23 -07001024
1025 if (ccBuf->size() > 0) {
1026#if 0
1027 dumpBytePair(ccBuf);
1028#endif
1029
1030 ccBuf->meta()->setInt32("trackIndex", mSelectedTrack);
1031 ccBuf->meta()->setInt64("timeUs", timeUs);
1032 ccBuf->meta()->setInt64("durationUs", 0ll);
1033
1034 sp<AMessage> msg = mNotify->dup();
1035 msg->setInt32("what", kWhatClosedCaptionData);
1036 msg->setBuffer("buffer", ccBuf);
1037 msg->post();
1038 }
1039
1040 // remove all entries before timeUs
1041 mCCMap.removeItemsAt(0, index + 1);
1042}
1043
Chong Zhangb86e68f2014-08-01 13:46:53 -07001044void NuPlayer::CCDecoder::flush() {
1045 mCCMap.clear();
1046}
1047
Andreas Huberf9334412010-12-15 15:17:42 -08001048} // namespace android
1049