blob: fbc081f3aeceee224960f27eb5de6e60a7cbae3f [file] [log] [blame]
Linus Nilsson0da327a2020-01-31 16:22:18 -08001/*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17// #define LOG_NDEBUG 0
18#define LOG_TAG "VideoTrackTranscoder"
19
20#include <android-base/logging.h>
Linus Nilsson93591892020-08-03 18:56:55 -070021#include <media/NdkCommon.h>
Linus Nilsson0da327a2020-01-31 16:22:18 -080022#include <media/VideoTrackTranscoder.h>
Linus Nilssonb09aac22020-07-29 11:56:53 -070023#include <utils/AndroidThreads.h>
Linus Nilsson0da327a2020-01-31 16:22:18 -080024
Linus Nilsson7a127b22020-10-15 16:23:54 -070025using namespace AMediaFormatUtils;
26
Linus Nilsson0da327a2020-01-31 16:22:18 -080027namespace android {
28
29// Check that the codec sample flags have the expected NDK meaning.
30static_assert(SAMPLE_FLAG_CODEC_CONFIG == AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG,
31 "Sample flag mismatch: CODEC_CONFIG");
32static_assert(SAMPLE_FLAG_END_OF_STREAM == AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM,
33 "Sample flag mismatch: END_OF_STREAM");
34static_assert(SAMPLE_FLAG_PARTIAL_FRAME == AMEDIACODEC_BUFFER_FLAG_PARTIAL_FRAME,
35 "Sample flag mismatch: PARTIAL_FRAME");
36
Linus Nilssoncab39d82020-05-14 16:32:21 -070037// Color format defined by surface. (See MediaCodecInfo.CodecCapabilities#COLOR_FormatSurface.)
38static constexpr int32_t kColorFormatSurface = 0x7f000789;
39// Default key frame interval in seconds.
40static constexpr float kDefaultKeyFrameIntervalSeconds = 1.0f;
Linus Nilsson7a127b22020-10-15 16:23:54 -070041// Default codec operating rate.
42static constexpr int32_t kDefaultCodecOperatingRate = 240;
43// Default codec priority.
44static constexpr int32_t kDefaultCodecPriority = 1;
45// Default bitrate, in case source estimation fails.
46static constexpr int32_t kDefaultBitrateMbps = 10 * 1000 * 1000;
Linus Nilssonaf4a3212020-12-15 08:18:25 -080047// Default frame rate.
48static constexpr int32_t kDefaultFrameRate = 30;
Linus Nilssoncab39d82020-05-14 16:32:21 -070049
Linus Nilsson0da327a2020-01-31 16:22:18 -080050template <typename T>
51void VideoTrackTranscoder::BlockingQueue<T>::push(T const& value, bool front) {
52 {
Linus Nilsson93cf9132020-09-24 12:12:48 -070053 std::scoped_lock lock(mMutex);
54 if (mAborted) {
55 return;
56 }
57
Linus Nilsson0da327a2020-01-31 16:22:18 -080058 if (front) {
59 mQueue.push_front(value);
60 } else {
61 mQueue.push_back(value);
62 }
63 }
64 mCondition.notify_one();
65}
66
67template <typename T>
68T VideoTrackTranscoder::BlockingQueue<T>::pop() {
Linus Nilsson93cf9132020-09-24 12:12:48 -070069 std::unique_lock lock(mMutex);
Linus Nilsson0da327a2020-01-31 16:22:18 -080070 while (mQueue.empty()) {
71 mCondition.wait(lock);
72 }
73 T value = mQueue.front();
74 mQueue.pop_front();
75 return value;
76}
77
Linus Nilsson93cf9132020-09-24 12:12:48 -070078// Note: Do not call if another thread might waiting in pop.
79template <typename T>
80void VideoTrackTranscoder::BlockingQueue<T>::abort() {
81 std::scoped_lock lock(mMutex);
82 mAborted = true;
83 mQueue.clear();
84}
85
Linus Nilssone4716f22020-07-10 16:07:57 -070086// The CodecWrapper class is used to let AMediaCodec instances outlive the transcoder object itself
87// by giving the codec a weak pointer to the transcoder. Codecs wrapped in this object are kept
88// alive by the transcoder and the codec's outstanding buffers. Once the transcoder stops and all
89// output buffers have been released by downstream components the codec will also be released.
90class VideoTrackTranscoder::CodecWrapper {
91public:
92 CodecWrapper(AMediaCodec* codec, const std::weak_ptr<VideoTrackTranscoder>& transcoder)
93 : mCodec(codec), mTranscoder(transcoder), mCodecStarted(false) {}
94 ~CodecWrapper() {
95 if (mCodecStarted) {
96 AMediaCodec_stop(mCodec);
97 }
98 AMediaCodec_delete(mCodec);
99 }
100
101 AMediaCodec* getCodec() { return mCodec; }
102 std::shared_ptr<VideoTrackTranscoder> getTranscoder() const { return mTranscoder.lock(); };
103 void setStarted() { mCodecStarted = true; }
104
105private:
106 AMediaCodec* mCodec;
107 std::weak_ptr<VideoTrackTranscoder> mTranscoder;
108 bool mCodecStarted;
109};
110
Linus Nilsson0da327a2020-01-31 16:22:18 -0800111// Dispatch responses to codec callbacks onto the message queue.
112struct AsyncCodecCallbackDispatch {
113 static void onAsyncInputAvailable(AMediaCodec* codec, void* userdata, int32_t index) {
Linus Nilssone4716f22020-07-10 16:07:57 -0700114 VideoTrackTranscoder::CodecWrapper* wrapper =
115 static_cast<VideoTrackTranscoder::CodecWrapper*>(userdata);
116 if (auto transcoder = wrapper->getTranscoder()) {
117 if (codec == transcoder->mDecoder) {
118 transcoder->mCodecMessageQueue.push(
119 [transcoder, index] { transcoder->enqueueInputSample(index); });
120 }
Linus Nilsson0da327a2020-01-31 16:22:18 -0800121 }
122 }
123
124 static void onAsyncOutputAvailable(AMediaCodec* codec, void* userdata, int32_t index,
125 AMediaCodecBufferInfo* bufferInfoPtr) {
Linus Nilssone4716f22020-07-10 16:07:57 -0700126 VideoTrackTranscoder::CodecWrapper* wrapper =
127 static_cast<VideoTrackTranscoder::CodecWrapper*>(userdata);
Linus Nilsson0da327a2020-01-31 16:22:18 -0800128 AMediaCodecBufferInfo bufferInfo = *bufferInfoPtr;
Linus Nilssone4716f22020-07-10 16:07:57 -0700129 if (auto transcoder = wrapper->getTranscoder()) {
130 transcoder->mCodecMessageQueue.push([transcoder, index, codec, bufferInfo] {
131 if (codec == transcoder->mDecoder) {
132 transcoder->transferBuffer(index, bufferInfo);
133 } else if (codec == transcoder->mEncoder->getCodec()) {
134 transcoder->dequeueOutputSample(index, bufferInfo);
135 }
136 });
137 }
Linus Nilsson0da327a2020-01-31 16:22:18 -0800138 }
139
140 static void onAsyncFormatChanged(AMediaCodec* codec, void* userdata, AMediaFormat* format) {
Linus Nilssone4716f22020-07-10 16:07:57 -0700141 VideoTrackTranscoder::CodecWrapper* wrapper =
142 static_cast<VideoTrackTranscoder::CodecWrapper*>(userdata);
143 if (auto transcoder = wrapper->getTranscoder()) {
144 const char* kCodecName = (codec == transcoder->mDecoder ? "Decoder" : "Encoder");
145 LOG(DEBUG) << kCodecName << " format changed: " << AMediaFormat_toString(format);
146 if (codec == transcoder->mEncoder->getCodec()) {
147 transcoder->mCodecMessageQueue.push(
148 [transcoder, format] { transcoder->updateTrackFormat(format); });
149 }
Chong Zhanga2cc86b2020-06-17 16:56:49 -0700150 }
Linus Nilsson0da327a2020-01-31 16:22:18 -0800151 }
152
153 static void onAsyncError(AMediaCodec* codec, void* userdata, media_status_t error,
154 int32_t actionCode, const char* detail) {
155 LOG(ERROR) << "Error from codec " << codec << ", userdata " << userdata << ", error "
156 << error << ", action " << actionCode << ", detail " << detail;
Linus Nilssone4716f22020-07-10 16:07:57 -0700157 VideoTrackTranscoder::CodecWrapper* wrapper =
158 static_cast<VideoTrackTranscoder::CodecWrapper*>(userdata);
159 if (auto transcoder = wrapper->getTranscoder()) {
160 transcoder->mCodecMessageQueue.push(
Linus Nilssonfdb3e332020-09-18 17:11:41 -0700161 [transcoder, error] { transcoder->mStatus = error; }, true);
Linus Nilssone4716f22020-07-10 16:07:57 -0700162 }
Linus Nilsson0da327a2020-01-31 16:22:18 -0800163 }
164};
165
Linus Nilssone4716f22020-07-10 16:07:57 -0700166// static
167std::shared_ptr<VideoTrackTranscoder> VideoTrackTranscoder::create(
168 const std::weak_ptr<MediaTrackTranscoderCallback>& transcoderCallback) {
169 return std::shared_ptr<VideoTrackTranscoder>(new VideoTrackTranscoder(transcoderCallback));
170}
171
Linus Nilsson0da327a2020-01-31 16:22:18 -0800172VideoTrackTranscoder::~VideoTrackTranscoder() {
173 if (mDecoder != nullptr) {
174 AMediaCodec_delete(mDecoder);
175 }
176
Linus Nilsson0da327a2020-01-31 16:22:18 -0800177 if (mSurface != nullptr) {
178 ANativeWindow_release(mSurface);
179 }
180}
181
182// Creates and configures the codecs.
183media_status_t VideoTrackTranscoder::configureDestinationFormat(
184 const std::shared_ptr<AMediaFormat>& destinationFormat) {
185 media_status_t status = AMEDIA_OK;
186
187 if (destinationFormat == nullptr) {
Linus Nilssoncab39d82020-05-14 16:32:21 -0700188 LOG(ERROR) << "Destination format is null, use passthrough transcoder";
Linus Nilsson0da327a2020-01-31 16:22:18 -0800189 return AMEDIA_ERROR_INVALID_PARAMETER;
190 }
191
Linus Nilssoncab39d82020-05-14 16:32:21 -0700192 AMediaFormat* encoderFormat = AMediaFormat_new();
193 if (!encoderFormat || AMediaFormat_copy(encoderFormat, destinationFormat.get()) != AMEDIA_OK) {
194 LOG(ERROR) << "Unable to copy destination format";
195 return AMEDIA_ERROR_INVALID_PARAMETER;
196 }
197
Linus Nilsson800793f2020-07-31 16:16:38 -0700198 int32_t bitrate;
199 if (!AMediaFormat_getInt32(encoderFormat, AMEDIAFORMAT_KEY_BIT_RATE, &bitrate)) {
200 status = mMediaSampleReader->getEstimatedBitrateForTrack(mTrackIndex, &bitrate);
201 if (status != AMEDIA_OK) {
202 LOG(ERROR) << "Unable to estimate bitrate. Using default " << kDefaultBitrateMbps;
203 bitrate = kDefaultBitrateMbps;
204 }
205
206 LOG(INFO) << "Configuring bitrate " << bitrate;
207 AMediaFormat_setInt32(encoderFormat, AMEDIAFORMAT_KEY_BIT_RATE, bitrate);
208 }
209
Linus Nilsson7a127b22020-10-15 16:23:54 -0700210 SetDefaultFormatValueFloat(AMEDIAFORMAT_KEY_I_FRAME_INTERVAL, encoderFormat,
211 kDefaultKeyFrameIntervalSeconds);
212 SetDefaultFormatValueInt32(AMEDIAFORMAT_KEY_OPERATING_RATE, encoderFormat,
213 kDefaultCodecOperatingRate);
214 SetDefaultFormatValueInt32(AMEDIAFORMAT_KEY_PRIORITY, encoderFormat, kDefaultCodecPriority);
Linus Nilssonaf4a3212020-12-15 08:18:25 -0800215 SetDefaultFormatValueInt32(AMEDIAFORMAT_KEY_FRAME_RATE, encoderFormat, kDefaultFrameRate);
Linus Nilssoncab39d82020-05-14 16:32:21 -0700216 AMediaFormat_setInt32(encoderFormat, AMEDIAFORMAT_KEY_COLOR_FORMAT, kColorFormatSurface);
217
Chong Zhangd6e4aec2020-06-22 14:13:07 -0700218 // Always encode without rotation. The rotation degree will be transferred directly to
219 // MediaSampleWriter track format, and MediaSampleWriter will call AMediaMuxer_setOrientationHint.
220 AMediaFormat_setInt32(encoderFormat, AMEDIAFORMAT_KEY_ROTATION, 0);
221
Linus Nilssoncab39d82020-05-14 16:32:21 -0700222 mDestinationFormat = std::shared_ptr<AMediaFormat>(encoderFormat, &AMediaFormat_delete);
Linus Nilsson0da327a2020-01-31 16:22:18 -0800223
224 // Create and configure the encoder.
225 const char* destinationMime = nullptr;
226 bool ok = AMediaFormat_getString(mDestinationFormat.get(), AMEDIAFORMAT_KEY_MIME,
227 &destinationMime);
228 if (!ok) {
229 LOG(ERROR) << "Destination MIME type is required for transcoding.";
230 return AMEDIA_ERROR_INVALID_PARAMETER;
231 }
232
Linus Nilssonc6221db2020-03-18 14:46:22 -0700233 AMediaCodec* encoder = AMediaCodec_createEncoderByType(destinationMime);
234 if (encoder == nullptr) {
Linus Nilsson0da327a2020-01-31 16:22:18 -0800235 LOG(ERROR) << "Unable to create encoder for type " << destinationMime;
236 return AMEDIA_ERROR_UNSUPPORTED;
237 }
Linus Nilssone4716f22020-07-10 16:07:57 -0700238 mEncoder = std::make_shared<CodecWrapper>(encoder, shared_from_this());
Linus Nilsson0da327a2020-01-31 16:22:18 -0800239
Linus Nilssonaf4a3212020-12-15 08:18:25 -0800240 LOG(DEBUG) << "Configuring encoder with: " << AMediaFormat_toString(mDestinationFormat.get());
Linus Nilssone4716f22020-07-10 16:07:57 -0700241 status = AMediaCodec_configure(mEncoder->getCodec(), mDestinationFormat.get(),
242 NULL /* surface */, NULL /* crypto */,
243 AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
Linus Nilsson0da327a2020-01-31 16:22:18 -0800244 if (status != AMEDIA_OK) {
245 LOG(ERROR) << "Unable to configure video encoder: " << status;
246 return status;
247 }
248
Linus Nilssone4716f22020-07-10 16:07:57 -0700249 status = AMediaCodec_createInputSurface(mEncoder->getCodec(), &mSurface);
Linus Nilsson0da327a2020-01-31 16:22:18 -0800250 if (status != AMEDIA_OK) {
251 LOG(ERROR) << "Unable to create an encoder input surface: %d" << status;
252 return status;
253 }
254
255 // Create and configure the decoder.
256 const char* sourceMime = nullptr;
257 ok = AMediaFormat_getString(mSourceFormat.get(), AMEDIAFORMAT_KEY_MIME, &sourceMime);
258 if (!ok) {
259 LOG(ERROR) << "Source MIME type is required for transcoding.";
260 return AMEDIA_ERROR_INVALID_PARAMETER;
261 }
262
263 mDecoder = AMediaCodec_createDecoderByType(sourceMime);
264 if (mDecoder == nullptr) {
265 LOG(ERROR) << "Unable to create decoder for type " << sourceMime;
266 return AMEDIA_ERROR_UNSUPPORTED;
267 }
268
Linus Nilsson93591892020-08-03 18:56:55 -0700269 auto decoderFormat = std::shared_ptr<AMediaFormat>(AMediaFormat_new(), &AMediaFormat_delete);
270 if (!decoderFormat ||
271 AMediaFormat_copy(decoderFormat.get(), mSourceFormat.get()) != AMEDIA_OK) {
272 LOG(ERROR) << "Unable to copy source format";
273 return AMEDIA_ERROR_INVALID_PARAMETER;
274 }
275
276 // Prevent decoder from overwriting frames that the encoder has not yet consumed.
277 AMediaFormat_setInt32(decoderFormat.get(), TBD_AMEDIACODEC_PARAMETER_KEY_ALLOW_FRAME_DROP, 0);
278
Linus Nilsson16d772b2020-09-29 19:21:11 -0700279 // Copy over configurations that apply to both encoder and decoder.
Linus Nilsson7a127b22020-10-15 16:23:54 -0700280 static const EntryCopier kEncoderEntriesToCopy[] = {
Linus Nilsson16d772b2020-09-29 19:21:11 -0700281 ENTRY_COPIER2(AMEDIAFORMAT_KEY_OPERATING_RATE, Float, Int32),
282 ENTRY_COPIER(AMEDIAFORMAT_KEY_PRIORITY, Int32),
283 };
284 const size_t entryCount = sizeof(kEncoderEntriesToCopy) / sizeof(kEncoderEntriesToCopy[0]);
Linus Nilsson7a127b22020-10-15 16:23:54 -0700285 CopyFormatEntries(mDestinationFormat.get(), decoderFormat.get(), kEncoderEntriesToCopy,
286 entryCount);
Linus Nilsson16d772b2020-09-29 19:21:11 -0700287
Linus Nilssonaf4a3212020-12-15 08:18:25 -0800288 LOG(DEBUG) << "Configuring decoder with: " << AMediaFormat_toString(decoderFormat.get());
Linus Nilsson93591892020-08-03 18:56:55 -0700289 status = AMediaCodec_configure(mDecoder, decoderFormat.get(), mSurface, NULL /* crypto */,
Linus Nilsson0da327a2020-01-31 16:22:18 -0800290 0 /* flags */);
291 if (status != AMEDIA_OK) {
292 LOG(ERROR) << "Unable to configure video decoder: " << status;
293 return status;
294 }
295
296 // Configure codecs to run in async mode.
297 AMediaCodecOnAsyncNotifyCallback asyncCodecCallbacks = {
298 .onAsyncInputAvailable = AsyncCodecCallbackDispatch::onAsyncInputAvailable,
299 .onAsyncOutputAvailable = AsyncCodecCallbackDispatch::onAsyncOutputAvailable,
300 .onAsyncFormatChanged = AsyncCodecCallbackDispatch::onAsyncFormatChanged,
301 .onAsyncError = AsyncCodecCallbackDispatch::onAsyncError};
302
Linus Nilssone4716f22020-07-10 16:07:57 -0700303 // Note: The decoder does not need its own wrapper because its lifetime is tied to the
304 // transcoder. But the same callbacks are reused for decoder and encoder so we pass the encoder
305 // wrapper as userdata here but never read the codec from it in the callback.
306 status = AMediaCodec_setAsyncNotifyCallback(mDecoder, asyncCodecCallbacks, mEncoder.get());
Linus Nilsson0da327a2020-01-31 16:22:18 -0800307 if (status != AMEDIA_OK) {
308 LOG(ERROR) << "Unable to set decoder to async mode: " << status;
309 return status;
310 }
311
Linus Nilssone4716f22020-07-10 16:07:57 -0700312 status = AMediaCodec_setAsyncNotifyCallback(mEncoder->getCodec(), asyncCodecCallbacks,
313 mEncoder.get());
Linus Nilsson0da327a2020-01-31 16:22:18 -0800314 if (status != AMEDIA_OK) {
315 LOG(ERROR) << "Unable to set encoder to async mode: " << status;
316 return status;
317 }
318
319 return AMEDIA_OK;
320}
321
322void VideoTrackTranscoder::enqueueInputSample(int32_t bufferIndex) {
323 media_status_t status = AMEDIA_OK;
324
Linus Nilssonc6221db2020-03-18 14:46:22 -0700325 if (mEosFromSource) {
Linus Nilsson0da327a2020-01-31 16:22:18 -0800326 return;
327 }
328
329 status = mMediaSampleReader->getSampleInfoForTrack(mTrackIndex, &mSampleInfo);
330 if (status != AMEDIA_OK && status != AMEDIA_ERROR_END_OF_STREAM) {
331 LOG(ERROR) << "Error getting next sample info: " << status;
332 mStatus = status;
333 return;
334 }
335 const bool endOfStream = (status == AMEDIA_ERROR_END_OF_STREAM);
336
337 if (!endOfStream) {
338 size_t bufferSize = 0;
339 uint8_t* sourceBuffer = AMediaCodec_getInputBuffer(mDecoder, bufferIndex, &bufferSize);
340 if (sourceBuffer == nullptr) {
341 LOG(ERROR) << "Decoder returned a NULL input buffer.";
342 mStatus = AMEDIA_ERROR_UNKNOWN;
343 return;
344 } else if (bufferSize < mSampleInfo.size) {
345 LOG(ERROR) << "Decoder returned an input buffer that is smaller than the sample.";
346 mStatus = AMEDIA_ERROR_UNKNOWN;
347 return;
348 }
349
350 status = mMediaSampleReader->readSampleDataForTrack(mTrackIndex, sourceBuffer,
351 mSampleInfo.size);
352 if (status != AMEDIA_OK) {
353 LOG(ERROR) << "Unable to read next sample data. Aborting transcode.";
354 mStatus = status;
355 return;
356 }
Linus Nilsson0da327a2020-01-31 16:22:18 -0800357 } else {
358 LOG(DEBUG) << "EOS from source.";
Linus Nilssonc6221db2020-03-18 14:46:22 -0700359 mEosFromSource = true;
Linus Nilsson0da327a2020-01-31 16:22:18 -0800360 }
361
362 status = AMediaCodec_queueInputBuffer(mDecoder, bufferIndex, 0, mSampleInfo.size,
363 mSampleInfo.presentationTimeUs, mSampleInfo.flags);
364 if (status != AMEDIA_OK) {
365 LOG(ERROR) << "Unable to queue input buffer for decode: " << status;
366 mStatus = status;
367 return;
368 }
369}
370
371void VideoTrackTranscoder::transferBuffer(int32_t bufferIndex, AMediaCodecBufferInfo bufferInfo) {
372 if (bufferIndex >= 0) {
373 bool needsRender = bufferInfo.size > 0;
374 AMediaCodec_releaseOutputBuffer(mDecoder, bufferIndex, needsRender);
375 }
376
377 if (bufferInfo.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
378 LOG(DEBUG) << "EOS from decoder.";
Linus Nilssone4716f22020-07-10 16:07:57 -0700379 media_status_t status = AMediaCodec_signalEndOfInputStream(mEncoder->getCodec());
Linus Nilsson0da327a2020-01-31 16:22:18 -0800380 if (status != AMEDIA_OK) {
381 LOG(ERROR) << "SignalEOS on encoder returned error: " << status;
382 mStatus = status;
383 }
384 }
385}
386
387void VideoTrackTranscoder::dequeueOutputSample(int32_t bufferIndex,
388 AMediaCodecBufferInfo bufferInfo) {
389 if (bufferIndex >= 0) {
390 size_t sampleSize = 0;
Linus Nilssone4716f22020-07-10 16:07:57 -0700391 uint8_t* buffer =
392 AMediaCodec_getOutputBuffer(mEncoder->getCodec(), bufferIndex, &sampleSize);
Linus Nilssonc6221db2020-03-18 14:46:22 -0700393
Linus Nilssone4716f22020-07-10 16:07:57 -0700394 MediaSample::OnSampleReleasedCallback bufferReleaseCallback =
395 [encoder = mEncoder](MediaSample* sample) {
396 AMediaCodec_releaseOutputBuffer(encoder->getCodec(), sample->bufferId,
397 false /* render */);
398 };
Linus Nilsson0da327a2020-01-31 16:22:18 -0800399
400 std::shared_ptr<MediaSample> sample = MediaSample::createWithReleaseCallback(
Linus Nilssonc6221db2020-03-18 14:46:22 -0700401 buffer, bufferInfo.offset, bufferIndex, bufferReleaseCallback);
Linus Nilsson0da327a2020-01-31 16:22:18 -0800402 sample->info.size = bufferInfo.size;
403 sample->info.flags = bufferInfo.flags;
404 sample->info.presentationTimeUs = bufferInfo.presentationTimeUs;
405
Linus Nilssonc31d2492020-09-23 12:30:00 -0700406 onOutputSampleAvailable(sample);
Linus Nilssonfdb3e332020-09-18 17:11:41 -0700407
408 mLastSampleWasSync = sample->info.flags & SAMPLE_FLAG_SYNC_SAMPLE;
Linus Nilsson0da327a2020-01-31 16:22:18 -0800409 } else if (bufferIndex == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
Linus Nilssone4716f22020-07-10 16:07:57 -0700410 AMediaFormat* newFormat = AMediaCodec_getOutputFormat(mEncoder->getCodec());
Linus Nilsson0da327a2020-01-31 16:22:18 -0800411 LOG(DEBUG) << "Encoder output format changed: " << AMediaFormat_toString(newFormat);
412 }
413
414 if (bufferInfo.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
415 LOG(DEBUG) << "EOS from encoder.";
Linus Nilssonc6221db2020-03-18 14:46:22 -0700416 mEosFromEncoder = true;
Linus Nilsson0da327a2020-01-31 16:22:18 -0800417 }
418}
419
Chong Zhanga2cc86b2020-06-17 16:56:49 -0700420void VideoTrackTranscoder::updateTrackFormat(AMediaFormat* outputFormat) {
421 if (mActualOutputFormat != nullptr) {
422 LOG(WARNING) << "Ignoring duplicate format change.";
423 return;
424 }
425
426 AMediaFormat* formatCopy = AMediaFormat_new();
427 if (!formatCopy || AMediaFormat_copy(formatCopy, outputFormat) != AMEDIA_OK) {
428 LOG(ERROR) << "Unable to copy outputFormat";
429 AMediaFormat_delete(formatCopy);
430 mStatus = AMEDIA_ERROR_INVALID_PARAMETER;
431 return;
432 }
433
434 // Generate the actual track format for muxer based on the encoder output format,
435 // since many vital information comes in the encoder format (eg. CSD).
436 // Transfer necessary fields from the user-configured track format (derived from
437 // source track format and user transcoding request) where needed.
438
439 // Transfer SAR settings:
440 // If mDestinationFormat has SAR set, it means the original source has SAR specified
441 // at container level. This is supposed to override any SAR settings in the bitstream,
442 // thus should always be transferred to the container of the transcoded file.
443 int32_t sarWidth, sarHeight;
Chong Zhangd6e4aec2020-06-22 14:13:07 -0700444 if (AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_SAR_WIDTH, &sarWidth) &&
Chong Zhanga2cc86b2020-06-17 16:56:49 -0700445 (sarWidth > 0) &&
Chong Zhangd6e4aec2020-06-22 14:13:07 -0700446 AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_SAR_HEIGHT, &sarHeight) &&
Chong Zhanga2cc86b2020-06-17 16:56:49 -0700447 (sarHeight > 0)) {
448 AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_SAR_WIDTH, sarWidth);
449 AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_SAR_HEIGHT, sarHeight);
450 }
451 // Transfer DAR settings.
452 int32_t displayWidth, displayHeight;
Chong Zhangd6e4aec2020-06-22 14:13:07 -0700453 if (AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_DISPLAY_WIDTH, &displayWidth) &&
Chong Zhanga2cc86b2020-06-17 16:56:49 -0700454 (displayWidth > 0) &&
Chong Zhangd6e4aec2020-06-22 14:13:07 -0700455 AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_DISPLAY_HEIGHT,
Chong Zhanga2cc86b2020-06-17 16:56:49 -0700456 &displayHeight) &&
457 (displayHeight > 0)) {
458 AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_DISPLAY_WIDTH, displayWidth);
459 AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_DISPLAY_HEIGHT, displayHeight);
460 }
461
Chong Zhangd6e4aec2020-06-22 14:13:07 -0700462 // Transfer rotation settings.
463 // Note that muxer itself doesn't take rotation from the track format. It requires
464 // AMediaMuxer_setOrientationHint to set the rotation. Here we pass the rotation to
465 // MediaSampleWriter using the track format. MediaSampleWriter will then call
466 // AMediaMuxer_setOrientationHint as needed.
467 int32_t rotation;
468 if (AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_ROTATION, &rotation) &&
469 (rotation != 0)) {
470 AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_ROTATION, rotation);
471 }
472
Linus Nilsson42a971b2020-07-01 16:41:11 -0700473 // Transfer track duration.
474 // Preserve the source track duration by sending it to MediaSampleWriter.
475 int64_t durationUs;
476 if (AMediaFormat_getInt64(mSourceFormat.get(), AMEDIAFORMAT_KEY_DURATION, &durationUs) &&
477 durationUs > 0) {
478 AMediaFormat_setInt64(formatCopy, AMEDIAFORMAT_KEY_DURATION, durationUs);
479 }
480
Chong Zhanga2cc86b2020-06-17 16:56:49 -0700481 // TODO: transfer other fields as required.
482
483 mActualOutputFormat = std::shared_ptr<AMediaFormat>(formatCopy, &AMediaFormat_delete);
484
485 notifyTrackFormatAvailable();
486}
487
Linus Nilssonfdb3e332020-09-18 17:11:41 -0700488media_status_t VideoTrackTranscoder::runTranscodeLoop(bool* stopped) {
Linus Nilssonb09aac22020-07-29 11:56:53 -0700489 androidSetThreadPriority(0 /* tid (0 = current) */, ANDROID_PRIORITY_VIDEO);
490
Chong Zhangb55c5452020-06-26 14:32:12 -0700491 // Push start decoder and encoder as two messages, so that these are subject to the
Chong Zhangbc062482020-10-14 16:43:53 -0700492 // stop request as well. If the session is cancelled (or paused) immediately after start,
Chong Zhangb55c5452020-06-26 14:32:12 -0700493 // we don't need to waste time start then stop the codecs.
494 mCodecMessageQueue.push([this] {
495 media_status_t status = AMediaCodec_start(mDecoder);
496 if (status != AMEDIA_OK) {
497 LOG(ERROR) << "Unable to start video decoder: " << status;
498 mStatus = status;
499 }
500 });
Linus Nilsson0da327a2020-01-31 16:22:18 -0800501
Chong Zhangb55c5452020-06-26 14:32:12 -0700502 mCodecMessageQueue.push([this] {
Linus Nilssone4716f22020-07-10 16:07:57 -0700503 media_status_t status = AMediaCodec_start(mEncoder->getCodec());
Chong Zhangb55c5452020-06-26 14:32:12 -0700504 if (status != AMEDIA_OK) {
505 LOG(ERROR) << "Unable to start video encoder: " << status;
506 mStatus = status;
507 }
Linus Nilssone4716f22020-07-10 16:07:57 -0700508 mEncoder->setStarted();
Chong Zhangb55c5452020-06-26 14:32:12 -0700509 });
Linus Nilsson0da327a2020-01-31 16:22:18 -0800510
511 // Process codec events until EOS is reached, transcoding is stopped or an error occurs.
Linus Nilssonfdb3e332020-09-18 17:11:41 -0700512 while (mStopRequest != STOP_NOW && !mEosFromEncoder && mStatus == AMEDIA_OK) {
Linus Nilsson0da327a2020-01-31 16:22:18 -0800513 std::function<void()> message = mCodecMessageQueue.pop();
514 message();
Linus Nilssonfdb3e332020-09-18 17:11:41 -0700515
516 if (mStopRequest == STOP_ON_SYNC && mLastSampleWasSync) {
517 break;
518 }
Linus Nilsson0da327a2020-01-31 16:22:18 -0800519 }
520
Linus Nilsson93cf9132020-09-24 12:12:48 -0700521 mCodecMessageQueue.abort();
522 AMediaCodec_stop(mDecoder);
523
Linus Nilssonfdb3e332020-09-18 17:11:41 -0700524 // Signal if transcoding was stopped before it finished.
525 if (mStopRequest != NONE && !mEosFromEncoder && mStatus == AMEDIA_OK) {
526 *stopped = true;
Linus Nilsson0da327a2020-01-31 16:22:18 -0800527 }
528
Linus Nilsson0da327a2020-01-31 16:22:18 -0800529 return mStatus;
530}
531
532void VideoTrackTranscoder::abortTranscodeLoop() {
Linus Nilssonfdb3e332020-09-18 17:11:41 -0700533 if (mStopRequest == STOP_NOW) {
534 // Wake up transcoder thread.
535 mCodecMessageQueue.push([] {}, true /* front */);
536 }
Linus Nilsson0da327a2020-01-31 16:22:18 -0800537}
538
Linus Nilssoncab39d82020-05-14 16:32:21 -0700539std::shared_ptr<AMediaFormat> VideoTrackTranscoder::getOutputFormat() const {
Chong Zhanga2cc86b2020-06-17 16:56:49 -0700540 return mActualOutputFormat;
Linus Nilssoncab39d82020-05-14 16:32:21 -0700541}
542
Linus Nilsson0da327a2020-01-31 16:22:18 -0800543} // namespace android