Transcoder: Improve video transcoding with audio passthrough performance
MediaSampleWriter was stalling the video transcoding when processing
chunks of audio samples at a time. Since the underlying muxer
buffers ~1 sec worth of data and interleaves tracks anyway the
sample writer now prioritizes pulling samples from the track that is
farthest behind while making sure the tracks are kept reasonably
in sync, i.e. max ~1 second divergence.
This change also raises the priority of the video transcoding thread
since that will be the bottleneck when audio is pased through.
Test: Unit tests
Fixes: 160268606
Change-Id: I004583b2a31a57882ea543072be321f9f1347508
diff --git a/media/libmediatranscoding/transcoder/tests/MediaSampleQueueTests.cpp b/media/libmediatranscoding/transcoder/tests/MediaSampleQueueTests.cpp
index 2046ca0..6357e4d 100644
--- a/media/libmediatranscoding/transcoder/tests/MediaSampleQueueTests.cpp
+++ b/media/libmediatranscoding/transcoder/tests/MediaSampleQueueTests.cpp
@@ -46,10 +46,12 @@
static constexpr int kNumSamples = 4;
MediaSampleQueue sampleQueue;
+ EXPECT_TRUE(sampleQueue.isEmpty());
// Enqueue loop.
for (int i = 0; i < kNumSamples; ++i) {
sampleQueue.enqueue(newSample(i));
+ EXPECT_FALSE(sampleQueue.isEmpty());
}
// Dequeue loop.
@@ -60,6 +62,7 @@
EXPECT_EQ(sample->bufferId, i);
EXPECT_FALSE(aborted);
}
+ EXPECT_TRUE(sampleQueue.isEmpty());
}
TEST_F(MediaSampleQueueTests, TestInterleavedDequeueOrder) {
@@ -71,12 +74,14 @@
// Enqueue and dequeue.
for (int i = 0; i < kNumSamples; ++i) {
sampleQueue.enqueue(newSample(i));
+ EXPECT_FALSE(sampleQueue.isEmpty());
std::shared_ptr<MediaSample> sample;
bool aborted = sampleQueue.dequeue(&sample);
EXPECT_NE(sample, nullptr);
EXPECT_EQ(sample->bufferId, i);
EXPECT_FALSE(aborted);
+ EXPECT_TRUE(sampleQueue.isEmpty());
}
}
@@ -98,6 +103,7 @@
EXPECT_NE(sample, nullptr);
EXPECT_EQ(sample->bufferId, 1);
EXPECT_FALSE(aborted);
+ EXPECT_TRUE(sampleQueue.isEmpty());
enqueueThread.join();
}
@@ -160,7 +166,9 @@
EXPECT_FALSE(bufferReleased[i]);
}
+ EXPECT_FALSE(sampleQueue.isEmpty());
sampleQueue.abort();
+ EXPECT_TRUE(sampleQueue.isEmpty());
for (int i = 0; i < kNumSamples; ++i) {
EXPECT_TRUE(bufferReleased[i]);
diff --git a/media/libmediatranscoding/transcoder/tests/MediaSampleWriterTests.cpp b/media/libmediatranscoding/transcoder/tests/MediaSampleWriterTests.cpp
index c82ec28..c6e35c4 100644
--- a/media/libmediatranscoding/transcoder/tests/MediaSampleWriterTests.cpp
+++ b/media/libmediatranscoding/transcoder/tests/MediaSampleWriterTests.cpp
@@ -78,7 +78,22 @@
return {.type = Event::WriteSample, .trackIndex = trackIndex, .data = data, .info = *info};
}
- const Event& popEvent() {
+ static Event WriteSampleWithPts(size_t trackIndex, int64_t pts) {
+ return {.type = Event::WriteSample, .trackIndex = trackIndex, .info = {0, 0, pts, 0}};
+ }
+
+ void pushEvent(const Event& e) {
+ std::unique_lock<std::mutex> lock(mMutex);
+ mEventQueue.push_back(e);
+ mCondition.notify_one();
+ }
+
+ const Event& popEvent(bool wait = false) {
+ std::unique_lock<std::mutex> lock(mMutex);
+ while (wait && mEventQueue.empty()) {
+ mCondition.wait_for(lock, std::chrono::milliseconds(200));
+ }
+
if (mEventQueue.empty()) {
mPoppedEvent = NoEvent;
} else {
@@ -92,6 +107,8 @@
Event mPoppedEvent;
std::list<Event> mEventQueue;
ssize_t mTrackCount = 0;
+ std::mutex mMutex;
+ std::condition_variable mCondition;
};
bool operator==(const AMediaCodecBufferInfo& lhs, const AMediaCodecBufferInfo& rhs) {
@@ -250,6 +267,10 @@
return newSample(ptsUs, 0, sampleId, sampleId, reinterpret_cast<const uint8_t*>(sampleId));
}
+ static std::shared_ptr<MediaSample> newSampleWithPtsOnly(int64_t ptsUs) {
+ return newSample(ptsUs, 0, 0, 0, nullptr);
+ }
+
void SetUp() override {
LOG(DEBUG) << "MediaSampleWriterTests set up";
mTestMuxer = std::make_shared<TestMuxer>();
@@ -345,10 +366,9 @@
}
TEST_F(MediaSampleWriterTests, TestProgressUpdate) {
- static constexpr uint32_t kSegmentLengthUs = 1;
const TestMediaSource& mediaSource = getMediaSource();
- MediaSampleWriter writer{kSegmentLengthUs};
+ MediaSampleWriter writer{};
EXPECT_TRUE(writer.init(mTestMuxer, mTestCallbacks));
std::shared_ptr<AMediaFormat> videoFormat =
@@ -370,9 +390,7 @@
}
TEST_F(MediaSampleWriterTests, TestInterleaving) {
- static constexpr uint32_t kSegmentLength = MediaSampleWriter::kDefaultTrackSegmentLengthUs;
-
- MediaSampleWriter writer{kSegmentLength};
+ MediaSampleWriter writer{};
EXPECT_TRUE(writer.init(mTestMuxer, mTestCallbacks));
// Use two tracks for this test.
@@ -398,18 +416,19 @@
};
addSampleToTrackWithPts(0, 0);
- addSampleToTrackWithPts(0, kSegmentLength / 2);
- addSampleToTrackWithPts(0, kSegmentLength); // Track 0 reached 1st segment end
+ addSampleToTrackWithPts(1, 4);
- addSampleToTrackWithPts(1, 0);
- addSampleToTrackWithPts(1, kSegmentLength); // Track 1 reached 1st segment end
+ addSampleToTrackWithPts(0, 1);
+ addSampleToTrackWithPts(0, 2);
+ addSampleToTrackWithPts(0, 3);
+ addSampleToTrackWithPts(0, 10);
- addSampleToTrackWithPts(0, kSegmentLength * 2); // Track 0 reached 2nd segment end
+ addSampleToTrackWithPts(1, 5);
+ addSampleToTrackWithPts(1, 6);
+ addSampleToTrackWithPts(1, 11);
- addSampleToTrackWithPts(1, kSegmentLength + 1);
- addSampleToTrackWithPts(1, kSegmentLength * 2); // Track 1 reached 2nd segment end
-
- addSampleToTrackWithPts(0, kSegmentLength * 2 + 1);
+ addSampleToTrackWithPts(0, 12);
+ addSampleToTrackWithPts(1, 13);
for (int trackIndex = 0; trackIndex < kNumTracks; ++trackIndex) {
sampleQueues[trackIndex]->enqueue(newSampleEos());
@@ -443,7 +462,10 @@
int64_t duration = 0;
AMediaFormat_getInt64(trackFormat.get(), AMEDIAFORMAT_KEY_DURATION, &duration);
- const AMediaCodecBufferInfo info = {0, 0, duration, AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM};
+ // EOS timestamp = first sample timestamp + duration.
+ const int64_t endTime = duration + (trackIndex == 1 ? 4 : 0);
+ const AMediaCodecBufferInfo info = {0, 0, endTime, AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM};
+
EXPECT_EQ(mTestMuxer->popEvent(), TestMuxer::WriteSample(trackIndex, nullptr, &info));
}
@@ -452,6 +474,124 @@
EXPECT_TRUE(mTestCallbacks->hasFinished());
}
+TEST_F(MediaSampleWriterTests, TestMaxDivergence) {
+ static constexpr uint32_t kMaxDivergenceUs = 10;
+
+ MediaSampleWriter writer{kMaxDivergenceUs};
+ EXPECT_TRUE(writer.init(mTestMuxer, mTestCallbacks));
+
+ // Use two tracks for this test.
+ static constexpr int kNumTracks = 2;
+ std::shared_ptr<MediaSampleQueue> sampleQueues[kNumTracks];
+ const TestMediaSource& mediaSource = getMediaSource();
+
+ for (int trackIdx = 0; trackIdx < kNumTracks; ++trackIdx) {
+ sampleQueues[trackIdx] = std::make_shared<MediaSampleQueue>();
+
+ auto trackFormat = mediaSource.mTrackFormats[trackIdx % mediaSource.mTrackCount];
+ EXPECT_TRUE(writer.addTrack(sampleQueues[trackIdx], trackFormat));
+ EXPECT_EQ(mTestMuxer->popEvent(), TestMuxer::AddTrack(trackFormat.get()));
+ }
+
+ ASSERT_TRUE(writer.start());
+ EXPECT_EQ(mTestMuxer->popEvent(true), TestMuxer::Start());
+
+ // The first samples of each track can be written in any order since the writer does not have
+ // any previous timestamps to compare.
+ sampleQueues[0]->enqueue(newSampleWithPtsOnly(0));
+ sampleQueues[1]->enqueue(newSampleWithPtsOnly(1));
+ mTestMuxer->popEvent(true);
+ mTestMuxer->popEvent(true);
+
+ // The writer will now be waiting on track 0 since it has the lowest previous timestamp.
+ sampleQueues[0]->enqueue(newSampleWithPtsOnly(kMaxDivergenceUs + 1));
+ sampleQueues[0]->enqueue(newSampleWithPtsOnly(kMaxDivergenceUs + 2));
+
+ // The writer should dequeue the first sample above but not the second since track 0 now is too
+ // far ahead. Instead it should wait for track 1.
+ EXPECT_EQ(mTestMuxer->popEvent(true), TestMuxer::WriteSampleWithPts(0, kMaxDivergenceUs + 1));
+
+ // Enqueue a sample from track 1 that puts it within acceptable divergence range again. The
+ // writer should dequeue that sample and then go back to track 0 since track 1 is empty.
+ sampleQueues[1]->enqueue(newSampleWithPtsOnly(kMaxDivergenceUs));
+ EXPECT_EQ(mTestMuxer->popEvent(true), TestMuxer::WriteSampleWithPts(1, kMaxDivergenceUs));
+ EXPECT_EQ(mTestMuxer->popEvent(true), TestMuxer::WriteSampleWithPts(0, kMaxDivergenceUs + 2));
+
+ // Both tracks are now empty so the writer should wait for track 1 which is farthest behind.
+ sampleQueues[1]->enqueue(newSampleWithPtsOnly(kMaxDivergenceUs + 3));
+ EXPECT_EQ(mTestMuxer->popEvent(true), TestMuxer::WriteSampleWithPts(1, kMaxDivergenceUs + 3));
+
+ for (int trackIndex = 0; trackIndex < kNumTracks; ++trackIndex) {
+ sampleQueues[trackIndex]->enqueue(newSampleEos());
+ }
+
+ // Wait for writer to complete.
+ mTestCallbacks->waitForWritingFinished();
+
+ // Verify EOS samples.
+ for (int trackIndex = 0; trackIndex < kNumTracks; ++trackIndex) {
+ auto trackFormat = mediaSource.mTrackFormats[trackIndex % mediaSource.mTrackCount];
+ int64_t duration = 0;
+ AMediaFormat_getInt64(trackFormat.get(), AMEDIAFORMAT_KEY_DURATION, &duration);
+
+ // EOS timestamp = first sample timestamp + duration.
+ const int64_t endTime = duration + (trackIndex == 1 ? 1 : 0);
+ const AMediaCodecBufferInfo info = {0, 0, endTime, AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM};
+ EXPECT_EQ(mTestMuxer->popEvent(), TestMuxer::WriteSample(trackIndex, nullptr, &info));
+ }
+
+ EXPECT_EQ(mTestMuxer->popEvent(), TestMuxer::Stop());
+ EXPECT_TRUE(writer.stop());
+ EXPECT_TRUE(mTestCallbacks->hasFinished());
+}
+
+TEST_F(MediaSampleWriterTests, TestTimestampDivergenceOverflow) {
+ auto testCallbacks = std::make_shared<TestCallbacks>(false /* expectSuccess */);
+ MediaSampleWriter writer{};
+ EXPECT_TRUE(writer.init(mTestMuxer, testCallbacks));
+
+ // Use two tracks for this test.
+ static constexpr int kNumTracks = 2;
+ std::shared_ptr<MediaSampleQueue> sampleQueues[kNumTracks];
+ const TestMediaSource& mediaSource = getMediaSource();
+
+ for (int trackIdx = 0; trackIdx < kNumTracks; ++trackIdx) {
+ sampleQueues[trackIdx] = std::make_shared<MediaSampleQueue>();
+
+ auto trackFormat = mediaSource.mTrackFormats[trackIdx % mediaSource.mTrackCount];
+ EXPECT_TRUE(writer.addTrack(sampleQueues[trackIdx], trackFormat));
+ EXPECT_EQ(mTestMuxer->popEvent(), TestMuxer::AddTrack(trackFormat.get()));
+ }
+
+ // Prime track 0 with lower end of INT64 range, and track 1 with positive timestamps making the
+ // difference larger than INT64_MAX.
+ sampleQueues[0]->enqueue(newSampleWithPtsOnly(INT64_MIN + 1));
+ sampleQueues[1]->enqueue(newSampleWithPtsOnly(1000));
+ sampleQueues[1]->enqueue(newSampleWithPtsOnly(1001));
+
+ ASSERT_TRUE(writer.start());
+ EXPECT_EQ(mTestMuxer->popEvent(true), TestMuxer::Start());
+
+ // The first sample of each track can be pulled in any order.
+ mTestMuxer->popEvent(true);
+ mTestMuxer->popEvent(true);
+
+ // Wait to make sure the writer compares track 0 empty against track 1 non-empty. The writer
+ // should handle the large timestamp differences and chose to wait for track 0 even though
+ // track 1 has a sample ready.
+ std::this_thread::sleep_for(std::chrono::milliseconds(20));
+
+ sampleQueues[0]->enqueue(newSampleWithPtsOnly(INT64_MIN + 2));
+ sampleQueues[0]->enqueue(newSampleWithPtsOnly(1000)); // <-- Close the gap between the tracks.
+ EXPECT_EQ(mTestMuxer->popEvent(true), TestMuxer::WriteSampleWithPts(0, INT64_MIN + 2));
+ EXPECT_EQ(mTestMuxer->popEvent(true), TestMuxer::WriteSampleWithPts(0, 1000));
+ EXPECT_EQ(mTestMuxer->popEvent(true), TestMuxer::WriteSampleWithPts(1, 1001));
+
+ EXPECT_TRUE(writer.stop());
+ EXPECT_EQ(mTestMuxer->popEvent(), TestMuxer::Stop());
+ EXPECT_TRUE(testCallbacks->hasFinished());
+}
+
TEST_F(MediaSampleWriterTests, TestAbortInputQueue) {
MediaSampleWriter writer{};
std::shared_ptr<TestCallbacks> callbacks =