Transcoder: Improve AV transcoding speed by enforcing sequential sample access.
MediaSampleReader was bottlenecking the transcoding pipeline due to
non-sequential sample access. This commit adds an option to the sample
reader to enforce sequential sample access by blocking reads until
the underlying extractor advances to that specific track.
Follow-up: b/165374867 Make MediaSampleWriter robust against buffering track transcoders
Fixes: 160268606
Test: Transcoder unit tests, and benchmark tests.
Change-Id: Id2a363d06df927ea3e547462c52803594e0511e1
diff --git a/media/libmediatranscoding/transcoder/MediaSampleReaderNDK.cpp b/media/libmediatranscoding/transcoder/MediaSampleReaderNDK.cpp
index 6a00a10..53d567e 100644
--- a/media/libmediatranscoding/transcoder/MediaSampleReaderNDK.cpp
+++ b/media/libmediatranscoding/transcoder/MediaSampleReaderNDK.cpp
@@ -22,7 +22,6 @@
#include <algorithm>
#include <cmath>
-#include <vector>
namespace android {
@@ -47,12 +46,6 @@
}
auto sampleReader = std::shared_ptr<MediaSampleReaderNDK>(new MediaSampleReaderNDK(extractor));
- status = sampleReader->init();
- if (status != AMEDIA_OK) {
- LOG(ERROR) << "MediaSampleReaderNDK::init returned error: " << status;
- return nullptr;
- }
-
return sampleReader;
}
@@ -60,39 +53,42 @@
: mExtractor(extractor), mTrackCount(AMediaExtractor_getTrackCount(mExtractor)) {
if (mTrackCount > 0) {
mTrackCursors.resize(mTrackCount);
- mTrackCursors.resize(mTrackCount);
}
}
-media_status_t MediaSampleReaderNDK::init() {
- for (size_t trackIndex = 0; trackIndex < mTrackCount; trackIndex++) {
- media_status_t status = AMediaExtractor_selectTrack(mExtractor, trackIndex);
- if (status != AMEDIA_OK) {
- LOG(ERROR) << "AMediaExtractor_selectTrack returned error: " << status;
- return status;
- }
- }
-
- mExtractorTrackIndex = AMediaExtractor_getSampleTrackIndex(mExtractor);
- if (mExtractorTrackIndex >= 0) {
- mTrackCursors[mExtractorTrackIndex].current.set(mExtractorSampleIndex,
- AMediaExtractor_getSampleTime(mExtractor));
- } else if (mTrackCount > 0) {
- // The extractor track index is only allowed to be invalid if there are no tracks.
- LOG(ERROR) << "Track index " << mExtractorTrackIndex << " is invalid for track count "
- << mTrackCount;
- return AMEDIA_ERROR_MALFORMED;
- }
-
- return AMEDIA_OK;
-}
-
MediaSampleReaderNDK::~MediaSampleReaderNDK() {
if (mExtractor != nullptr) {
AMediaExtractor_delete(mExtractor);
}
}
+void MediaSampleReaderNDK::advanceTrack_l(int trackIndex) {
+ if (!mEnforceSequentialAccess) {
+ // Note: Positioning the extractor before advancing the track is needed for two reasons:
+ // 1. To enable multiple advances without explicitly letting the extractor catch up.
+ // 2. To prevent the extractor from being farther than "next".
+ (void)moveToTrack_l(trackIndex);
+ }
+
+ SampleCursor& cursor = mTrackCursors[trackIndex];
+ cursor.previous = cursor.current;
+ cursor.current = cursor.next;
+ cursor.next.reset();
+
+ if (mEnforceSequentialAccess && trackIndex == mExtractorTrackIndex) {
+ while (advanceExtractor_l()) {
+ SampleCursor& cursor = mTrackCursors[mExtractorTrackIndex];
+ if (cursor.current.isSet && cursor.current.index == mExtractorSampleIndex) {
+ if (mExtractorTrackIndex != trackIndex) {
+ mTrackSignals[mExtractorTrackIndex].notify_all();
+ }
+ break;
+ }
+ }
+ }
+ return;
+}
+
bool MediaSampleReaderNDK::advanceExtractor_l() {
// Reset the "next" sample time whenever the extractor advances past a sample that is current,
// to ensure that "next" is appropriately updated when the extractor advances over the next
@@ -103,6 +99,10 @@
}
if (!AMediaExtractor_advance(mExtractor)) {
+ mEosReached = true;
+ for (auto it = mTrackSignals.begin(); it != mTrackSignals.end(); ++it) {
+ it->second.notify_all();
+ }
return false;
}
@@ -117,6 +117,7 @@
cursor.next.set(mExtractorSampleIndex, AMediaExtractor_getSampleTime(mExtractor));
}
}
+
return true;
}
@@ -150,38 +151,15 @@
return AMEDIA_OK;
}
-void MediaSampleReaderNDK::advanceTrack(int trackIndex) {
- std::scoped_lock lock(mExtractorMutex);
-
- if (trackIndex < 0 || trackIndex >= mTrackCount) {
- LOG(ERROR) << "Invalid trackIndex " << trackIndex << " for trackCount " << mTrackCount;
- return;
- }
-
- // Note: Positioning the extractor before advancing the track is needed for two reasons:
- // 1. To enable multiple advances without explicitly letting the extractor catch up.
- // 2. To prevent the extractor from being farther than "next".
- (void)positionExtractorForTrack_l(trackIndex);
-
- SampleCursor& cursor = mTrackCursors[trackIndex];
- cursor.previous = cursor.current;
- cursor.current = cursor.next;
- cursor.next.reset();
-}
-
-media_status_t MediaSampleReaderNDK::positionExtractorForTrack_l(int trackIndex) {
- media_status_t status = AMEDIA_OK;
- const SampleCursor& cursor = mTrackCursors[trackIndex];
-
- // Seek backwards if the extractor is ahead of the current time.
- if (cursor.current.isSet && mExtractorSampleIndex > cursor.current.index) {
- status = seekExtractorBackwards_l(cursor.current.timeStampUs, trackIndex,
- cursor.current.index);
+media_status_t MediaSampleReaderNDK::moveToSample_l(SamplePosition& pos, int trackIndex) {
+ // Seek backwards if the extractor is ahead of the sample.
+ if (pos.isSet && mExtractorSampleIndex > pos.index) {
+ media_status_t status = seekExtractorBackwards_l(pos.timeStampUs, trackIndex, pos.index);
if (status != AMEDIA_OK) return status;
}
- // Advance until extractor points to the current sample.
- while (!(cursor.current.isSet && cursor.current.index == mExtractorSampleIndex)) {
+ // Advance until extractor points to the sample.
+ while (!(pos.isSet && pos.index == mExtractorSampleIndex)) {
if (!advanceExtractor_l()) {
return AMEDIA_ERROR_END_OF_STREAM;
}
@@ -190,28 +168,129 @@
return AMEDIA_OK;
}
-media_status_t MediaSampleReaderNDK::getEstimatedBitrateForTrack(int trackIndex, int32_t* bitrate) {
+media_status_t MediaSampleReaderNDK::moveToTrack_l(int trackIndex) {
+ return moveToSample_l(mTrackCursors[trackIndex].current, trackIndex);
+}
+
+media_status_t MediaSampleReaderNDK::waitForTrack_l(int trackIndex,
+ std::unique_lock<std::mutex>& lockHeld) {
+ while (trackIndex != mExtractorTrackIndex && !mEosReached && mEnforceSequentialAccess) {
+ mTrackSignals[trackIndex].wait(lockHeld);
+ }
+
+ if (mEosReached) {
+ return AMEDIA_ERROR_END_OF_STREAM;
+ }
+ return AMEDIA_OK;
+}
+
+media_status_t MediaSampleReaderNDK::primeExtractorForTrack_l(
+ int trackIndex, std::unique_lock<std::mutex>& lockHeld) {
+ if (mExtractorTrackIndex < 0) {
+ mExtractorTrackIndex = AMediaExtractor_getSampleTrackIndex(mExtractor);
+ if (mExtractorTrackIndex < 0) {
+ return AMEDIA_ERROR_END_OF_STREAM;
+ }
+ mTrackCursors[mExtractorTrackIndex].current.set(mExtractorSampleIndex,
+ AMediaExtractor_getSampleTime(mExtractor));
+ }
+
+ if (mEnforceSequentialAccess) {
+ return waitForTrack_l(trackIndex, lockHeld);
+ } else {
+ return moveToTrack_l(trackIndex);
+ }
+}
+
+media_status_t MediaSampleReaderNDK::selectTrack(int trackIndex) {
std::scoped_lock lock(mExtractorMutex);
- media_status_t status = AMEDIA_OK;
if (trackIndex < 0 || trackIndex >= mTrackCount) {
LOG(ERROR) << "Invalid trackIndex " << trackIndex << " for trackCount " << mTrackCount;
return AMEDIA_ERROR_INVALID_PARAMETER;
+ } else if (mTrackSignals.find(trackIndex) != mTrackSignals.end()) {
+ LOG(ERROR) << "TrackIndex " << trackIndex << " already selected";
+ return AMEDIA_ERROR_INVALID_PARAMETER;
+ } else if (mExtractorTrackIndex >= 0) {
+ LOG(ERROR) << "Tracks must be selected before sample reading begins.";
+ return AMEDIA_ERROR_UNSUPPORTED;
+ }
+
+ media_status_t status = AMediaExtractor_selectTrack(mExtractor, trackIndex);
+ if (status != AMEDIA_OK) {
+ LOG(ERROR) << "AMediaExtractor_selectTrack returned error: " << status;
+ return status;
+ }
+
+ mTrackSignals.emplace(std::piecewise_construct, std::forward_as_tuple(trackIndex),
+ std::forward_as_tuple());
+ return AMEDIA_OK;
+}
+
+media_status_t MediaSampleReaderNDK::setEnforceSequentialAccess(bool enforce) {
+ std::scoped_lock lock(mExtractorMutex);
+
+ if (mEnforceSequentialAccess && !enforce) {
+ // If switching from enforcing to not enforcing sequential access there may be threads
+ // waiting that needs to be woken up.
+ for (auto it = mTrackSignals.begin(); it != mTrackSignals.end(); ++it) {
+ it->second.notify_all();
+ }
+ } else if (!mEnforceSequentialAccess && enforce && mExtractorTrackIndex >= 0) {
+ // If switching from not enforcing to enforcing sequential access the extractor needs to be
+ // positioned for the track farthest behind so that it won't get stuck waiting.
+ struct {
+ SamplePosition* pos = nullptr;
+ int trackIndex = -1;
+ } earliestSample;
+
+ for (int trackIndex = 0; trackIndex < mTrackCount; ++trackIndex) {
+ SamplePosition& lastKnownTrackPosition = mTrackCursors[trackIndex].current.isSet
+ ? mTrackCursors[trackIndex].current
+ : mTrackCursors[trackIndex].previous;
+
+ if (lastKnownTrackPosition.isSet) {
+ if (earliestSample.pos == nullptr ||
+ earliestSample.pos->index > lastKnownTrackPosition.index) {
+ earliestSample.pos = &lastKnownTrackPosition;
+ earliestSample.trackIndex = trackIndex;
+ }
+ }
+ }
+
+ if (earliestSample.pos == nullptr) {
+ LOG(ERROR) << "No known sample position found";
+ return AMEDIA_ERROR_UNKNOWN;
+ }
+
+ media_status_t status = moveToSample_l(*earliestSample.pos, earliestSample.trackIndex);
+ if (status != AMEDIA_OK) return status;
+
+ while (!(mTrackCursors[mExtractorTrackIndex].current.isSet &&
+ mTrackCursors[mExtractorTrackIndex].current.index == mExtractorSampleIndex)) {
+ if (!advanceExtractor_l()) {
+ return AMEDIA_ERROR_END_OF_STREAM;
+ }
+ }
+ }
+
+ mEnforceSequentialAccess = enforce;
+ return AMEDIA_OK;
+}
+
+media_status_t MediaSampleReaderNDK::getEstimatedBitrateForTrack(int trackIndex, int32_t* bitrate) {
+ std::scoped_lock lock(mExtractorMutex);
+ media_status_t status = AMEDIA_OK;
+
+ if (mTrackSignals.find(trackIndex) == mTrackSignals.end()) {
+ LOG(ERROR) << "Track is not selected.";
+ return AMEDIA_ERROR_INVALID_PARAMETER;
} else if (bitrate == nullptr) {
LOG(ERROR) << "bitrate pointer is NULL.";
return AMEDIA_ERROR_INVALID_PARAMETER;
- }
-
- // Rewind the extractor and sample from the beginning of the file.
- if (mExtractorSampleIndex > 0) {
- status = AMediaExtractor_seekTo(mExtractor, 0, AMEDIAEXTRACTOR_SEEK_PREVIOUS_SYNC);
- if (status != AMEDIA_OK) {
- LOG(ERROR) << "Unable to reset extractor: " << status;
- return status;
- }
-
- mExtractorTrackIndex = AMediaExtractor_getSampleTrackIndex(mExtractor);
- mExtractorSampleIndex = 0;
+ } else if (mExtractorTrackIndex >= 0) {
+ LOG(ERROR) << "getEstimatedBitrateForTrack must be called before sample reading begins.";
+ return AMEDIA_ERROR_UNSUPPORTED;
}
// Sample the track.
@@ -222,7 +301,7 @@
int64_t lastSampleTimeUs = 0;
do {
- if (mExtractorTrackIndex == trackIndex) {
+ if (AMediaExtractor_getSampleTrackIndex(mExtractor) == trackIndex) {
lastSampleTimeUs = AMediaExtractor_getSampleTime(mExtractor);
if (totalSampleSize == 0) {
firstSampleTimeUs = lastSampleTimeUs;
@@ -231,7 +310,15 @@
lastSampleSize = AMediaExtractor_getSampleSize(mExtractor);
totalSampleSize += lastSampleSize;
}
- } while ((lastSampleTimeUs - firstSampleTimeUs) < kSamplingDurationUs && advanceExtractor_l());
+ } while ((lastSampleTimeUs - firstSampleTimeUs) < kSamplingDurationUs &&
+ AMediaExtractor_advance(mExtractor));
+
+ // Reset the extractor to the beginning.
+ status = AMediaExtractor_seekTo(mExtractor, 0, AMEDIAEXTRACTOR_SEEK_PREVIOUS_SYNC);
+ if (status != AMEDIA_OK) {
+ LOG(ERROR) << "Unable to reset extractor: " << status;
+ return status;
+ }
int64_t durationUs = 0;
const int64_t sampledDurationUs = lastSampleTimeUs - firstSampleTimeUs;
@@ -263,17 +350,17 @@
}
media_status_t MediaSampleReaderNDK::getSampleInfoForTrack(int trackIndex, MediaSampleInfo* info) {
- std::scoped_lock lock(mExtractorMutex);
+ std::unique_lock<std::mutex> lock(mExtractorMutex);
- if (trackIndex < 0 || trackIndex >= mTrackCount) {
- LOG(ERROR) << "Invalid trackIndex " << trackIndex << " for trackCount " << mTrackCount;
+ if (mTrackSignals.find(trackIndex) == mTrackSignals.end()) {
+ LOG(ERROR) << "Track not selected.";
return AMEDIA_ERROR_INVALID_PARAMETER;
} else if (info == nullptr) {
LOG(ERROR) << "MediaSampleInfo pointer is NULL.";
return AMEDIA_ERROR_INVALID_PARAMETER;
}
- media_status_t status = positionExtractorForTrack_l(trackIndex);
+ media_status_t status = primeExtractorForTrack_l(trackIndex, lock);
if (status == AMEDIA_OK) {
info->presentationTimeUs = AMediaExtractor_getSampleTime(mExtractor);
info->flags = AMediaExtractor_getSampleFlags(mExtractor);
@@ -283,24 +370,25 @@
info->flags = SAMPLE_FLAG_END_OF_STREAM;
info->size = 0;
}
-
return status;
}
media_status_t MediaSampleReaderNDK::readSampleDataForTrack(int trackIndex, uint8_t* buffer,
size_t bufferSize) {
- std::scoped_lock lock(mExtractorMutex);
+ std::unique_lock<std::mutex> lock(mExtractorMutex);
- if (trackIndex < 0 || trackIndex >= mTrackCount) {
- LOG(ERROR) << "Invalid trackIndex " << trackIndex << " for trackCount " << mTrackCount;
+ if (mTrackSignals.find(trackIndex) == mTrackSignals.end()) {
+ LOG(ERROR) << "Track not selected.";
return AMEDIA_ERROR_INVALID_PARAMETER;
} else if (buffer == nullptr) {
LOG(ERROR) << "buffer pointer is NULL";
return AMEDIA_ERROR_INVALID_PARAMETER;
}
- media_status_t status = positionExtractorForTrack_l(trackIndex);
- if (status != AMEDIA_OK) return status;
+ media_status_t status = primeExtractorForTrack_l(trackIndex, lock);
+ if (status != AMEDIA_OK) {
+ return status;
+ }
ssize_t sampleSize = AMediaExtractor_getSampleSize(mExtractor);
if (bufferSize < sampleSize) {
@@ -314,9 +402,21 @@
return AMEDIA_ERROR_INVALID_PARAMETER;
}
+ advanceTrack_l(trackIndex);
+
return AMEDIA_OK;
}
+void MediaSampleReaderNDK::advanceTrack(int trackIndex) {
+ std::scoped_lock lock(mExtractorMutex);
+
+ if (mTrackSignals.find(trackIndex) != mTrackSignals.end()) {
+ advanceTrack_l(trackIndex);
+ } else {
+ LOG(ERROR) << "Trying to advance a track that is not selected (#" << trackIndex << ")";
+ }
+}
+
AMediaFormat* MediaSampleReaderNDK::getFileFormat() {
return AMediaExtractor_getFileFormat(mExtractor);
}
diff --git a/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp b/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp
index 58e2066..92ce60a 100644
--- a/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp
@@ -92,6 +92,7 @@
if (mState == STARTED) {
abortTranscodeLoop();
+ mMediaSampleReader->setEnforceSequentialAccess(false);
mTranscodingThread.join();
mOutputQueue->abort(); // Wake up any threads waiting for samples.
mState = STOPPED;
diff --git a/media/libmediatranscoding/transcoder/MediaTranscoder.cpp b/media/libmediatranscoding/transcoder/MediaTranscoder.cpp
index ed702db..fbed5c2 100644
--- a/media/libmediatranscoding/transcoder/MediaTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/MediaTranscoder.cpp
@@ -133,6 +133,12 @@
mTracksAdded.insert(transcoder);
if (mTracksAdded.size() == mTrackTranscoders.size()) {
+ // Enable sequential access mode on the sample reader to achieve optimal read performance.
+ // This has to wait until all tracks have delivered their output formats and the sample
+ // writer is started. Otherwise the tracks will not get their output sample queues drained
+ // and the transcoder could hang due to one track running out of buffers and blocking the
+ // other tracks from reading source samples before they could output their formats.
+ mSampleReader->setEnforceSequentialAccess(true);
LOG(INFO) << "Starting sample writer.";
bool started = mSampleWriter->start();
if (!started) {
@@ -229,6 +235,12 @@
return AMEDIA_ERROR_INVALID_PARAMETER;
}
+ media_status_t status = mSampleReader->selectTrack(trackIndex);
+ if (status != AMEDIA_OK) {
+ LOG(ERROR) << "Unable to select track " << trackIndex;
+ return status;
+ }
+
std::shared_ptr<MediaTrackTranscoder> transcoder;
std::shared_ptr<AMediaFormat> format;
@@ -270,7 +282,7 @@
format = std::shared_ptr<AMediaFormat>(mergedFormat, &AMediaFormat_delete);
}
- media_status_t status = transcoder->configure(mSampleReader, trackIndex, format);
+ transcoder->configure(mSampleReader, trackIndex, format);
if (status != AMEDIA_OK) {
LOG(ERROR) << "Configure track transcoder for track #" << trackIndex << " returned error "
<< status;
@@ -344,6 +356,7 @@
}
mSampleWriter->stop();
+ mSampleReader->setEnforceSequentialAccess(false);
for (auto& transcoder : mTrackTranscoders) {
transcoder->stop();
}
diff --git a/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp b/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp
index e2cc6b6..e7c0271 100644
--- a/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp
@@ -142,8 +142,6 @@
LOG(ERROR) << "Output queue aborted";
return AMEDIA_ERROR_IO;
}
-
- mMediaSampleReader->advanceTrack(mTrackIndex);
}
if (mStopRequested && !mEosFromSource) {
diff --git a/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp b/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
index 5702627..b0bf59f 100644
--- a/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
@@ -326,8 +326,6 @@
mStatus = status;
return;
}
-
- mMediaSampleReader->advanceTrack(mTrackIndex);
} else {
LOG(DEBUG) << "EOS from source.";
mEosFromSource = true;
diff --git a/media/libmediatranscoding/transcoder/benchmark/MediaSampleReaderBenchmark.cpp b/media/libmediatranscoding/transcoder/benchmark/MediaSampleReaderBenchmark.cpp
index a651fa2..f0b9304 100644
--- a/media/libmediatranscoding/transcoder/benchmark/MediaSampleReaderBenchmark.cpp
+++ b/media/libmediatranscoding/transcoder/benchmark/MediaSampleReaderBenchmark.cpp
@@ -42,8 +42,8 @@
using namespace android;
static void ReadMediaSamples(benchmark::State& state, const std::string& srcFileName,
- bool readAudio) {
- // Asset directory
+ bool readAudio, bool sequentialAccess = false) {
+ // Asset directory.
static const std::string kAssetDirectory = "/data/local/tmp/TranscodingBenchmark/";
int srcFd = 0;
@@ -59,9 +59,13 @@
for (auto _ : state) {
auto sampleReader = MediaSampleReaderNDK::createFromFd(srcFd, 0, fileSize);
+ if (sampleReader->setEnforceSequentialAccess(sequentialAccess) != AMEDIA_OK) {
+ state.SkipWithError("setEnforceSequentialAccess failed");
+ return;
+ }
- std::vector<std::thread> trackThreads;
-
+ // Select tracks.
+ std::vector<int> trackIndices;
for (int trackIndex = 0; trackIndex < sampleReader->getTrackCount(); ++trackIndex) {
const char* mime = nullptr;
@@ -78,6 +82,13 @@
continue;
}
+ trackIndices.push_back(trackIndex);
+ sampleReader->selectTrack(trackIndex);
+ }
+
+ // Start threads.
+ std::vector<std::thread> trackThreads;
+ for (auto trackIndex : trackIndices) {
trackThreads.emplace_back([trackIndex, sampleReader, &state] {
LOG(INFO) << "Track " << trackIndex << " started";
MediaSampleInfo info;
@@ -102,14 +113,13 @@
state.SkipWithError("Error reading sample data");
break;
}
-
- sampleReader->advanceTrack(trackIndex);
}
LOG(INFO) << "Track " << trackIndex << " finished";
});
}
+ // Join threads.
for (auto& thread : trackThreads) {
thread.join();
}
@@ -122,17 +132,23 @@
#define TRANSCODER_BENCHMARK(func) \
BENCHMARK(func)->UseRealTime()->MeasureProcessCPUTime()->Unit(benchmark::kMillisecond)
-static void BM_MediaSampleReader_AudioVideo(benchmark::State& state) {
+static void BM_MediaSampleReader_AudioVideo_Parallel(benchmark::State& state) {
ReadMediaSamples(state, "video_1920x1080_3648frame_h264_22Mbps_30fps_aac.mp4",
true /* readAudio */);
}
+static void BM_MediaSampleReader_AudioVideo_Sequential(benchmark::State& state) {
+ ReadMediaSamples(state, "video_1920x1080_3648frame_h264_22Mbps_30fps_aac.mp4",
+ true /* readAudio */, true /* sequentialAccess */);
+}
+
static void BM_MediaSampleReader_Video(benchmark::State& state) {
ReadMediaSamples(state, "video_1920x1080_3648frame_h264_22Mbps_30fps_aac.mp4",
false /* readAudio */);
}
-TRANSCODER_BENCHMARK(BM_MediaSampleReader_AudioVideo);
+TRANSCODER_BENCHMARK(BM_MediaSampleReader_AudioVideo_Parallel);
+TRANSCODER_BENCHMARK(BM_MediaSampleReader_AudioVideo_Sequential);
TRANSCODER_BENCHMARK(BM_MediaSampleReader_Video);
BENCHMARK_MAIN();
diff --git a/media/libmediatranscoding/transcoder/benchmark/MediaTranscoderBenchmark.cpp b/media/libmediatranscoding/transcoder/benchmark/MediaTranscoderBenchmark.cpp
index b31b675..ccd9353 100644
--- a/media/libmediatranscoding/transcoder/benchmark/MediaTranscoderBenchmark.cpp
+++ b/media/libmediatranscoding/transcoder/benchmark/MediaTranscoderBenchmark.cpp
@@ -77,7 +77,8 @@
};
static void TranscodeMediaFile(benchmark::State& state, const std::string& srcFileName,
- const std::string& dstFileName, bool includeAudio) {
+ const std::string& dstFileName, bool includeAudio,
+ bool transcodeVideo = true) {
// Default bitrate
static constexpr int32_t kVideoBitRate = 20 * 1000 * 1000; // 20Mbs
// Write-only, create file if non-existent.
@@ -132,8 +133,10 @@
}
if (strncmp(mime, "video/", 6) == 0) {
- dstFormat = AMediaFormat_new();
- AMediaFormat_setInt32(dstFormat, AMEDIAFORMAT_KEY_BIT_RATE, kVideoBitRate);
+ if (transcodeVideo) {
+ dstFormat = AMediaFormat_new();
+ AMediaFormat_setInt32(dstFormat, AMEDIAFORMAT_KEY_BIT_RATE, kVideoBitRate);
+ }
int32_t frameCount;
if (AMediaFormat_getInt32(srcFormat, AMEDIAFORMAT_KEY_FRAME_COUNT, &frameCount)) {
@@ -198,8 +201,21 @@
false /* includeAudio */);
}
+static void BM_TranscodeAudioVideoPassthrough(benchmark::State& state) {
+ TranscodeMediaFile(state, "video_1920x1080_3648frame_h264_22Mbps_30fps_aac.mp4",
+ "video_1920x1080_3648frame_h264_22Mbps_30fps_aac_passthrough_AV.mp4",
+ true /* includeAudio */, false /* transcodeVideo */);
+}
+static void BM_TranscodeVideoPassthrough(benchmark::State& state) {
+ TranscodeMediaFile(state, "video_1920x1080_3648frame_h264_22Mbps_30fps.mp4",
+ "video_1920x1080_3648frame_h264_22Mbps_30fps_passthrough_AV.mp4",
+ false /* includeAudio */, false /* transcodeVideo */);
+}
+
TRANSCODER_BENCHMARK(BM_TranscodeAvc2AvcAudioVideo2AudioVideo);
TRANSCODER_BENCHMARK(BM_TranscodeAvc2AvcAudioVideo2Video);
TRANSCODER_BENCHMARK(BM_TranscodeAvc2AvcVideo2Video);
+TRANSCODER_BENCHMARK(BM_TranscodeAudioVideoPassthrough);
+TRANSCODER_BENCHMARK(BM_TranscodeVideoPassthrough);
BENCHMARK_MAIN();
diff --git a/media/libmediatranscoding/transcoder/include/media/MediaSampleReader.h b/media/libmediatranscoding/transcoder/include/media/MediaSampleReader.h
index 25df7ab..7b6fbef 100644
--- a/media/libmediatranscoding/transcoder/include/media/MediaSampleReader.h
+++ b/media/libmediatranscoding/transcoder/include/media/MediaSampleReader.h
@@ -24,12 +24,15 @@
namespace android {
/**
- * MediaSampleReader is an interface for reading media samples from a container.
- * MediaSampleReader allows for reading samples from multiple tracks independently of each other
- * while preserving the order of samples within each individual track.
- * MediaSampleReader implementations are thread safe and can be used by multiple threads
- * concurrently. But note that MediaSampleReader only maintains one state per track so concurrent
- * usage of the same track from multiple threads has no benefit.
+ * MediaSampleReader is an interface for reading media samples from a container. MediaSampleReader
+ * allows for reading samples from multiple tracks on individual threads independently of each other
+ * while preserving the order of samples. Due to poor non-sequential access performance of the
+ * underlying extractor, MediaSampleReader can optionally enforce sequential sample access by
+ * blocking requests for tracks that the underlying extractor does not currently point to. Waiting
+ * threads are serviced once the reader advances to a sample from the specified track. Due to this
+ * it is important to read samples and advance the reader from all selected tracks to avoid hanging
+ * other tracks. MediaSampleReader implementations are thread safe and sample access should be done
+ * on one thread per selected track.
*/
class MediaSampleReader {
public:
@@ -57,6 +60,24 @@
virtual AMediaFormat* getTrackFormat(int trackIndex) = 0;
/**
+ * Select a track for sample access. Tracks must be selected in order for sample information and
+ * sample data to be available for that track. Samples for selected tracks must be accessed on
+ * its own thread to avoid blocking other tracks.
+ * @param trackIndex The track to select.
+ * @return AMEDIA_OK on success.
+ */
+ virtual media_status_t selectTrack(int trackIndex) = 0;
+
+ /**
+ * Toggles sequential access enforcement on or off. When the reader enforces sequential access
+ * calls to read sample information will block unless the underlying extractor points to the
+ * specified track.
+ * @param enforce True to enforce sequential access.
+ * @return AMEDIA_OK on success.
+ */
+ virtual media_status_t setEnforceSequentialAccess(bool enforce) = 0;
+
+ /**
* Estimates the bitrate of a source track by sampling sample sizes. The bitrate is returned in
* megabits per second (Mbps). This method will fail if the track only contains a single sample
* and does not have an associated duration.
@@ -67,7 +88,9 @@
virtual media_status_t getEstimatedBitrateForTrack(int trackIndex, int32_t* bitrate);
/**
- * Returns the sample information for the current sample in the specified track.
+ * Returns the sample information for the current sample in the specified track. Note that this
+ * method will block until the reader advances to a sample belonging to the requested track if
+ * the reader is in sequential access mode.
* @param trackIndex The track index (zero-based).
* @param info Pointer to a MediaSampleInfo object where the sample information is written.
* @return AMEDIA_OK on success, AMEDIA_ERROR_END_OF_STREAM if there are no more samples to read
@@ -77,7 +100,10 @@
virtual media_status_t getSampleInfoForTrack(int trackIndex, MediaSampleInfo* info) = 0;
/**
- * Reads the current sample's data into the supplied buffer.
+ * Returns the sample data for the current sample in the specified track into the supplied
+ * buffer. Note that this method will block until the reader advances to a sample belonging to
+ * the requested track if the reader is in sequential access mode. Upon successful return this
+ * method will also advance the specified track to the next sample.
* @param trackIndex The track index (zero-based).
* @param buffer The buffer to write the sample's data to.
* @param bufferSize The size of the supplied buffer.
@@ -90,7 +116,9 @@
size_t bufferSize) = 0;
/**
- * Advance the specified track to the next sample.
+ * Advance the specified track to the next sample. If the reader is in sequential access mode
+ * and the current sample belongs to the specified track, the reader will also advance to the
+ * next sample and wake up any threads waiting on the new track.
* @param trackIndex The track index (zero-based).
*/
virtual void advanceTrack(int trackIndex) = 0;
diff --git a/media/libmediatranscoding/transcoder/include/media/MediaSampleReaderNDK.h b/media/libmediatranscoding/transcoder/include/media/MediaSampleReaderNDK.h
index 6c5019d..5f9822d 100644
--- a/media/libmediatranscoding/transcoder/include/media/MediaSampleReaderNDK.h
+++ b/media/libmediatranscoding/transcoder/include/media/MediaSampleReaderNDK.h
@@ -20,6 +20,7 @@
#include <media/MediaSampleReader.h>
#include <media/NdkMediaExtractor.h>
+#include <map>
#include <memory>
#include <mutex>
#include <vector>
@@ -46,6 +47,8 @@
AMediaFormat* getFileFormat() override;
size_t getTrackCount() const override;
AMediaFormat* getTrackFormat(int trackIndex) override;
+ media_status_t selectTrack(int trackIndex) override;
+ media_status_t setEnforceSequentialAccess(bool enforce) override;
media_status_t getEstimatedBitrateForTrack(int trackIndex, int32_t* bitrate) override;
media_status_t getSampleInfoForTrack(int trackIndex, MediaSampleInfo* info) override;
media_status_t readSampleDataForTrack(int trackIndex, uint8_t* buffer,
@@ -55,20 +58,6 @@
virtual ~MediaSampleReaderNDK() override;
private:
- /**
- * Creates a new MediaSampleReaderNDK object from an AMediaExtractor. The extractor needs to be
- * initialized with a valid data source before attempting to create a MediaSampleReaderNDK.
- * @param extractor The initialized media extractor.
- */
- MediaSampleReaderNDK(AMediaExtractor* extractor);
- media_status_t init();
-
- AMediaExtractor* mExtractor = nullptr;
- std::mutex mExtractorMutex;
- const size_t mTrackCount;
-
- int mExtractorTrackIndex = -1;
- uint64_t mExtractorSampleIndex = 0;
/**
* SamplePosition describes the position of a single sample in the media file using its
@@ -100,13 +89,52 @@
SamplePosition next;
};
- /** Samples cursor for each track in the file. */
- std::vector<SampleCursor> mTrackCursors;
+ /**
+ * Creates a new MediaSampleReaderNDK object from an AMediaExtractor. The extractor needs to be
+ * initialized with a valid data source before attempting to create a MediaSampleReaderNDK.
+ * @param extractor The initialized media extractor.
+ */
+ MediaSampleReaderNDK(AMediaExtractor* extractor);
+ /** Advances the track to next sample. */
+ void advanceTrack_l(int trackIndex);
+
+ /** Advances the extractor to next sample. */
bool advanceExtractor_l();
- media_status_t positionExtractorForTrack_l(int trackIndex);
+
+ /** Moves the extractor backwards to the specified sample. */
media_status_t seekExtractorBackwards_l(int64_t targetTimeUs, int targetTrackIndex,
uint64_t targetSampleIndex);
+
+ /** Moves the extractor to the specified sample. */
+ media_status_t moveToSample_l(SamplePosition& pos, int trackIndex);
+
+ /** Moves the extractor to the next sample of the specified track. */
+ media_status_t moveToTrack_l(int trackIndex);
+
+ /** In sequential mode, waits for the extractor to reach the next sample for the track. */
+ media_status_t waitForTrack_l(int trackIndex, std::unique_lock<std::mutex>& lockHeld);
+
+ /**
+ * Ensures the extractor is ready for the next sample of the track regardless of access mode.
+ */
+ media_status_t primeExtractorForTrack_l(int trackIndex, std::unique_lock<std::mutex>& lockHeld);
+
+ AMediaExtractor* mExtractor = nullptr;
+ std::mutex mExtractorMutex;
+ const size_t mTrackCount;
+
+ int mExtractorTrackIndex = -1;
+ uint64_t mExtractorSampleIndex = 0;
+
+ bool mEosReached = false;
+ bool mEnforceSequentialAccess = false;
+
+ // Maps selected track indices to condition variables for sequential sample access control.
+ std::map<int, std::condition_variable> mTrackSignals;
+
+ // Samples cursor for each track in the file.
+ std::vector<SampleCursor> mTrackCursors;
};
} // namespace android
diff --git a/media/libmediatranscoding/transcoder/tests/MediaSampleReaderNDKTests.cpp b/media/libmediatranscoding/transcoder/tests/MediaSampleReaderNDKTests.cpp
index 323e5ae..e8acd48 100644
--- a/media/libmediatranscoding/transcoder/tests/MediaSampleReaderNDKTests.cpp
+++ b/media/libmediatranscoding/transcoder/tests/MediaSampleReaderNDKTests.cpp
@@ -26,10 +26,14 @@
#include <gtest/gtest.h>
#include <media/MediaSampleReaderNDK.h>
#include <utils/Timers.h>
-
#include <cmath>
+#include <mutex>
+#include <thread>
// TODO(b/153453392): Test more asset types and validate sample data from readSampleDataForTrack.
+// TODO(b/153453392): Test for sequential and parallel (single thread and multi thread) access.
+// TODO(b/153453392): Test for switching between sequential and parallel access in different points
+// of time.
namespace android {
@@ -121,48 +125,47 @@
MediaSampleReaderNDK::createFromFd(mSourceFd, 0, mFileSize);
ASSERT_TRUE(sampleReader);
- MediaSampleInfo info;
- int trackEosCount = 0;
- std::vector<bool> trackReachedEos(mTrackCount, false);
- std::vector<std::vector<int64_t>> readerTimestamps(mTrackCount);
+ for (int trackIndex = 0; trackIndex < mTrackCount; trackIndex++) {
+ EXPECT_EQ(sampleReader->selectTrack(trackIndex), AMEDIA_OK);
+ }
// Initialize the extractor timestamps.
initExtractorTimestamps();
- // Read 5s of each track at a time.
- const int64_t chunkDurationUs = SEC_TO_USEC(5);
- int64_t chunkEndTimeUs = chunkDurationUs;
+ std::mutex timestampMutex;
+ std::vector<std::thread> trackThreads;
+ std::vector<std::vector<int64_t>> readerTimestamps(mTrackCount);
- // Loop until all tracks have reached End Of Stream.
- while (trackEosCount < mTrackCount) {
- for (int trackIndex = 0; trackIndex < mTrackCount; trackIndex++) {
- if (trackReachedEos[trackIndex]) continue;
-
- // Advance current track to next chunk end time.
- do {
+ for (int trackIndex = 0; trackIndex < mTrackCount; trackIndex++) {
+ trackThreads.emplace_back([sampleReader, trackIndex, ×tampMutex, &readerTimestamps] {
+ MediaSampleInfo info;
+ while (true) {
media_status_t status = sampleReader->getSampleInfoForTrack(trackIndex, &info);
if (status != AMEDIA_OK) {
- ASSERT_EQ(status, AMEDIA_ERROR_END_OF_STREAM);
- ASSERT_TRUE((info.flags & SAMPLE_FLAG_END_OF_STREAM) != 0);
- trackReachedEos[trackIndex] = true;
- trackEosCount++;
+ EXPECT_EQ(status, AMEDIA_ERROR_END_OF_STREAM);
+ EXPECT_TRUE((info.flags & SAMPLE_FLAG_END_OF_STREAM) != 0);
break;
}
ASSERT_TRUE((info.flags & SAMPLE_FLAG_END_OF_STREAM) == 0);
+ timestampMutex.lock();
readerTimestamps[trackIndex].push_back(info.presentationTimeUs);
+ timestampMutex.unlock();
sampleReader->advanceTrack(trackIndex);
- } while (info.presentationTimeUs < chunkEndTimeUs);
- }
- chunkEndTimeUs += chunkDurationUs;
+ }
+ });
+ }
+
+ for (auto& thread : trackThreads) {
+ thread.join();
}
for (int trackIndex = 0; trackIndex < mTrackCount; trackIndex++) {
LOG(DEBUG) << "Track " << trackIndex << ", comparing "
<< readerTimestamps[trackIndex].size() << " samples.";
- ASSERT_EQ(readerTimestamps[trackIndex].size(), mExtractorTimestamps[trackIndex].size());
+ EXPECT_EQ(readerTimestamps[trackIndex].size(), mExtractorTimestamps[trackIndex].size());
for (size_t sampleIndex = 0; sampleIndex < readerTimestamps[trackIndex].size();
sampleIndex++) {
- ASSERT_EQ(readerTimestamps[trackIndex][sampleIndex],
+ EXPECT_EQ(readerTimestamps[trackIndex][sampleIndex],
mExtractorTimestamps[trackIndex][sampleIndex]);
}
}
@@ -178,6 +181,8 @@
std::vector<int32_t> actualTrackBitrates = getTrackBitrates();
for (int trackIndex = 0; trackIndex < mTrackCount; ++trackIndex) {
+ EXPECT_EQ(sampleReader->selectTrack(trackIndex), AMEDIA_OK);
+
int32_t bitrate;
EXPECT_EQ(sampleReader->getEstimatedBitrateForTrack(trackIndex, &bitrate), AMEDIA_OK);
EXPECT_GT(bitrate, 0);
diff --git a/media/libmediatranscoding/transcoder/tests/MediaTrackTranscoderTests.cpp b/media/libmediatranscoding/transcoder/tests/MediaTrackTranscoderTests.cpp
index 804571c..a46c2bd 100644
--- a/media/libmediatranscoding/transcoder/tests/MediaTrackTranscoderTests.cpp
+++ b/media/libmediatranscoding/transcoder/tests/MediaTrackTranscoderTests.cpp
@@ -111,6 +111,7 @@
}
ASSERT_NE(mSourceFormat, nullptr);
+ EXPECT_EQ(mMediaSampleReader->selectTrack(mTrackIndex), AMEDIA_OK);
}
// Drains the transcoder's output queue in a loop.
diff --git a/media/libmediatranscoding/transcoder/tests/PassthroughTrackTranscoderTests.cpp b/media/libmediatranscoding/transcoder/tests/PassthroughTrackTranscoderTests.cpp
index b79f58c..a2ffbe4 100644
--- a/media/libmediatranscoding/transcoder/tests/PassthroughTrackTranscoderTests.cpp
+++ b/media/libmediatranscoding/transcoder/tests/PassthroughTrackTranscoderTests.cpp
@@ -159,6 +159,7 @@
MediaSampleReaderNDK::createFromFd(mSourceFd, 0, mSourceFileSize);
EXPECT_NE(mediaSampleReader, nullptr);
+ EXPECT_EQ(mediaSampleReader->selectTrack(mTrackIndex), AMEDIA_OK);
EXPECT_EQ(transcoder.configure(mediaSampleReader, mTrackIndex, nullptr /* destinationFormat */),
AMEDIA_OK);
ASSERT_TRUE(transcoder.start());
diff --git a/media/libmediatranscoding/transcoder/tests/VideoTrackTranscoderTests.cpp b/media/libmediatranscoding/transcoder/tests/VideoTrackTranscoderTests.cpp
index b432553..e809cbd 100644
--- a/media/libmediatranscoding/transcoder/tests/VideoTrackTranscoderTests.cpp
+++ b/media/libmediatranscoding/transcoder/tests/VideoTrackTranscoderTests.cpp
@@ -97,6 +97,7 @@
std::shared_ptr<TestCallback> callback = std::make_shared<TestCallback>();
auto transcoder = VideoTrackTranscoder::create(callback);
+ EXPECT_EQ(mMediaSampleReader->selectTrack(mTrackIndex), AMEDIA_OK);
EXPECT_EQ(transcoder->configure(mMediaSampleReader, mTrackIndex, mDestinationFormat),
AMEDIA_OK);
ASSERT_TRUE(transcoder->start());
@@ -152,6 +153,11 @@
mSourceFormat.get(), false /* includeBitrate*/);
EXPECT_NE(destFormat, nullptr);
+ EXPECT_EQ(mMediaSampleReader->selectTrack(mTrackIndex), AMEDIA_OK);
+
+ int32_t srcBitrate;
+ EXPECT_EQ(mMediaSampleReader->getEstimatedBitrateForTrack(mTrackIndex, &srcBitrate), AMEDIA_OK);
+
ASSERT_EQ(transcoder->configure(mMediaSampleReader, mTrackIndex, destFormat), AMEDIA_OK);
ASSERT_TRUE(transcoder->start());
@@ -166,9 +172,6 @@
int32_t outBitrate;
EXPECT_TRUE(AMediaFormat_getInt32(outputFormat.get(), AMEDIAFORMAT_KEY_BIT_RATE, &outBitrate));
- int32_t srcBitrate;
- EXPECT_EQ(mMediaSampleReader->getEstimatedBitrateForTrack(mTrackIndex, &srcBitrate), AMEDIA_OK);
-
EXPECT_EQ(srcBitrate, outBitrate);
}
@@ -206,6 +209,7 @@
auto callback = std::make_shared<TestCallback>();
auto transcoder = VideoTrackTranscoder::create(callback);
+ EXPECT_EQ(mMediaSampleReader->selectTrack(mTrackIndex), AMEDIA_OK);
EXPECT_EQ(transcoder->configure(mMediaSampleReader, mTrackIndex, mDestinationFormat),
AMEDIA_OK);
ASSERT_TRUE(transcoder->start());
@@ -214,7 +218,7 @@
std::vector<std::shared_ptr<MediaSample>> samples;
std::thread sampleConsumerThread([&outputQueue, &samples, &semaphore] {
std::shared_ptr<MediaSample> sample;
- while (samples.size() < 10 && !outputQueue->dequeue(&sample)) {
+ while (samples.size() < 4 && !outputQueue->dequeue(&sample)) {
ASSERT_NE(sample, nullptr);
samples.push_back(sample);