commit d6e4aec66184c1a7a78767113e2bf92df133dbfa
author Chong Zhang <chz@google.com> Mon Jun 22 14:13:07 2020 -0700
committer Chong Zhang <chz@google.com> Mon Jun 22 15:58:25 2020 -0700
tree 0c48c3346ac574df068db7f9cc3f7c743b1e7432
parent 1f16f12b608be724ede9c98de15af4e972532e80

transcoder: fix rotation handling

Transfer rotation degree from source format to track format
for MediaSampleWriter, and setOrietationHint on the muxer.
During transcoding using encoder/decoder, clear the rotation
degree so that the codec is guaranted to operate non-rotated.

bug: 154734285
bug: 159649977
bug: 152091443

test: unit testing (updated test with rotation)

Change-Id: I7e829ef777b1954d894411e8591a5e31ff36e275

media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp

diff --git a/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp b/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
index 96e2e61..4df3296 100644
--- a/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
@@ -139,6 +139,10 @@
     }
     AMediaFormat_setInt32(encoderFormat, AMEDIAFORMAT_KEY_COLOR_FORMAT, kColorFormatSurface);
 
+    // Always encode without rotation. The rotation degree will be transferred directly to
+    // MediaSampleWriter track format, and MediaSampleWriter will call AMediaMuxer_setOrientationHint.
+    AMediaFormat_setInt32(encoderFormat, AMEDIAFORMAT_KEY_ROTATION, 0);
+
     mDestinationFormat = std::shared_ptr<AMediaFormat>(encoderFormat, &AMediaFormat_delete);
 
     // Create and configure the encoder.
@@ -339,25 +343,35 @@
     // at container level. This is supposed to override any SAR settings in the bitstream,
     // thus should always be transferred to the container of the transcoded file.
     int32_t sarWidth, sarHeight;
-    if (AMediaFormat_getInt32(mDestinationFormat.get(), AMEDIAFORMAT_KEY_SAR_WIDTH, &sarWidth) &&
+    if (AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_SAR_WIDTH, &sarWidth) &&
         (sarWidth > 0) &&
-        AMediaFormat_getInt32(mDestinationFormat.get(), AMEDIAFORMAT_KEY_SAR_HEIGHT, &sarHeight) &&
+        AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_SAR_HEIGHT, &sarHeight) &&
         (sarHeight > 0)) {
         AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_SAR_WIDTH, sarWidth);
         AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_SAR_HEIGHT, sarHeight);
     }
     // Transfer DAR settings.
     int32_t displayWidth, displayHeight;
-    if (AMediaFormat_getInt32(mDestinationFormat.get(), AMEDIAFORMAT_KEY_DISPLAY_WIDTH,
-                              &displayWidth) &&
+    if (AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_DISPLAY_WIDTH, &displayWidth) &&
         (displayWidth > 0) &&
-        AMediaFormat_getInt32(mDestinationFormat.get(), AMEDIAFORMAT_KEY_DISPLAY_HEIGHT,
+        AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_DISPLAY_HEIGHT,
                               &displayHeight) &&
         (displayHeight > 0)) {
         AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_DISPLAY_WIDTH, displayWidth);
         AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_DISPLAY_HEIGHT, displayHeight);
     }
 
+    // Transfer rotation settings.
+    // Note that muxer itself doesn't take rotation from the track format. It requires
+    // AMediaMuxer_setOrientationHint to set the rotation. Here we pass the rotation to
+    // MediaSampleWriter using the track format. MediaSampleWriter will then call
+    // AMediaMuxer_setOrientationHint as needed.
+    int32_t rotation;
+    if (AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_ROTATION, &rotation) &&
+        (rotation != 0)) {
+        AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_ROTATION, rotation);
+    }
+
     // TODO: transfer other fields as required.
 
     mActualOutputFormat = std::shared_ptr<AMediaFormat>(formatCopy, &AMediaFormat_delete);