In "special" mode we now establish a UDP RTCP channel in addition to the
TCP RTP channel and provide feedback on the latency of arriving packets from
the sink back to the source. This information is then used to throttle
video bitrate.
Change-Id: Ic589a3cb65e4893a3ff67de947da6063d32a1c6e
diff --git a/media/libstagefright/wifi-display/ANetworkSession.cpp b/media/libstagefright/wifi-display/ANetworkSession.cpp
index df20ae2..88ca1cc 100644
--- a/media/libstagefright/wifi-display/ANetworkSession.cpp
+++ b/media/libstagefright/wifi-display/ANetworkSession.cpp
@@ -565,7 +565,7 @@
mSawSendFailure = true;
}
-#if 1
+#if 0
int numBytesQueued;
int res = ioctl(mSocket, SIOCOUTQ, &numBytesQueued);
if (res == 0 && numBytesQueued > 50 * 1024) {
@@ -576,7 +576,7 @@
int64_t nowUs = ALooper::GetNowUs();
if (mLastStallReportUs < 0ll
- || nowUs > mLastStallReportUs + 500000ll) {
+ || nowUs > mLastStallReportUs + 100000ll) {
sp<AMessage> msg = mNotify->dup();
msg->setInt32("sessionID", mSessionID);
msg->setInt32("reason", kWhatNetworkStall);
diff --git a/media/libstagefright/wifi-display/MediaSender.cpp b/media/libstagefright/wifi-display/MediaSender.cpp
index 123bc1c..33af66d 100644
--- a/media/libstagefright/wifi-display/MediaSender.cpp
+++ b/media/libstagefright/wifi-display/MediaSender.cpp
@@ -85,10 +85,11 @@
status_t MediaSender::initAsync(
ssize_t trackIndex,
- RTPSender::TransportMode transportMode,
const char *remoteHost,
int32_t remoteRTPPort,
+ RTPSender::TransportMode rtpMode,
int32_t remoteRTCPPort,
+ RTPSender::TransportMode rtcpMode,
int32_t *localRTPPort) {
if (trackIndex < 0) {
if (mMode != MODE_UNDEFINED) {
@@ -126,12 +127,9 @@
err = mTSSender->initAsync(
remoteHost,
remoteRTPPort,
- transportMode, // rtpMode
+ rtpMode,
remoteRTCPPort,
- (transportMode == RTPSender::TRANSPORT_UDP
- && remoteRTCPPort >= 0)
- ? transportMode
- : RTPSender::TRANSPORT_NONE, // rtcpMode
+ rtcpMode,
localRTPPort);
if (err != OK) {
@@ -180,11 +178,9 @@
status_t err = info->mSender->initAsync(
remoteHost,
remoteRTPPort,
- transportMode, // rtpMode
+ rtpMode,
remoteRTCPPort,
- (transportMode == RTPSender::TRANSPORT_UDP && remoteRTCPPort >= 0)
- ? transportMode
- : RTPSender::TRANSPORT_NONE, // rtcpMode
+ rtcpMode,
localRTPPort);
if (err != OK) {
@@ -345,6 +341,22 @@
break;
}
+ case kWhatInformSender:
+ {
+ int64_t avgLatencyUs;
+ CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs));
+
+ int64_t maxLatencyUs;
+ CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs));
+
+ sp<AMessage> notify = mNotify->dup();
+ notify->setInt32("what", kWhatInformSender);
+ notify->setInt64("avgLatencyUs", avgLatencyUs);
+ notify->setInt64("maxLatencyUs", maxLatencyUs);
+ notify->post();
+ break;
+ }
+
default:
TRESPASS();
}
diff --git a/media/libstagefright/wifi-display/MediaSender.h b/media/libstagefright/wifi-display/MediaSender.h
index 447abf7..04538ea 100644
--- a/media/libstagefright/wifi-display/MediaSender.h
+++ b/media/libstagefright/wifi-display/MediaSender.h
@@ -43,6 +43,7 @@
kWhatInitDone,
kWhatError,
kWhatNetworkStall,
+ kWhatInformSender,
};
MediaSender(
@@ -59,10 +60,11 @@
// If trackIndex == -1, initialize for transport stream muxing.
status_t initAsync(
ssize_t trackIndex,
- RTPSender::TransportMode transportMode,
const char *remoteHost,
int32_t remoteRTPPort,
+ RTPSender::TransportMode rtpMode,
int32_t remoteRTCPPort,
+ RTPSender::TransportMode rtcpMode,
int32_t *localRTPPort);
status_t queueAccessUnit(
diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.cpp b/media/libstagefright/wifi-display/rtp/RTPSender.cpp
index c686e01..9eeeabd 100644
--- a/media/libstagefright/wifi-display/rtp/RTPSender.cpp
+++ b/media/libstagefright/wifi-display/rtp/RTPSender.cpp
@@ -91,8 +91,8 @@
CHECK_NE(rtpMode, TRANSPORT_TCP_INTERLEAVED);
CHECK_NE(rtcpMode, TRANSPORT_TCP_INTERLEAVED);
- if (rtcpMode == TRANSPORT_NONE && remoteRTCPPort >= 0
- || rtcpMode != TRANSPORT_NONE && remoteRTCPPort < 0) {
+ if ((rtcpMode == TRANSPORT_NONE && remoteRTCPPort >= 0)
+ || (rtcpMode != TRANSPORT_NONE && remoteRTCPPort < 0)) {
return INVALID_OPERATION;
}
@@ -616,6 +616,7 @@
break;
case 204: // APP
+ parseAPP(data, headerLength);
break;
case 205: // TSFB (transport layer specific feedback)
@@ -721,6 +722,21 @@
return OK;
}
+status_t RTPSender::parseAPP(const uint8_t *data, size_t size) {
+ if (!memcmp("late", &data[8], 4)) {
+ int64_t avgLatencyUs = (int64_t)U64_AT(&data[12]);
+ int64_t maxLatencyUs = (int64_t)U64_AT(&data[20]);
+
+ sp<AMessage> notify = mNotify->dup();
+ notify->setInt32("what", kWhatInformSender);
+ notify->setInt64("avgLatencyUs", avgLatencyUs);
+ notify->setInt64("maxLatencyUs", maxLatencyUs);
+ notify->post();
+ }
+
+ return OK;
+}
+
void RTPSender::notifyInitDone(status_t err) {
sp<AMessage> notify = mNotify->dup();
notify->setInt32("what", kWhatInitDone);
diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.h b/media/libstagefright/wifi-display/rtp/RTPSender.h
index 8409b8d..3a926ea 100644
--- a/media/libstagefright/wifi-display/rtp/RTPSender.h
+++ b/media/libstagefright/wifi-display/rtp/RTPSender.h
@@ -37,6 +37,7 @@
kWhatInitDone,
kWhatError,
kWhatNetworkStall,
+ kWhatInformSender,
};
RTPSender(
const sp<ANetworkSession> &netSession,
@@ -105,6 +106,7 @@
status_t onRTCPData(const sp<ABuffer> &data);
status_t parseReceiverReport(const uint8_t *data, size_t size);
status_t parseTSFB(const uint8_t *data, size_t size);
+ status_t parseAPP(const uint8_t *data, size_t size);
void notifyInitDone(status_t err);
void notifyError(status_t err);
diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp
index bb8c387..d41e1e6 100644
--- a/media/libstagefright/wifi-display/source/Converter.cpp
+++ b/media/libstagefright/wifi-display/source/Converter.cpp
@@ -622,6 +622,7 @@
}
status_t Converter::doMoreWork() {
+#if 0
if (mIsVideo) {
int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000);
if (videoBitrate != mPrevVideoBitrate) {
@@ -633,6 +634,7 @@
mPrevVideoBitrate = videoBitrate;
}
}
+#endif
status_t err;
@@ -708,4 +710,19 @@
(new AMessage(kWhatDropAFrame, id()))->post();
}
+int32_t Converter::getVideoBitrate() const {
+ return mPrevVideoBitrate;
+}
+
+void Converter::setVideoBitrate(int32_t bitRate) {
+ if (mIsVideo && mEncoder != NULL && bitRate != mPrevVideoBitrate) {
+ sp<AMessage> params = new AMessage;
+ params->setInt32("videoBitrate", bitRate);
+
+ mEncoder->setParameters(params);
+
+ mPrevVideoBitrate = bitRate;
+ }
+}
+
} // namespace android
diff --git a/media/libstagefright/wifi-display/source/Converter.h b/media/libstagefright/wifi-display/source/Converter.h
index a418f69..538f10a 100644
--- a/media/libstagefright/wifi-display/source/Converter.h
+++ b/media/libstagefright/wifi-display/source/Converter.h
@@ -70,6 +70,9 @@
void shutdownAsync();
+ int32_t getVideoBitrate() const;
+ void setVideoBitrate(int32_t bitrate);
+
protected:
virtual ~Converter();
virtual void onMessageReceived(const sp<AMessage> &msg);
diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp
index a3b6542..68aa9cb 100644
--- a/media/libstagefright/wifi-display/source/PlaybackSession.cpp
+++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp
@@ -362,8 +362,11 @@
}
status_t WifiDisplaySource::PlaybackSession::init(
- const char *clientIP, int32_t clientRtp, int32_t clientRtcp,
- RTPSender::TransportMode transportMode,
+ const char *clientIP,
+ int32_t clientRtp,
+ RTPSender::TransportMode rtpMode,
+ int32_t clientRtcp,
+ RTPSender::TransportMode rtcpMode,
bool enableAudio,
bool usePCMAudio,
bool enableVideo,
@@ -385,10 +388,11 @@
if (err == OK) {
err = mMediaSender->initAsync(
-1 /* trackIndex */,
- transportMode,
clientIP,
clientRtp,
+ rtpMode,
clientRtcp,
+ rtcpMode,
&mLocalRTPPort);
}
@@ -548,6 +552,8 @@
converter->dropAFrame();
}
}
+ } else if (what == MediaSender::kWhatInformSender) {
+ onSinkFeedback(msg);
} else {
TRESPASS();
}
@@ -643,6 +649,46 @@
}
}
+void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) {
+ int64_t avgLatencyUs;
+ CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs));
+
+ int64_t maxLatencyUs;
+ CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs));
+
+ ALOGI("sink reports avg. latency of %lld ms (max %lld ms)",
+ avgLatencyUs / 1000ll,
+ maxLatencyUs / 1000ll);
+
+ if (mVideoTrackIndex >= 0) {
+ const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex);
+ sp<Converter> converter = videoTrack->converter();
+ if (converter != NULL) {
+ int32_t videoBitrate = converter->getVideoBitrate();
+
+ if (avgLatencyUs > 300000ll) {
+ videoBitrate *= 0.6;
+
+ if (videoBitrate < 500000) {
+ videoBitrate = 500000; // cap at 500kbit/sec
+ }
+ } else if (avgLatencyUs < 100000ll) {
+ videoBitrate *= 1.1;
+
+ if (videoBitrate > 10000000) {
+ videoBitrate = 10000000; // cap at 10Mbit/sec
+ }
+ }
+
+ if (videoBitrate != converter->getVideoBitrate()) {
+ ALOGI("setting video bitrate to %d bps", videoBitrate);
+
+ converter->setVideoBitrate(videoBitrate);
+ }
+ }
+ }
+}
+
status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer(
bool enableAudio, bool enableVideo) {
DataSource::RegisterDefaultSniffers();
diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.h b/media/libstagefright/wifi-display/source/PlaybackSession.h
index da207e2..39086a1 100644
--- a/media/libstagefright/wifi-display/source/PlaybackSession.h
+++ b/media/libstagefright/wifi-display/source/PlaybackSession.h
@@ -44,8 +44,11 @@
const char *path = NULL);
status_t init(
- const char *clientIP, int32_t clientRtp, int32_t clientRtcp,
- RTPSender::TransportMode transportMode,
+ const char *clientIP,
+ int32_t clientRtp,
+ RTPSender::TransportMode rtpMode,
+ int32_t clientRtcp,
+ RTPSender::TransportMode rtcpMode,
bool enableAudio,
bool usePCMAudio,
bool enableVideo,
@@ -149,6 +152,8 @@
void schedulePullExtractor();
void onPullExtractor();
+ void onSinkFeedback(const sp<AMessage> &msg);
+
DISALLOW_EVIL_CONSTRUCTORS(PlaybackSession);
};
diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp
index 5167cb3..f2e659a 100644
--- a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp
+++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp
@@ -1159,7 +1159,7 @@
return ERROR_MALFORMED;
}
- RTPSender::TransportMode transportMode = RTPSender::TRANSPORT_UDP;
+ RTPSender::TransportMode rtpMode = RTPSender::TRANSPORT_UDP;
int clientRtp, clientRtcp;
if (transport.startsWith("RTP/AVP/TCP;")) {
@@ -1168,7 +1168,7 @@
transport.c_str(), "interleaved", &interleaved)
&& sscanf(interleaved.c_str(), "%d-%d",
&clientRtp, &clientRtcp) == 2) {
- transportMode = RTPSender::TRANSPORT_TCP_INTERLEAVED;
+ rtpMode = RTPSender::TRANSPORT_TCP_INTERLEAVED;
} else {
bool badRequest = false;
@@ -1190,7 +1190,7 @@
return ERROR_MALFORMED;
}
- transportMode = RTPSender::TRANSPORT_TCP;
+ rtpMode = RTPSender::TRANSPORT_TCP;
}
} else if (transport.startsWith("RTP/AVP;unicast;")
|| transport.startsWith("RTP/AVP/UDP;unicast;")) {
@@ -1249,11 +1249,17 @@
return ERROR_MALFORMED;
}
+ RTPSender::TransportMode rtcpMode = RTPSender::TRANSPORT_UDP;
+ if (clientRtcp < 0) {
+ rtcpMode = RTPSender::TRANSPORT_NONE;
+ }
+
status_t err = playbackSession->init(
mClientInfo.mRemoteIP.c_str(),
clientRtp,
+ rtpMode,
clientRtcp,
- transportMode,
+ rtcpMode,
mSinkSupportsAudio,
mUsingPCMAudio,
mSinkSupportsVideo,
@@ -1282,7 +1288,7 @@
AString response = "RTSP/1.0 200 OK\r\n";
AppendCommonResponse(&response, cseq, playbackSessionID);
- if (transportMode == RTPSender::TRANSPORT_TCP_INTERLEAVED) {
+ if (rtpMode == RTPSender::TRANSPORT_TCP_INTERLEAVED) {
response.append(
StringPrintf(
"Transport: RTP/AVP/TCP;interleaved=%d-%d;",
@@ -1291,7 +1297,7 @@
int32_t serverRtp = playbackSession->getRTPPort();
AString transportString = "UDP";
- if (transportMode == RTPSender::TRANSPORT_TCP) {
+ if (rtpMode == RTPSender::TRANSPORT_TCP) {
transportString = "TCP";
}