Merge "Support device role for capture preset."
diff --git a/camera/CameraMetadata.cpp b/camera/CameraMetadata.cpp
index 135384a..7e4c91e 100644
--- a/camera/CameraMetadata.cpp
+++ b/camera/CameraMetadata.cpp
@@ -22,6 +22,7 @@
#include <binder/Parcel.h>
#include <camera/CameraMetadata.h>
+#include <camera_metadata_hidden.h>
namespace android {
@@ -872,5 +873,8 @@
return OK;
}
+metadata_vendor_id_t CameraMetadata::getVendorId() {
+ return get_camera_metadata_vendor_id(mBuffer);
+}
}; // namespace android
diff --git a/camera/VendorTagDescriptor.cpp b/camera/VendorTagDescriptor.cpp
index d713d2d..24fa912 100644
--- a/camera/VendorTagDescriptor.cpp
+++ b/camera/VendorTagDescriptor.cpp
@@ -660,6 +660,16 @@
return sGlobalVendorTagDescriptorCache;
}
+bool VendorTagDescriptorCache::isVendorCachePresent(metadata_vendor_id_t vendorId) {
+ Mutex::Autolock al(sLock);
+ if ((sGlobalVendorTagDescriptorCache.get() != nullptr) &&
+ (sGlobalVendorTagDescriptorCache->getVendorIdsAndTagDescriptors().find(vendorId) !=
+ sGlobalVendorTagDescriptorCache->getVendorIdsAndTagDescriptors().end())) {
+ return true;
+ }
+ return false;
+}
+
extern "C" {
int vendor_tag_descriptor_get_tag_count(const vendor_tag_ops_t* /*v*/) {
diff --git a/camera/include/camera/CameraMetadata.h b/camera/include/camera/CameraMetadata.h
index 9d1b5c7..83abdb6 100644
--- a/camera/include/camera/CameraMetadata.h
+++ b/camera/include/camera/CameraMetadata.h
@@ -237,6 +237,11 @@
static status_t getTagFromName(const char *name,
const VendorTagDescriptor* vTags, uint32_t *tag);
+ /**
+ * Return the current vendor tag id associated with this metadata.
+ */
+ metadata_vendor_id_t getVendorId();
+
private:
camera_metadata_t *mBuffer;
mutable bool mLocked;
diff --git a/camera/include/camera/VendorTagDescriptor.h b/camera/include/camera/VendorTagDescriptor.h
index b2fbf3a..b3440d5 100644
--- a/camera/include/camera/VendorTagDescriptor.h
+++ b/camera/include/camera/VendorTagDescriptor.h
@@ -249,6 +249,12 @@
*/
static void clearGlobalVendorTagCache();
+ /**
+ * Return true if given vendor id is present in the vendor tag caches, return
+ * false otherwise.
+ */
+ static bool isVendorCachePresent(metadata_vendor_id_t vendorId);
+
};
} /* namespace android */
diff --git a/media/codec2/components/aac/DrcPresModeWrap.cpp b/media/codec2/components/aac/DrcPresModeWrap.cpp
index bee969b..7ce5c9d 100644
--- a/media/codec2/components/aac/DrcPresModeWrap.cpp
+++ b/media/codec2/components/aac/DrcPresModeWrap.cpp
@@ -161,7 +161,7 @@
int newHeavy = mDesHeavy;
if (mDataUpdate) {
- // sanity check
+ // Validation check
if ((mDesTarget < MAX_TARGET_LEVEL) && (mDesTarget != -1)){
mDesTarget = MAX_TARGET_LEVEL; // limit target level to -10 dB or below
newTarget = MAX_TARGET_LEVEL;
@@ -217,7 +217,7 @@
}
else { // handle other used encoder target levels
- // Sanity check: DRC presentation mode is only specified for max. 5.1 channels
+ // Validation check: DRC presentation mode is only specified for max. 5.1 channels
if (mStreamNrAACChan > 6) {
drcPresMode = 0;
}
@@ -308,7 +308,7 @@
} // switch()
} // if (mEncoderTarget == GPM_ENCODER_TARGET_LEVEL)
- // sanity again
+ // Validation check again
if (newHeavy == 1) {
newBoostFactor=127; // not really needed as the same would be done by the decoder anyway
newAttFactor = 127;
diff --git a/media/codec2/sfplugin/CCodecBuffers.cpp b/media/codec2/sfplugin/CCodecBuffers.cpp
index e58a1e4..c49a16c 100644
--- a/media/codec2/sfplugin/CCodecBuffers.cpp
+++ b/media/codec2/sfplugin/CCodecBuffers.cpp
@@ -272,8 +272,6 @@
// The output format can be processed without a registered slot.
if (outputFormat) {
- ALOGD("[%s] popFromStashAndRegister: output format changed to %s",
- mName, outputFormat->debugString().c_str());
updateSkipCutBuffer(outputFormat, entry.notify);
}
@@ -301,6 +299,10 @@
}
if (!entry.notify) {
+ if (outputFormat) {
+ ALOGD("[%s] popFromStashAndRegister: output format changed to %s",
+ mName, outputFormat->debugString().c_str());
+ }
mPending.pop_front();
return DISCARD;
}
@@ -317,6 +319,10 @@
// Append information from the front stash entry to outBuffer.
(*outBuffer)->meta()->setInt64("timeUs", entry.timestamp);
(*outBuffer)->meta()->setInt32("flags", entry.flags);
+ if (outputFormat) {
+ ALOGD("[%s] popFromStashAndRegister: output format changed to %s",
+ mName, outputFormat->debugString().c_str());
+ }
ALOGV("[%s] popFromStashAndRegister: "
"out buffer index = %zu [%p] => %p + %zu (%lld)",
mName, *index, outBuffer->get(),
diff --git a/media/codec2/tests/C2UtilTest.cpp b/media/codec2/tests/C2UtilTest.cpp
index 59cd313..2d66df1 100644
--- a/media/codec2/tests/C2UtilTest.cpp
+++ b/media/codec2/tests/C2UtilTest.cpp
@@ -78,7 +78,7 @@
{ "value2", Enum3Value2 },
{ "value4", Enum3Value4 },
{ "invalid", Invalid } });
- Enum3 e3;
+ Enum3 e3(Invalid);
C2FieldDescriptor::namedValuesFor(e3);
// upper case
diff --git a/media/janitors/OWNERS-codecs b/media/janitors/OWNERS-codecs
new file mode 100644
index 0000000..e201399
--- /dev/null
+++ b/media/janitors/OWNERS-codecs
@@ -0,0 +1,5 @@
+# gerrit owner/approvers for the actual software codec libraries
+# differentiated from plugins connecting those codecs to either omx or codec2 infrastructure
+essick@google.com
+lajos@google.com
+marcone@google.com
diff --git a/media/janitors/README b/media/janitors/README
new file mode 100644
index 0000000..9db8e0e
--- /dev/null
+++ b/media/janitors/README
@@ -0,0 +1,4 @@
+A collection of OWNERS files that we reference from other projects,
+such as the software codecs in directories like external/libavc.
+This is to simplify our owner/approver management across the multiple
+projects related to media.
diff --git a/media/libaaudio/include/aaudio/AAudio.h b/media/libaaudio/include/aaudio/AAudio.h
index c269430..6666788 100644
--- a/media/libaaudio/include/aaudio/AAudio.h
+++ b/media/libaaudio/include/aaudio/AAudio.h
@@ -1037,6 +1037,11 @@
* but still allow queries to the stream to occur from other threads. This often
* happens if you are monitoring stream progress from a UI thread.
*
+ * NOTE: This function is only fully implemented for MMAP streams,
+ * which are low latency streams supported by some devices.
+ * On other "Legacy" streams some audio resources will still be in use
+ * and some callbacks may still be in process after this call.
+ *
* @param stream reference provided by AAudioStreamBuilder_openStream()
* @return {@link #AAUDIO_OK} or a negative error.
*/
diff --git a/media/libaaudio/src/client/AudioStreamInternalCapture.cpp b/media/libaaudio/src/client/AudioStreamInternalCapture.cpp
index d014608..55fc986 100644
--- a/media/libaaudio/src/client/AudioStreamInternalCapture.cpp
+++ b/media/libaaudio/src/client/AudioStreamInternalCapture.cpp
@@ -14,8 +14,6 @@
* limitations under the License.
*/
-#define LOG_TAG (mInService ? "AudioStreamInternalCapture_Service" \
- : "AudioStreamInternalCapture_Client")
//#define LOG_NDEBUG 0
#include <utils/Log.h>
@@ -29,6 +27,14 @@
#define ATRACE_TAG ATRACE_TAG_AUDIO
#include <utils/Trace.h>
+// We do this after the #includes because if a header uses ALOG.
+// it would fail on the reference to mInService.
+#undef LOG_TAG
+// This file is used in both client and server processes.
+// This is needed to make sense of the logs more easily.
+#define LOG_TAG (mInService ? "AudioStreamInternalCapture_Service" \
+ : "AudioStreamInternalCapture_Client")
+
using android::WrappingBuffer;
using namespace aaudio;
diff --git a/media/libaaudio/src/client/AudioStreamInternalPlay.cpp b/media/libaaudio/src/client/AudioStreamInternalPlay.cpp
index 6337b53..b47b472 100644
--- a/media/libaaudio/src/client/AudioStreamInternalPlay.cpp
+++ b/media/libaaudio/src/client/AudioStreamInternalPlay.cpp
@@ -14,8 +14,6 @@
* limitations under the License.
*/
-#define LOG_TAG (mInService ? "AudioStreamInternalPlay_Service" \
- : "AudioStreamInternalPlay_Client")
//#define LOG_NDEBUG 0
#include <utils/Log.h>
@@ -26,6 +24,14 @@
#include "client/AudioStreamInternalPlay.h"
#include "utility/AudioClock.h"
+// We do this after the #includes because if a header uses ALOG.
+// it would fail on the reference to mInService.
+#undef LOG_TAG
+// This file is used in both client and server processes.
+// This is needed to make sense of the logs more easily.
+#define LOG_TAG (mInService ? "AudioStreamInternalPlay_Service" \
+ : "AudioStreamInternalPlay_Client")
+
using android::WrappingBuffer;
using namespace aaudio;
diff --git a/media/libaaudio/src/core/AAudioAudio.cpp b/media/libaaudio/src/core/AAudioAudio.cpp
index 8965875..cfa7221 100644
--- a/media/libaaudio/src/core/AAudioAudio.cpp
+++ b/media/libaaudio/src/core/AAudioAudio.cpp
@@ -255,17 +255,16 @@
if (audioStream != nullptr) {
aaudio_stream_id_t id = audioStream->getId();
ALOGD("%s(s#%u) called ---------------", __func__, id);
- result = audioStream->safeRelease();
- // safeRelease will only fail if called illegally, for example, from a callback.
+ result = audioStream->safeReleaseClose();
+ // safeReleaseClose will only fail if called illegally, for example, from a callback.
// That would result in deleting an active stream, which would cause a crash.
if (result != AAUDIO_OK) {
ALOGW("%s(s#%u) failed. Close it from another thread.",
__func__, id);
} else {
audioStream->unregisterPlayerBase();
- // Mark CLOSED to keep destructors from asserting.
- audioStream->closeFinal();
- delete audioStream;
+ // Allow the stream to be deleted.
+ AudioStreamBuilder::stopUsingStream(audioStream);
}
ALOGD("%s(s#%u) returned %d ---------", __func__, id, result);
}
diff --git a/media/libaaudio/src/core/AudioStream.cpp b/media/libaaudio/src/core/AudioStream.cpp
index 983887b..ac2da57 100644
--- a/media/libaaudio/src/core/AudioStream.cpp
+++ b/media/libaaudio/src/core/AudioStream.cpp
@@ -39,7 +39,7 @@
}
AudioStream::AudioStream()
- : mPlayerBase(new MyPlayerBase(this))
+ : mPlayerBase(new MyPlayerBase())
, mStreamId(AAudio_getNextStreamId())
{
// mThread is a pthread_t of unknown size so we need memset.
@@ -48,6 +48,10 @@
}
AudioStream::~AudioStream() {
+ // Please preserve this log because there have been several bugs related to
+ // AudioStream deletion and late callbacks.
+ ALOGD("%s(s#%u) mPlayerBase strongCount = %d",
+ __func__, getId(), mPlayerBase->getStrongCount());
// If the stream is deleted when OPEN or in use then audio resources will leak.
// This would indicate an internal error. So we want to find this ASAP.
LOG_ALWAYS_FATAL_IF(!(getState() == AAUDIO_STREAM_STATE_CLOSED
@@ -55,8 +59,6 @@
|| getState() == AAUDIO_STREAM_STATE_DISCONNECTED),
"~AudioStream() - still in use, state = %s",
AudioGlobal_convertStreamStateToText(getState()));
-
- mPlayerBase->clearParentReference(); // remove reference to this AudioStream
}
aaudio_result_t AudioStream::open(const AudioStreamBuilder& builder)
@@ -301,18 +303,29 @@
}
aaudio_result_t AudioStream::safeRelease() {
- // This get temporarily unlocked in the release() when joining callback threads.
+ // This get temporarily unlocked in the MMAP release() when joining callback threads.
std::lock_guard<std::mutex> lock(mStreamLock);
if (collidesWithCallback()) {
ALOGE("%s cannot be called from a callback!", __func__);
return AAUDIO_ERROR_INVALID_STATE;
}
- if (getState() == AAUDIO_STREAM_STATE_CLOSING) {
+ if (getState() == AAUDIO_STREAM_STATE_CLOSING) { // already released?
return AAUDIO_OK;
}
return release_l();
}
+aaudio_result_t AudioStream::safeReleaseClose() {
+ // This get temporarily unlocked in the MMAP release() when joining callback threads.
+ std::lock_guard<std::mutex> lock(mStreamLock);
+ if (collidesWithCallback()) {
+ ALOGE("%s cannot be called from a callback!", __func__);
+ return AAUDIO_ERROR_INVALID_STATE;
+ }
+ releaseCloseFinal();
+ return AAUDIO_OK;
+}
+
void AudioStream::setState(aaudio_stream_state_t state) {
ALOGD("%s(s#%d) from %d to %d", __func__, getId(), mState, state);
if (state == mState) {
@@ -523,11 +536,18 @@
}
#if AAUDIO_USE_VOLUME_SHAPER
-android::media::VolumeShaper::Status AudioStream::applyVolumeShaper(
- const android::media::VolumeShaper::Configuration& configuration __unused,
- const android::media::VolumeShaper::Operation& operation __unused) {
- ALOGW("applyVolumeShaper() is not supported");
- return android::media::VolumeShaper::Status::ok();
+::android::binder::Status AudioStream::MyPlayerBase::applyVolumeShaper(
+ const ::android::media::VolumeShaper::Configuration& configuration,
+ const ::android::media::VolumeShaper::Operation& operation) {
+ android::sp<AudioStream> audioStream;
+ {
+ std::lock_guard<std::mutex> lock(mParentLock);
+ audioStream = mParent.promote();
+ }
+ if (audioStream) {
+ return audioStream->applyVolumeShaper(configuration, operation);
+ }
+ return android::NO_ERROR;
}
#endif
@@ -537,26 +557,36 @@
doSetVolume(); // apply this change
}
-AudioStream::MyPlayerBase::MyPlayerBase(AudioStream *parent) : mParent(parent) {
-}
-
-AudioStream::MyPlayerBase::~MyPlayerBase() {
-}
-
-void AudioStream::MyPlayerBase::registerWithAudioManager() {
+void AudioStream::MyPlayerBase::registerWithAudioManager(const android::sp<AudioStream>& parent) {
+ std::lock_guard<std::mutex> lock(mParentLock);
+ mParent = parent;
if (!mRegistered) {
- init(android::PLAYER_TYPE_AAUDIO, AAudioConvert_usageToInternal(mParent->getUsage()));
+ init(android::PLAYER_TYPE_AAUDIO, AAudioConvert_usageToInternal(parent->getUsage()));
mRegistered = true;
}
}
void AudioStream::MyPlayerBase::unregisterWithAudioManager() {
+ std::lock_guard<std::mutex> lock(mParentLock);
if (mRegistered) {
baseDestroy();
mRegistered = false;
}
}
+android::status_t AudioStream::MyPlayerBase::playerSetVolume() {
+ android::sp<AudioStream> audioStream;
+ {
+ std::lock_guard<std::mutex> lock(mParentLock);
+ audioStream = mParent.promote();
+ }
+ if (audioStream) {
+ // No pan and only left volume is taken into account from IPLayer interface
+ audioStream->setDuckAndMuteVolume(mVolumeMultiplierL /* * mPanMultiplierL */);
+ }
+ return android::NO_ERROR;
+}
+
void AudioStream::MyPlayerBase::destroy() {
unregisterWithAudioManager();
}
diff --git a/media/libaaudio/src/core/AudioStream.h b/media/libaaudio/src/core/AudioStream.h
index fb71c36..e0bd9d8 100644
--- a/media/libaaudio/src/core/AudioStream.h
+++ b/media/libaaudio/src/core/AudioStream.h
@@ -25,8 +25,10 @@
#include <binder/Status.h>
#include <utils/StrongPointer.h>
-#include "media/VolumeShaper.h"
-#include "media/PlayerBase.h"
+#include <media/AudioSystem.h>
+#include <media/PlayerBase.h>
+#include <media/VolumeShaper.h>
+
#include "utility/AAudioUtilities.h"
#include "utility/MonotonicCounter.h"
@@ -45,7 +47,8 @@
/**
* AAudio audio stream.
*/
-class AudioStream {
+// By extending AudioDeviceCallback, we also inherit from RefBase.
+class AudioStream : public android::AudioSystem::AudioDeviceCallback {
public:
AudioStream();
@@ -117,6 +120,17 @@
virtual void logOpen();
void logReleaseBufferState();
+ /* Note about naming for "release" and "close" related methods.
+ *
+ * These names are intended to match the public AAudio API.
+ * The original AAudio API had an AAudioStream_close() function that
+ * released the hardware and deleted the stream. That made it difficult
+ * because apps want to release the HW ASAP but are not in a rush to delete
+ * the stream object. So in R we added an AAudioStream_release() function
+ * that just released the hardware.
+ * The AAudioStream_close() method releases if needed and then closes.
+ */
+
/**
* Free any hardware or system resources from the open() call.
* It is safe to call release_l() multiple times.
@@ -126,22 +140,27 @@
return AAUDIO_OK;
}
- aaudio_result_t closeFinal() {
+ /**
+ * Free any resources not already freed by release_l().
+ * Assume release_l() already called.
+ */
+ virtual void close_l() {
+ // Releasing the stream will set the state to CLOSING.
+ assert(getState() == AAUDIO_STREAM_STATE_CLOSING);
+ // setState() prevents a transition from CLOSING to any state other than CLOSED.
// State is checked by destructor.
setState(AAUDIO_STREAM_STATE_CLOSED);
- return AAUDIO_OK;
}
/**
* Release then close the stream.
- * @return AAUDIO_OK or negative error.
*/
- aaudio_result_t releaseCloseFinal() {
- aaudio_result_t result = release_l(); // TODO review locking
- if (result == AAUDIO_OK) {
- result = closeFinal();
+ void releaseCloseFinal() {
+ if (getState() != AAUDIO_STREAM_STATE_CLOSING) { // not already released?
+ // Ignore result and keep closing.
+ (void) release_l();
}
- return result;
+ close_l();
}
// This is only used to identify a stream in the logs without
@@ -328,6 +347,10 @@
*/
bool collidesWithCallback() const;
+ // Implement AudioDeviceCallback
+ void onAudioDeviceUpdate(audio_io_handle_t audioIo,
+ audio_port_handle_t deviceId) override {};
+
// ============== I/O ===========================
// A Stream will only implement read() or write() depending on its direction.
virtual aaudio_result_t write(const void *buffer __unused,
@@ -366,7 +389,7 @@
*/
void registerPlayerBase() {
if (getDirection() == AAUDIO_DIRECTION_OUTPUT) {
- mPlayerBase->registerWithAudioManager();
+ mPlayerBase->registerWithAudioManager(this);
}
}
@@ -395,21 +418,33 @@
*/
aaudio_result_t systemStopFromCallback();
+ /**
+ * Safely RELEASE a stream after taking mStreamLock and checking
+ * to make sure we are not being called from a callback.
+ * @return AAUDIO_OK or a negative error
+ */
aaudio_result_t safeRelease();
+ /**
+ * Safely RELEASE and CLOSE a stream after taking mStreamLock and checking
+ * to make sure we are not being called from a callback.
+ * @return AAUDIO_OK or a negative error
+ */
+ aaudio_result_t safeReleaseClose();
+
protected:
// PlayerBase allows the system to control the stream volume.
class MyPlayerBase : public android::PlayerBase {
public:
- explicit MyPlayerBase(AudioStream *parent);
+ MyPlayerBase() {};
- virtual ~MyPlayerBase();
+ virtual ~MyPlayerBase() = default;
/**
* Register for volume changes and remote control.
*/
- void registerWithAudioManager();
+ void registerWithAudioManager(const android::sp<AudioStream>& parent);
/**
* UnRegister.
@@ -421,8 +456,6 @@
*/
void destroy() override;
- void clearParentReference() { mParent = nullptr; }
-
// Just a stub. The ability to start audio through PlayerBase is being deprecated.
android::status_t playerStart() override {
return android::NO_ERROR;
@@ -438,18 +471,10 @@
return android::NO_ERROR;
}
- android::status_t playerSetVolume() override {
- // No pan and only left volume is taken into account from IPLayer interface
- mParent->setDuckAndMuteVolume(mVolumeMultiplierL /* * mPanMultiplierL */);
- return android::NO_ERROR;
- }
+ android::status_t playerSetVolume() override;
#if AAUDIO_USE_VOLUME_SHAPER
- ::android::binder::Status applyVolumeShaper(
- const ::android::media::VolumeShaper::Configuration& configuration,
- const ::android::media::VolumeShaper::Operation& operation) {
- return mParent->applyVolumeShaper(configuration, operation);
- }
+ ::android::binder::Status applyVolumeShaper();
#endif
aaudio_result_t getResult() {
@@ -457,9 +482,12 @@
}
private:
- AudioStream *mParent;
- aaudio_result_t mResult = AAUDIO_OK;
- bool mRegistered = false;
+ // Use a weak pointer so the AudioStream can be deleted.
+
+ std::mutex mParentLock;
+ android::wp<AudioStream> mParent;
+ aaudio_result_t mResult = AAUDIO_OK;
+ bool mRegistered = false;
};
/**
diff --git a/media/libaaudio/src/core/AudioStreamBuilder.cpp b/media/libaaudio/src/core/AudioStreamBuilder.cpp
index 60dad84..630b289 100644
--- a/media/libaaudio/src/core/AudioStreamBuilder.cpp
+++ b/media/libaaudio/src/core/AudioStreamBuilder.cpp
@@ -63,27 +63,26 @@
static aaudio_result_t builder_createStream(aaudio_direction_t direction,
aaudio_sharing_mode_t sharingMode,
bool tryMMap,
- AudioStream **audioStreamPtr) {
- *audioStreamPtr = nullptr;
+ android::sp<AudioStream> &stream) {
aaudio_result_t result = AAUDIO_OK;
switch (direction) {
case AAUDIO_DIRECTION_INPUT:
if (tryMMap) {
- *audioStreamPtr = new AudioStreamInternalCapture(AAudioBinderClient::getInstance(),
+ stream = new AudioStreamInternalCapture(AAudioBinderClient::getInstance(),
false);
} else {
- *audioStreamPtr = new AudioStreamRecord();
+ stream = new AudioStreamRecord();
}
break;
case AAUDIO_DIRECTION_OUTPUT:
if (tryMMap) {
- *audioStreamPtr = new AudioStreamInternalPlay(AAudioBinderClient::getInstance(),
+ stream = new AudioStreamInternalPlay(AAudioBinderClient::getInstance(),
false);
} else {
- *audioStreamPtr = new AudioStreamTrack();
+ stream = new AudioStreamTrack();
}
break;
@@ -98,7 +97,7 @@
// Fall back to Legacy path if MMAP not available.
// Exact behavior is controlled by MMapPolicy.
aaudio_result_t AudioStreamBuilder::build(AudioStream** streamPtr) {
- AudioStream *audioStream = nullptr;
+
if (streamPtr == nullptr) {
ALOGE("%s() streamPtr is null", __func__);
return AAUDIO_ERROR_NULL;
@@ -171,41 +170,48 @@
setPrivacySensitive(true);
}
- result = builder_createStream(getDirection(), sharingMode, allowMMap, &audioStream);
+ android::sp<AudioStream> audioStream;
+ result = builder_createStream(getDirection(), sharingMode, allowMMap, audioStream);
if (result == AAUDIO_OK) {
// Open the stream using the parameters from the builder.
result = audioStream->open(*this);
- if (result == AAUDIO_OK) {
- *streamPtr = audioStream;
- } else {
+ if (result != AAUDIO_OK) {
bool isMMap = audioStream->isMMap();
- delete audioStream;
- audioStream = nullptr;
-
if (isMMap && allowLegacy) {
ALOGV("%s() MMAP stream did not open so try Legacy path", __func__);
// If MMAP stream failed to open then TRY using a legacy stream.
result = builder_createStream(getDirection(), sharingMode,
- false, &audioStream);
+ false, audioStream);
if (result == AAUDIO_OK) {
result = audioStream->open(*this);
- if (result == AAUDIO_OK) {
- *streamPtr = audioStream;
- } else {
- delete audioStream;
- audioStream = nullptr;
- }
}
}
}
- if (audioStream != nullptr) {
+ if (result == AAUDIO_OK) {
audioStream->logOpen();
- }
+ *streamPtr = startUsingStream(audioStream);
+ } // else audioStream will go out of scope and be deleted
}
return result;
}
+AudioStream *AudioStreamBuilder::startUsingStream(android::sp<AudioStream> &audioStream) {
+ // Increment the smart pointer so it will not get deleted when
+ // we pass it to the C caller and it goes out of scope.
+ // The C code cannot hold a smart pointer so we increment the reference
+ // count to indicate that the C app owns a reference.
+ audioStream->incStrong(nullptr);
+ return audioStream.get();
+}
+
+void AudioStreamBuilder::stopUsingStream(AudioStream *stream) {
+ // Undo the effect of startUsingStream()
+ android::sp<AudioStream> spAudioStream(stream);
+ ALOGV("%s() strongCount = %d", __func__, spAudioStream->getStrongCount());
+ spAudioStream->decStrong(nullptr);
+}
+
aaudio_result_t AudioStreamBuilder::validate() const {
// Check for values that are ridiculously out of range to prevent math overflow exploits.
diff --git a/media/libaaudio/src/core/AudioStreamBuilder.h b/media/libaaudio/src/core/AudioStreamBuilder.h
index d5fb80d..9f93341 100644
--- a/media/libaaudio/src/core/AudioStreamBuilder.h
+++ b/media/libaaudio/src/core/AudioStreamBuilder.h
@@ -108,9 +108,16 @@
virtual aaudio_result_t validate() const override;
+
void logParameters() const;
+ // Mark the stream so it can be deleted.
+ static void stopUsingStream(AudioStream *stream);
+
private:
+ // Extract a raw pointer that we can pass to a 'C' app.
+ static AudioStream *startUsingStream(android::sp<AudioStream> &spAudioStream);
+
bool mSharingModeMatchRequired = false; // must match sharing mode requested
aaudio_performance_mode_t mPerformanceMode = AAUDIO_PERFORMANCE_MODE_NONE;
diff --git a/media/libaaudio/src/legacy/AudioStreamLegacy.cpp b/media/libaaudio/src/legacy/AudioStreamLegacy.cpp
index c062882..33c1bf5 100644
--- a/media/libaaudio/src/legacy/AudioStreamLegacy.cpp
+++ b/media/libaaudio/src/legacy/AudioStreamLegacy.cpp
@@ -34,8 +34,7 @@
using namespace aaudio;
AudioStreamLegacy::AudioStreamLegacy()
- : AudioStream()
- , mDeviceCallback(new StreamDeviceCallback(this)) {
+ : AudioStream() {
}
AudioStreamLegacy::~AudioStreamLegacy() {
@@ -163,7 +162,11 @@
}
void AudioStreamLegacy::forceDisconnect(bool errorCallbackEnabled) {
- if (getState() != AAUDIO_STREAM_STATE_DISCONNECTED) {
+ // There is no need to disconnect if already in these states.
+ if (getState() != AAUDIO_STREAM_STATE_DISCONNECTED
+ && getState() != AAUDIO_STREAM_STATE_CLOSING
+ && getState() != AAUDIO_STREAM_STATE_CLOSED
+ ) {
setState(AAUDIO_STREAM_STATE_DISCONNECTED);
if (errorCallbackEnabled) {
maybeCallErrorCallback(AAUDIO_ERROR_DISCONNECTED);
@@ -205,24 +208,30 @@
return AAudioConvert_androidToAAudioResult(status);
}
-void AudioStreamLegacy::onAudioDeviceUpdate(audio_port_handle_t deviceId)
-{
+void AudioStreamLegacy::onAudioDeviceUpdate(audio_io_handle_t /* audioIo */,
+ audio_port_handle_t deviceId) {
// Device routing is a common source of errors and DISCONNECTS.
- // Please leave this log in place.
- ALOGD("%s() devId %d => %d", __func__, (int) getDeviceId(), (int)deviceId);
- if (getDeviceId() != AAUDIO_UNSPECIFIED && getDeviceId() != deviceId &&
- getState() != AAUDIO_STREAM_STATE_DISCONNECTED) {
+ // Please leave this log in place. If there is a bug then this might
+ // get called after the stream has been deleted so log before we
+ // touch the stream object.
+ ALOGD("%s(deviceId = %d)", __func__, (int)deviceId);
+ if (getDeviceId() != AAUDIO_UNSPECIFIED
+ && getDeviceId() != deviceId
+ && getState() != AAUDIO_STREAM_STATE_DISCONNECTED
+ ) {
// Note that isDataCallbackActive() is affected by state so call it before DISCONNECTING.
// If we have a data callback and the stream is active, then ask the data callback
// to DISCONNECT and call the error callback.
if (isDataCallbackActive()) {
- ALOGD("onAudioDeviceUpdate() request DISCONNECT in data callback due to device change");
+ ALOGD("%s() request DISCONNECT in data callback, device %d => %d",
+ __func__, (int) getDeviceId(), (int) deviceId);
// If the stream is stopped before the data callback has a chance to handle the
// request then the requestStop() and requestPause() methods will handle it after
// the callback has stopped.
mRequestDisconnect.request();
} else {
- ALOGD("onAudioDeviceUpdate() DISCONNECT the stream now");
+ ALOGD("%s() DISCONNECT the stream now, device %d => %d",
+ __func__, (int) getDeviceId(), (int) deviceId);
forceDisconnect();
}
}
diff --git a/media/libaaudio/src/legacy/AudioStreamLegacy.h b/media/libaaudio/src/legacy/AudioStreamLegacy.h
index 9c24b2b..fefe6e0 100644
--- a/media/libaaudio/src/legacy/AudioStreamLegacy.h
+++ b/media/libaaudio/src/legacy/AudioStreamLegacy.h
@@ -87,29 +87,13 @@
protected:
- class StreamDeviceCallback : public android::AudioSystem::AudioDeviceCallback
- {
- public:
-
- StreamDeviceCallback(AudioStreamLegacy *parent) : mParent(parent) {}
- virtual ~StreamDeviceCallback() {}
-
- virtual void onAudioDeviceUpdate(audio_io_handle_t audioIo __unused,
- audio_port_handle_t deviceId) {
- if (mParent != nullptr) {
- mParent->onAudioDeviceUpdate(deviceId);
- }
- }
-
- AudioStreamLegacy *mParent;
- };
-
aaudio_result_t getBestTimestamp(clockid_t clockId,
int64_t *framePosition,
int64_t *timeNanoseconds,
android::ExtendedTimestamp *extendedTimestamp);
- void onAudioDeviceUpdate(audio_port_handle_t deviceId);
+ void onAudioDeviceUpdate(audio_io_handle_t audioIo,
+ audio_port_handle_t deviceId) override;
/*
* Check to see whether a callback thread has requested a disconnected.
@@ -140,7 +124,6 @@
int32_t mBlockAdapterBytesPerFrame = 0;
aaudio_wrapping_frames_t mPositionWhenStarting = 0;
int32_t mCallbackBufferSize = 0;
- const android::sp<StreamDeviceCallback> mDeviceCallback;
AtomicRequestor mRequestDisconnect;
diff --git a/media/libaaudio/src/legacy/AudioStreamRecord.cpp b/media/libaaudio/src/legacy/AudioStreamRecord.cpp
index 3bfa2b7..43b63d6 100644
--- a/media/libaaudio/src/legacy/AudioStreamRecord.cpp
+++ b/media/libaaudio/src/legacy/AudioStreamRecord.cpp
@@ -279,7 +279,7 @@
: (aaudio_session_id_t) mAudioRecord->getSessionId();
setSessionId(actualSessionId);
- mAudioRecord->addAudioDeviceCallback(mDeviceCallback);
+ mAudioRecord->addAudioDeviceCallback(this);
return AAUDIO_OK;
}
@@ -288,16 +288,24 @@
// TODO add close() or release() to AudioFlinger's AudioRecord API.
// Then call it from here
if (getState() != AAUDIO_STREAM_STATE_CLOSING) {
- mAudioRecord->removeAudioDeviceCallback(mDeviceCallback);
+ mAudioRecord->removeAudioDeviceCallback(this);
logReleaseBufferState();
- mAudioRecord.clear();
- mFixedBlockWriter.close();
+ // Data callbacks may still be running!
return AudioStream::release_l();
} else {
return AAUDIO_OK; // already released
}
}
+void AudioStreamRecord::close_l() {
+ mAudioRecord.clear();
+ // Do not close mFixedBlockWriter because a data callback
+ // thread might still be running if someone else has a reference
+ // to mAudioRecord.
+ // It has a unique_ptr to its buffer so it will clean up by itself.
+ AudioStream::close_l();
+}
+
const void * AudioStreamRecord::maybeConvertDeviceData(const void *audioData, int32_t numFrames) {
if (mFormatConversionBufferFloat.get() != nullptr) {
LOG_ALWAYS_FATAL_IF(numFrames > mFormatConversionBufferSizeInFrames,
diff --git a/media/libaaudio/src/legacy/AudioStreamRecord.h b/media/libaaudio/src/legacy/AudioStreamRecord.h
index c5944c7..e4ef1c0 100644
--- a/media/libaaudio/src/legacy/AudioStreamRecord.h
+++ b/media/libaaudio/src/legacy/AudioStreamRecord.h
@@ -39,6 +39,7 @@
aaudio_result_t open(const AudioStreamBuilder & builder) override;
aaudio_result_t release_l() override;
+ void close_l() override;
aaudio_result_t requestStart() override;
aaudio_result_t requestStop() override;
diff --git a/media/libaaudio/src/legacy/AudioStreamTrack.cpp b/media/libaaudio/src/legacy/AudioStreamTrack.cpp
index 0427220..55907b3 100644
--- a/media/libaaudio/src/legacy/AudioStreamTrack.cpp
+++ b/media/libaaudio/src/legacy/AudioStreamTrack.cpp
@@ -119,7 +119,8 @@
// that is some multiple of the burst size.
notificationFrames = 0 - DEFAULT_BURSTS_PER_BUFFER_CAPACITY;
} else {
- notificationFrames = builder.getFramesPerDataCallback();
+ // To avoid glitching, let AudioFlinger pick the optimal burst size.
+ notificationFrames = 0;
}
}
mCallbackBufferSize = builder.getFramesPerDataCallback();
@@ -221,7 +222,7 @@
mInitialBufferCapacity = getBufferCapacity();
mInitialFramesPerBurst = getFramesPerBurst();
- mAudioTrack->addAudioDeviceCallback(mDeviceCallback);
+ mAudioTrack->addAudioDeviceCallback(this);
// Update performance mode based on the actual stream flags.
// For example, if the sample rate is not allowed then you won't get a FAST track.
@@ -250,19 +251,26 @@
aaudio_result_t AudioStreamTrack::release_l() {
if (getState() != AAUDIO_STREAM_STATE_CLOSING) {
- mAudioTrack->removeAudioDeviceCallback(mDeviceCallback);
+ status_t err = mAudioTrack->removeAudioDeviceCallback(this);
+ ALOGE_IF(err, "%s() removeAudioDeviceCallback returned %d", __func__, err);
logReleaseBufferState();
- // TODO Investigate why clear() causes a hang in test_various.cpp
- // if I call close() from a data callback.
- // But the same thing in AudioRecord is OK!
- // mAudioTrack.clear();
- mFixedBlockReader.close();
+ // Data callbacks may still be running!
return AudioStream::release_l();
} else {
return AAUDIO_OK; // already released
}
}
+void AudioStreamTrack::close_l() {
+ // Stop callbacks before deleting mFixedBlockReader memory.
+ mAudioTrack.clear();
+ // Do not close mFixedBlockReader because a data callback
+ // thread might still be running if someone else has a reference
+ // to mAudioRecord.
+ // It has a unique_ptr to its buffer so it will clean up by itself.
+ AudioStream::close_l();
+}
+
void AudioStreamTrack::processCallback(int event, void *info) {
switch (event) {
diff --git a/media/libaaudio/src/legacy/AudioStreamTrack.h b/media/libaaudio/src/legacy/AudioStreamTrack.h
index 93a1ff4..6334f66 100644
--- a/media/libaaudio/src/legacy/AudioStreamTrack.h
+++ b/media/libaaudio/src/legacy/AudioStreamTrack.h
@@ -42,6 +42,7 @@
aaudio_result_t open(const AudioStreamBuilder & builder) override;
aaudio_result_t release_l() override;
+ void close_l() override;
aaudio_result_t requestStart() override;
aaudio_result_t requestPause() override;
diff --git a/media/libaudioclient/AudioRecord.cpp b/media/libaudioclient/AudioRecord.cpp
index 509e063..d6671e3 100644
--- a/media/libaudioclient/AudioRecord.cpp
+++ b/media/libaudioclient/AudioRecord.cpp
@@ -742,6 +742,8 @@
void *iMemPointer;
audio_track_cblk_t* cblk;
status_t status;
+ static const int32_t kMaxCreateAttempts = 3;
+ int32_t remainingAttempts = kMaxCreateAttempts;
if (audioFlinger == 0) {
ALOGE("%s(%d): Could not get audioflinger", __func__, mPortId);
@@ -803,15 +805,24 @@
input.sessionId = mSessionId;
originalSessionId = mSessionId;
- record = audioFlinger->createRecord(input,
- output,
- &status);
+ do {
+ record = audioFlinger->createRecord(input, output, &status);
+ if (status == NO_ERROR) {
+ break;
+ }
+ if (status != FAILED_TRANSACTION || --remainingAttempts <= 0) {
+ ALOGE("%s(%d): AudioFlinger could not create record track, status: %d",
+ __func__, mPortId, status);
+ goto exit;
+ }
+ // FAILED_TRANSACTION happens under very specific conditions causing a state mismatch
+ // between audio policy manager and audio flinger during the input stream open sequence
+ // and can be recovered by retrying.
+ // Leave time for race condition to clear before retrying and randomize delay
+ // to reduce the probability of concurrent retries in locked steps.
+ usleep((20 + rand() % 30) * 10000);
+ } while (1);
- if (status != NO_ERROR) {
- ALOGE("%s(%d): AudioFlinger could not create record track, status: %d",
- __func__, mPortId, status);
- goto exit;
- }
ALOG_ASSERT(record != 0);
// AudioFlinger now owns the reference to the I/O handle,
diff --git a/media/libaudioclient/AudioSystem.cpp b/media/libaudioclient/AudioSystem.cpp
index 4bbd4f7..edb0889 100644
--- a/media/libaudioclient/AudioSystem.cpp
+++ b/media/libaudioclient/AudioSystem.cpp
@@ -47,8 +47,9 @@
record_config_callback AudioSystem::gRecordConfigCallback = NULL;
// Required to be held while calling into gSoundTriggerCaptureStateListener.
+class CaptureStateListenerImpl;
Mutex gSoundTriggerCaptureStateListenerLock;
-sp<AudioSystem::CaptureStateListener> gSoundTriggerCaptureStateListener = nullptr;
+sp<CaptureStateListenerImpl> gSoundTriggerCaptureStateListener = nullptr;
// establish binder interface to AudioFlinger service
const sp<IAudioFlinger> AudioSystem::get_audio_flinger()
@@ -1690,42 +1691,54 @@
class CaptureStateListenerImpl : public media::BnCaptureStateListener,
public IBinder::DeathRecipient {
public:
+ CaptureStateListenerImpl(
+ const sp<IAudioPolicyService>& aps,
+ const sp<AudioSystem::CaptureStateListener>& listener)
+ : mAps(aps), mListener(listener) {}
+
+ void init() {
+ bool active;
+ status_t status = mAps->registerSoundTriggerCaptureStateListener(this, &active);
+ if (status != NO_ERROR) {
+ mListener->onServiceDied();
+ return;
+ }
+ mListener->onStateChanged(active);
+ IInterface::asBinder(mAps)->linkToDeath(this);
+ }
+
binder::Status setCaptureState(bool active) override {
Mutex::Autolock _l(gSoundTriggerCaptureStateListenerLock);
- gSoundTriggerCaptureStateListener->onStateChanged(active);
+ mListener->onStateChanged(active);
return binder::Status::ok();
}
void binderDied(const wp<IBinder>&) override {
Mutex::Autolock _l(gSoundTriggerCaptureStateListenerLock);
- gSoundTriggerCaptureStateListener->onServiceDied();
+ mListener->onServiceDied();
gSoundTriggerCaptureStateListener = nullptr;
}
+
+private:
+ // Need this in order to keep the death receipent alive.
+ sp<IAudioPolicyService> mAps;
+ sp<AudioSystem::CaptureStateListener> mListener;
};
status_t AudioSystem::registerSoundTriggerCaptureStateListener(
const sp<CaptureStateListener>& listener) {
+ LOG_ALWAYS_FATAL_IF(listener == nullptr);
+
const sp<IAudioPolicyService>& aps =
AudioSystem::get_audio_policy_service();
if (aps == 0) {
return PERMISSION_DENIED;
}
- sp<CaptureStateListenerImpl> wrapper = new CaptureStateListenerImpl();
-
Mutex::Autolock _l(gSoundTriggerCaptureStateListenerLock);
+ gSoundTriggerCaptureStateListener = new CaptureStateListenerImpl(aps, listener);
+ gSoundTriggerCaptureStateListener->init();
- bool active;
- status_t status =
- aps->registerSoundTriggerCaptureStateListener(wrapper, &active);
- if (status != NO_ERROR) {
- listener->onServiceDied();
- return NO_ERROR;
- }
- gSoundTriggerCaptureStateListener = listener;
- listener->onStateChanged(active);
- sp<IBinder> binder = IInterface::asBinder(aps);
- binder->linkToDeath(wrapper);
return NO_ERROR;
}
diff --git a/media/libaudioclient/AudioTrack.cpp b/media/libaudioclient/AudioTrack.cpp
index 807aa13..41af78c 100644
--- a/media/libaudioclient/AudioTrack.cpp
+++ b/media/libaudioclient/AudioTrack.cpp
@@ -210,7 +210,11 @@
return NO_ERROR;
}
-AudioTrack::AudioTrack()
+AudioTrack::AudioTrack() : AudioTrack("" /*opPackageName*/)
+{
+}
+
+AudioTrack::AudioTrack(const std::string& opPackageName)
: mStatus(NO_INIT),
mState(STATE_STOPPED),
mPreviousPriority(ANDROID_PRIORITY_NORMAL),
@@ -218,6 +222,7 @@
mPausedPosition(0),
mSelectedDeviceId(AUDIO_PORT_HANDLE_NONE),
mRoutedDeviceId(AUDIO_PORT_HANDLE_NONE),
+ mOpPackageName(opPackageName),
mAudioTrackCallback(new AudioTrackCallback())
{
mAttributes.content_type = AUDIO_CONTENT_TYPE_UNKNOWN;
@@ -244,12 +249,14 @@
const audio_attributes_t* pAttributes,
bool doNotReconnect,
float maxRequiredSpeed,
- audio_port_handle_t selectedDeviceId)
+ audio_port_handle_t selectedDeviceId,
+ const std::string& opPackageName)
: mStatus(NO_INIT),
mState(STATE_STOPPED),
mPreviousPriority(ANDROID_PRIORITY_NORMAL),
mPreviousSchedulingGroup(SP_DEFAULT),
mPausedPosition(0),
+ mOpPackageName(opPackageName),
mAudioTrackCallback(new AudioTrackCallback())
{
mAttributes = AUDIO_ATTRIBUTES_INITIALIZER;
@@ -277,13 +284,15 @@
pid_t pid,
const audio_attributes_t* pAttributes,
bool doNotReconnect,
- float maxRequiredSpeed)
+ float maxRequiredSpeed,
+ const std::string& opPackageName)
: mStatus(NO_INIT),
mState(STATE_STOPPED),
mPreviousPriority(ANDROID_PRIORITY_NORMAL),
mPreviousSchedulingGroup(SP_DEFAULT),
mPausedPosition(0),
mSelectedDeviceId(AUDIO_PORT_HANDLE_NONE),
+ mOpPackageName(opPackageName),
mAudioTrackCallback(new AudioTrackCallback())
{
mAttributes = AUDIO_ATTRIBUTES_INITIALIZER;
@@ -1555,6 +1564,7 @@
input.selectedDeviceId = mSelectedDeviceId;
input.sessionId = mSessionId;
input.audioTrackCallback = mAudioTrackCallback;
+ input.opPackageName = mOpPackageName;
IAudioFlinger::CreateTrackOutput output;
diff --git a/media/libaudioclient/include/media/AudioTrack.h b/media/libaudioclient/include/media/AudioTrack.h
index 0dbd842..a9946da 100644
--- a/media/libaudioclient/include/media/AudioTrack.h
+++ b/media/libaudioclient/include/media/AudioTrack.h
@@ -26,6 +26,8 @@
#include <media/Modulo.h>
#include <utils/threads.h>
+#include <string>
+
#include "android/media/BnAudioTrackCallback.h"
#include "android/media/IAudioTrackCallback.h"
@@ -177,6 +179,8 @@
*/
AudioTrack();
+ AudioTrack(const std::string& opPackageName);
+
/* Creates an AudioTrack object and registers it with AudioFlinger.
* Once created, the track needs to be started before it can be used.
* Unspecified values are set to appropriate default values.
@@ -258,7 +262,8 @@
const audio_attributes_t* pAttributes = NULL,
bool doNotReconnect = false,
float maxRequiredSpeed = 1.0f,
- audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE);
+ audio_port_handle_t selectedDeviceId = AUDIO_PORT_HANDLE_NONE,
+ const std::string& opPackageName = "");
/* Creates an audio track and registers it with AudioFlinger.
* With this constructor, the track is configured for static buffer mode.
@@ -288,7 +293,8 @@
pid_t pid = -1,
const audio_attributes_t* pAttributes = NULL,
bool doNotReconnect = false,
- float maxRequiredSpeed = 1.0f);
+ float maxRequiredSpeed = 1.0f,
+ const std::string& opPackageName = "");
/* Terminates the AudioTrack and unregisters it from AudioFlinger.
* Also destroys all resources associated with the AudioTrack.
@@ -1236,6 +1242,8 @@
sp<media::VolumeHandler> mVolumeHandler;
+ const std::string mOpPackageName;
+
private:
class DeathNotifier : public IBinder::DeathRecipient {
public:
diff --git a/media/libaudioclient/include/media/IAudioFlinger.h b/media/libaudioclient/include/media/IAudioFlinger.h
index b950d0f..a01b681 100644
--- a/media/libaudioclient/include/media/IAudioFlinger.h
+++ b/media/libaudioclient/include/media/IAudioFlinger.h
@@ -35,6 +35,7 @@
#include <system/audio_policy.h>
#include <utils/String8.h>
#include <media/MicrophoneInfo.h>
+#include <string>
#include <vector>
#include "android/media/IAudioRecord.h"
@@ -85,6 +86,11 @@
speed = parcel->readFloat();
audioTrackCallback = interface_cast<media::IAudioTrackCallback>(
parcel->readStrongBinder());
+ const char* opPackageNamePtr = parcel->readCString();
+ if (opPackageNamePtr == nullptr) {
+ return FAILED_TRANSACTION;
+ }
+ opPackageName = opPackageNamePtr;
/* input/output arguments*/
(void)parcel->read(&flags, sizeof(audio_output_flags_t));
@@ -109,6 +115,7 @@
(void)parcel->writeInt32(notificationsPerBuffer);
(void)parcel->writeFloat(speed);
(void)parcel->writeStrongBinder(IInterface::asBinder(audioTrackCallback));
+ (void)parcel->writeCString(opPackageName.c_str());
/* input/output arguments*/
(void)parcel->write(&flags, sizeof(audio_output_flags_t));
@@ -127,6 +134,7 @@
uint32_t notificationsPerBuffer;
float speed;
sp<media::IAudioTrackCallback> audioTrackCallback;
+ std::string opPackageName;
/* input/output */
audio_output_flags_t flags;
diff --git a/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp b/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp
index 9192a31..80e2b87 100644
--- a/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp
+++ b/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp
@@ -37,7 +37,7 @@
EffectsFactoryHalHidl::EffectsFactoryHalHidl(sp<IEffectsFactory> effectsFactory)
: ConversionHelperHidl("EffectsFactory") {
- ALOG_ASSERT(effectsFactory != nullptr, "Provided IDevicesFactory service is NULL");
+ ALOG_ASSERT(effectsFactory != nullptr, "Provided IEffectsFactory service is NULL");
mEffectsFactory = effectsFactory;
}
diff --git a/media/libaudiohal/impl/EffectsFactoryHalHidl.h b/media/libaudiohal/impl/EffectsFactoryHalHidl.h
index dece1bb..5fa85e7 100644
--- a/media/libaudiohal/impl/EffectsFactoryHalHidl.h
+++ b/media/libaudiohal/impl/EffectsFactoryHalHidl.h
@@ -54,6 +54,8 @@
virtual status_t dumpEffects(int fd);
+ virtual float getHalVersion() { return MAJOR_VERSION + (float)MINOR_VERSION / 10; }
+
status_t allocateBuffer(size_t size, sp<EffectBufferHalInterface>* buffer) override;
status_t mirrorBuffer(void* external, size_t size,
sp<EffectBufferHalInterface>* buffer) override;
diff --git a/media/libaudiohal/include/media/audiohal/EffectsFactoryHalInterface.h b/media/libaudiohal/include/media/audiohal/EffectsFactoryHalInterface.h
index 3a76f9f..9fb56ae 100644
--- a/media/libaudiohal/include/media/audiohal/EffectsFactoryHalInterface.h
+++ b/media/libaudiohal/include/media/audiohal/EffectsFactoryHalInterface.h
@@ -46,6 +46,8 @@
virtual status_t dumpEffects(int fd) = 0;
+ virtual float getHalVersion() = 0;
+
static sp<EffectsFactoryHalInterface> create();
virtual status_t allocateBuffer(size_t size, sp<EffectBufferHalInterface>* buffer) = 0;
diff --git a/media/libmedia/IMediaPlayerService.cpp b/media/libmedia/IMediaPlayerService.cpp
index bd18a40..11005c6 100644
--- a/media/libmedia/IMediaPlayerService.cpp
+++ b/media/libmedia/IMediaPlayerService.cpp
@@ -62,11 +62,13 @@
}
virtual sp<IMediaPlayer> create(
- const sp<IMediaPlayerClient>& client, audio_session_t audioSessionId) {
+ const sp<IMediaPlayerClient>& client, audio_session_t audioSessionId,
+ const std::string opPackageName) {
Parcel data, reply;
data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor());
data.writeStrongBinder(IInterface::asBinder(client));
data.writeInt32(audioSessionId);
+ data.writeCString(opPackageName.c_str());
remote()->transact(CREATE, data, &reply);
return interface_cast<IMediaPlayer>(reply.readStrongBinder());
@@ -127,7 +129,12 @@
sp<IMediaPlayerClient> client =
interface_cast<IMediaPlayerClient>(data.readStrongBinder());
audio_session_t audioSessionId = (audio_session_t) data.readInt32();
- sp<IMediaPlayer> player = create(client, audioSessionId);
+ const char* opPackageName = data.readCString();
+ if (opPackageName == nullptr) {
+ return FAILED_TRANSACTION;
+ }
+ std::string opPackageNameStr(opPackageName);
+ sp<IMediaPlayer> player = create(client, audioSessionId, opPackageNameStr);
reply->writeStrongBinder(IInterface::asBinder(player));
return NO_ERROR;
} break;
diff --git a/media/libmedia/include/media/IMediaPlayerService.h b/media/libmedia/include/media/IMediaPlayerService.h
index f2e2060..a4207eb 100644
--- a/media/libmedia/include/media/IMediaPlayerService.h
+++ b/media/libmedia/include/media/IMediaPlayerService.h
@@ -28,6 +28,8 @@
#include <media/IMediaPlayerClient.h>
#include <media/IMediaMetadataRetriever.h>
+#include <string>
+
namespace android {
class IMediaPlayer;
@@ -47,7 +49,8 @@
virtual sp<IMediaRecorder> createMediaRecorder(const String16 &opPackageName) = 0;
virtual sp<IMediaMetadataRetriever> createMetadataRetriever() = 0;
virtual sp<IMediaPlayer> create(const sp<IMediaPlayerClient>& client,
- audio_session_t audioSessionId = AUDIO_SESSION_ALLOCATE) = 0;
+ audio_session_t audioSessionId = AUDIO_SESSION_ALLOCATE,
+ const std::string opPackage = "") = 0;
virtual sp<IMediaCodecList> getCodecList() const = 0;
// Connects to a remote display.
diff --git a/media/libmedia/include/media/mediaplayer.h b/media/libmedia/include/media/mediaplayer.h
index d0a8e38..71c0bc5 100644
--- a/media/libmedia/include/media/mediaplayer.h
+++ b/media/libmedia/include/media/mediaplayer.h
@@ -33,6 +33,8 @@
#include <utils/KeyedVector.h>
#include <utils/String8.h>
+#include <string>
+
struct ANativeWindow;
namespace android {
@@ -178,7 +180,10 @@
KEY_PARAMETER_PLAYBACK_RATE_PERMILLE = 1300, // set only
// Set a Parcel containing the value of a parcelled Java AudioAttribute instance
- KEY_PARAMETER_AUDIO_ATTRIBUTES = 1400 // set only
+ KEY_PARAMETER_AUDIO_ATTRIBUTES = 1400, // set only
+
+ // Set a Parcel containing the values of RTP attribute
+ KEY_PARAMETER_RTP_ATTRIBUTES = 2000 // set only
};
// Keep INVOKE_ID_* in sync with MediaPlayer.java.
@@ -206,7 +211,7 @@
public virtual IMediaDeathNotifier
{
public:
- MediaPlayer();
+ MediaPlayer(const std::string opPackageName = "");
~MediaPlayer();
void died();
void disconnect();
@@ -310,6 +315,7 @@
float mSendLevel;
struct sockaddr_in mRetransmitEndpoint;
bool mRetransmitEndpointValid;
+ const std::string mOpPackageName;
};
}; // namespace android
diff --git a/media/libmedia/mediaplayer.cpp b/media/libmedia/mediaplayer.cpp
index 1b89fc7..30c5006 100644
--- a/media/libmedia/mediaplayer.cpp
+++ b/media/libmedia/mediaplayer.cpp
@@ -41,7 +41,7 @@
using media::VolumeShaper;
-MediaPlayer::MediaPlayer()
+MediaPlayer::MediaPlayer(const std::string opPackageName) : mOpPackageName(opPackageName)
{
ALOGV("constructor");
mListener = NULL;
@@ -152,7 +152,7 @@
if (url != NULL) {
const sp<IMediaPlayerService> service(getMediaPlayerService());
if (service != 0) {
- sp<IMediaPlayer> player(service->create(this, mAudioSessionId));
+ sp<IMediaPlayer> player(service->create(this, mAudioSessionId, mOpPackageName));
if ((NO_ERROR != doSetRetransmitEndpoint(player)) ||
(NO_ERROR != player->setDataSource(httpService, url, headers))) {
player.clear();
@@ -169,7 +169,7 @@
status_t err = UNKNOWN_ERROR;
const sp<IMediaPlayerService> service(getMediaPlayerService());
if (service != 0) {
- sp<IMediaPlayer> player(service->create(this, mAudioSessionId));
+ sp<IMediaPlayer> player(service->create(this, mAudioSessionId, mOpPackageName));
if ((NO_ERROR != doSetRetransmitEndpoint(player)) ||
(NO_ERROR != player->setDataSource(fd, offset, length))) {
player.clear();
@@ -185,7 +185,7 @@
status_t err = UNKNOWN_ERROR;
const sp<IMediaPlayerService> service(getMediaPlayerService());
if (service != 0) {
- sp<IMediaPlayer> player(service->create(this, mAudioSessionId));
+ sp<IMediaPlayer> player(service->create(this, mAudioSessionId, mOpPackageName));
if ((NO_ERROR != doSetRetransmitEndpoint(player)) ||
(NO_ERROR != player->setDataSource(source))) {
player.clear();
@@ -201,7 +201,7 @@
status_t err = UNKNOWN_ERROR;
const sp<IMediaPlayerService> service(getMediaPlayerService());
if (service != 0) {
- sp<IMediaPlayer> player(service->create(this, mAudioSessionId));
+ sp<IMediaPlayer> player(service->create(this, mAudioSessionId, mOpPackageName));
if ((NO_ERROR != doSetRetransmitEndpoint(player)) ||
(NO_ERROR != player->setDataSource(rtpParams))) {
player.clear();
diff --git a/media/libmediaplayerservice/MediaPlayerService.cpp b/media/libmediaplayerservice/MediaPlayerService.cpp
index 555f459..4d90d98 100644
--- a/media/libmediaplayerservice/MediaPlayerService.cpp
+++ b/media/libmediaplayerservice/MediaPlayerService.cpp
@@ -480,14 +480,14 @@
}
sp<IMediaPlayer> MediaPlayerService::create(const sp<IMediaPlayerClient>& client,
- audio_session_t audioSessionId)
+ audio_session_t audioSessionId, std::string opPackageName)
{
pid_t pid = IPCThreadState::self()->getCallingPid();
int32_t connId = android_atomic_inc(&mNextConnId);
sp<Client> c = new Client(
this, pid, connId, client, audioSessionId,
- IPCThreadState::self()->getCallingUid());
+ IPCThreadState::self()->getCallingUid(), opPackageName);
ALOGV("Create new client(%d) from pid %d, uid %d, ", connId, pid,
IPCThreadState::self()->getCallingUid());
@@ -733,7 +733,8 @@
MediaPlayerService::Client::Client(
const sp<MediaPlayerService>& service, pid_t pid,
int32_t connId, const sp<IMediaPlayerClient>& client,
- audio_session_t audioSessionId, uid_t uid)
+ audio_session_t audioSessionId, uid_t uid, const std::string& opPackageName)
+ : mOpPackageName(opPackageName)
{
ALOGV("Client(%d) constructor", connId);
mPid = pid;
@@ -922,7 +923,7 @@
if (!p->hardwareOutput()) {
mAudioOutput = new AudioOutput(mAudioSessionId, IPCThreadState::self()->getCallingUid(),
- mPid, mAudioAttributes, mAudioDeviceUpdatedListener);
+ mPid, mAudioAttributes, mAudioDeviceUpdatedListener, mOpPackageName);
static_cast<MediaPlayerInterface*>(p.get())->setAudioSink(mAudioOutput);
}
@@ -1772,7 +1773,8 @@
#undef LOG_TAG
#define LOG_TAG "AudioSink"
MediaPlayerService::AudioOutput::AudioOutput(audio_session_t sessionId, uid_t uid, int pid,
- const audio_attributes_t* attr, const sp<AudioSystem::AudioDeviceCallback>& deviceCallback)
+ const audio_attributes_t* attr, const sp<AudioSystem::AudioDeviceCallback>& deviceCallback,
+ const std::string& opPackageName)
: mCallback(NULL),
mCallbackCookie(NULL),
mCallbackData(NULL),
@@ -1793,7 +1795,8 @@
mSelectedDeviceId(AUDIO_PORT_HANDLE_NONE),
mRoutedDeviceId(AUDIO_PORT_HANDLE_NONE),
mDeviceCallbackEnabled(false),
- mDeviceCallback(deviceCallback)
+ mDeviceCallback(deviceCallback),
+ mOpPackageName(opPackageName)
{
ALOGV("AudioOutput(%d)", sessionId);
if (attr != NULL) {
@@ -2187,7 +2190,8 @@
mAttributes,
doNotReconnect,
1.0f, // default value for maxRequiredSpeed
- mSelectedDeviceId);
+ mSelectedDeviceId,
+ mOpPackageName);
} else {
// TODO: Due to buffer memory concerns, we use a max target playback speed
// based on mPlaybackRate at the time of open (instead of kMaxRequiredSpeed),
@@ -2215,7 +2219,8 @@
mAttributes,
doNotReconnect,
targetSpeed,
- mSelectedDeviceId);
+ mSelectedDeviceId,
+ mOpPackageName);
}
// Set caller name so it can be logged in destructor.
// MediaMetricsConstants.h: AMEDIAMETRICS_PROP_CALLERNAME_VALUE_MEDIA
diff --git a/media/libmediaplayerservice/MediaPlayerService.h b/media/libmediaplayerservice/MediaPlayerService.h
index 3d596a5..b2f1b9b 100644
--- a/media/libmediaplayerservice/MediaPlayerService.h
+++ b/media/libmediaplayerservice/MediaPlayerService.h
@@ -19,6 +19,7 @@
#define ANDROID_MEDIAPLAYERSERVICE_H
#include <arpa/inet.h>
+#include <string>
#include <utils/threads.h>
#include <utils/Errors.h>
@@ -81,7 +82,8 @@
uid_t uid,
int pid,
const audio_attributes_t * attr,
- const sp<AudioSystem::AudioDeviceCallback>& deviceCallback);
+ const sp<AudioSystem::AudioDeviceCallback>& deviceCallback,
+ const std::string& opPackageName);
virtual ~AudioOutput();
virtual bool ready() const { return mTrack != 0; }
@@ -178,6 +180,7 @@
bool mDeviceCallbackEnabled;
wp<AudioSystem::AudioDeviceCallback> mDeviceCallback;
mutable Mutex mLock;
+ const std::string mOpPackageName;
// static variables below not protected by mutex
static bool mIsOnEmulator;
@@ -235,7 +238,8 @@
virtual sp<IMediaMetadataRetriever> createMetadataRetriever();
virtual sp<IMediaPlayer> create(const sp<IMediaPlayerClient>& client,
- audio_session_t audioSessionId);
+ audio_session_t audioSessionId,
+ const std::string opPackageName);
virtual sp<IMediaCodecList> getCodecList() const;
@@ -411,7 +415,8 @@
int32_t connId,
const sp<IMediaPlayerClient>& client,
audio_session_t audioSessionId,
- uid_t uid);
+ uid_t uid,
+ const std::string& opPackageName);
Client();
virtual ~Client();
@@ -468,6 +473,7 @@
bool mRetransmitEndpointValid;
sp<Client> mNextClient;
sp<MediaPlayerBase::Listener> mListener;
+ const std::string mOpPackageName;
// Metadata filters.
media::Metadata::Filter mMetadataAllow; // protected by mLock
diff --git a/media/libmediaplayerservice/StagefrightRecorder.cpp b/media/libmediaplayerservice/StagefrightRecorder.cpp
index 7e5fe56..02ae456 100644
--- a/media/libmediaplayerservice/StagefrightRecorder.cpp
+++ b/media/libmediaplayerservice/StagefrightRecorder.cpp
@@ -17,6 +17,9 @@
//#define LOG_NDEBUG 0
#define LOG_TAG "StagefrightRecorder"
#include <inttypes.h>
+// TODO/workaround: including base logging now as it conflicts with ADebug.h
+// and it must be included first.
+#include <android-base/logging.h>
#include <utils/Log.h>
#include "WebmWriter.h"
@@ -575,12 +578,14 @@
mVideoBitRate = bitRate;
// A new bitrate(TMMBR) should be applied on runtime as well if OutputFormat is RTP_AVP
- if (mOutputFormat == OUTPUT_FORMAT_RTP_AVP && mStarted && mPauseStartTimeUs == 0) {
+ if (mOutputFormat == OUTPUT_FORMAT_RTP_AVP) {
// Regular I frames may overload the network so we reduce the bitrate to allow
// margins for the I frame overruns.
// Still send requested bitrate (TMMBR) in the reply (TMMBN).
const float coefficient = 0.8f;
mVideoBitRate = (bitRate * coefficient) / 1000 * 1000;
+ }
+ if (mOutputFormat == OUTPUT_FORMAT_RTP_AVP && mStarted && mPauseStartTimeUs == 0) {
mVideoEncoderSource->setEncodingBitrate(mVideoBitRate);
ARTPWriter* rtpWriter = static_cast<ARTPWriter*>(mWriter.get());
rtpWriter->setTMMBNInfo(mOpponentID, bitRate);
@@ -1967,10 +1972,6 @@
format->setInt32("stride", stride);
format->setInt32("slice-height", sliceHeight);
format->setInt32("color-format", colorFormat);
- if (mOutputFormat == OUTPUT_FORMAT_RTP_AVP) {
- // This indicates that a raw image provided to encoder needs to be rotated.
- format->setInt32("rotation-degrees", mRotationDegrees);
- }
} else {
format->setInt32("width", mVideoWidth);
format->setInt32("height", mVideoHeight);
@@ -1988,6 +1989,11 @@
}
}
+ if (mOutputFormat == OUTPUT_FORMAT_RTP_AVP) {
+ // This indicates that a raw image provided to encoder needs to be rotated.
+ format->setInt32("rotation-degrees", mRotationDegrees);
+ }
+
format->setInt32("bitrate", mVideoBitRate);
format->setInt32("bitrate-mode", mVideoBitRateMode);
format->setInt32("frame-rate", mFrameRate);
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
index 4e7daa5..47362ef 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
@@ -1702,6 +1702,12 @@
updateRebufferingTimer(false /* stopping */, false /* exiting */);
}
+void NuPlayer::setTargetBitrate(int bitrate) {
+ if (mSource != NULL) {
+ mSource->setTargetBitrate(bitrate);
+ }
+}
+
void NuPlayer::onPause() {
updatePlaybackTimer(true /* stopping */, "onPause");
@@ -2868,6 +2874,27 @@
}
break;
}
+ case NuPlayer::RTPSource::RTP_QUALITY:
+ {
+ int32_t feedbackType, bitrate;
+ int32_t highestSeqNum, baseSeqNum, prevExpected;
+ int32_t numBufRecv, prevNumBufRecv;
+ CHECK(msg->findInt32("feedback-type", &feedbackType));
+ CHECK(msg->findInt32("bit-rate", &bitrate));
+ CHECK(msg->findInt32("highest-seq-num", &highestSeqNum));
+ CHECK(msg->findInt32("base-seq-num", &baseSeqNum));
+ CHECK(msg->findInt32("prev-expected", &prevExpected));
+ CHECK(msg->findInt32("num-buf-recv", &numBufRecv));
+ CHECK(msg->findInt32("prev-num-buf-recv", &prevNumBufRecv));
+ in.writeInt32(feedbackType);
+ in.writeInt32(bitrate);
+ in.writeInt32(highestSeqNum);
+ in.writeInt32(baseSeqNum);
+ in.writeInt32(prevExpected);
+ in.writeInt32(numBufRecv);
+ in.writeInt32(prevNumBufRecv);
+ break;
+ }
case NuPlayer::RTPSource::RTP_CVO:
{
int32_t cvo;
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h
index 0105248..adb7075 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h
@@ -102,6 +102,8 @@
void updateInternalTimers();
+ void setTargetBitrate(int bitrate /* bps */);
+
protected:
virtual ~NuPlayer();
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
index 2d82944..2a50fc2 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
@@ -817,7 +817,11 @@
}
status_t NuPlayerDriver::setParameter(
- int /* key */, const Parcel & /* request */) {
+ int key, const Parcel &request ) {
+ if (key == KEY_PARAMETER_RTP_ATTRIBUTES) {
+ mPlayer->setTargetBitrate(request.readInt32());
+ return OK;
+ }
return INVALID_OPERATION;
}
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h
index eb39870..bf6b539 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h
@@ -132,6 +132,8 @@
virtual void setOffloadAudio(bool /* offload */) {}
+ virtual void setTargetBitrate(int32_t) {}
+
// Modular DRM
virtual status_t prepareDrm(
const uint8_t /*uuid*/[16], const Vector<uint8_t> &/*drmSessionId*/,
diff --git a/media/libmediaplayerservice/nuplayer/RTPSource.cpp b/media/libmediaplayerservice/nuplayer/RTPSource.cpp
index a6601cd..b1901e8 100644
--- a/media/libmediaplayerservice/nuplayer/RTPSource.cpp
+++ b/media/libmediaplayerservice/nuplayer/RTPSource.cpp
@@ -114,7 +114,8 @@
// index(i) should be started from 1. 0 is reserved for [root]
mRTPConn->addStream(sockRtp, sockRtcp, desc, i + 1, notify, false);
mRTPConn->setSelfID(info->mSelfID);
- mRTPConn->setMinMaxBitrate(kMinVideoBitrate, info->mAS * 1000 /* kbps */);
+ mRTPConn->setJbTime(
+ (info->mJbTimeMs <= 3000 && info->mJbTimeMs >= 40) ? info->mJbTimeMs : 300);
info->mRTPSocket = sockRtp;
info->mRTCPSocket = sockRtcp;
@@ -135,11 +136,16 @@
if (info->mIsAudio) {
mAudioTrack = source;
+ info->mTimeScale = 16000;
} else {
mVideoTrack = source;
+ info->mTimeScale = 90000;
}
info->mSource = source;
+ info->mRTPTime = 0;
+ info->mNormalPlaytimeUs = 0;
+ info->mNPTMappingValid = false;
}
if (mInPreparationPhase) {
@@ -280,20 +286,19 @@
}
int32_t cvo;
- if ((*accessUnit) != NULL && (*accessUnit)->meta()->findInt32("cvo", &cvo)) {
- if (cvo != mLastCVOUpdated) {
- sp<AMessage> msg = new AMessage();
- msg->setInt32("payload-type", NuPlayer::RTPSource::RTP_CVO);
- msg->setInt32("cvo", cvo);
+ if ((*accessUnit) != NULL && (*accessUnit)->meta()->findInt32("cvo", &cvo) &&
+ cvo != mLastCVOUpdated) {
+ sp<AMessage> msg = new AMessage();
+ msg->setInt32("payload-type", NuPlayer::RTPSource::RTP_CVO);
+ msg->setInt32("cvo", cvo);
- sp<AMessage> notify = dupNotify();
- notify->setInt32("what", kWhatIMSRxNotice);
- notify->setMessage("message", msg);
- notify->post();
+ sp<AMessage> notify = dupNotify();
+ notify->setInt32("what", kWhatIMSRxNotice);
+ notify->setMessage("message", msg);
+ notify->post();
- ALOGV("notify cvo updated (%d)->(%d) to upper layer", mLastCVOUpdated, cvo);
- mLastCVOUpdated = cvo;
- }
+ ALOGV("notify cvo updated (%d)->(%d) to upper layer", mLastCVOUpdated, cvo);
+ mLastCVOUpdated = cvo;
}
return finalResult;
@@ -347,6 +352,11 @@
schedulePollBuffering();
}
+bool NuPlayer::RTPSource::isRealTime() const {
+ ALOGD("RTPSource::isRealTime=%d", true);
+ return true;
+}
+
void NuPlayer::RTPSource::onMessageReceived(const sp<AMessage> &msg) {
ALOGV("onMessageReceived =%d", msg->what());
@@ -429,7 +439,6 @@
source->queueAccessUnit(accessUnit);
break;
}
- */
int64_t nptUs =
((double)rtpTime - (double)info->mRTPTime)
@@ -437,7 +446,8 @@
* 1000000ll
+ info->mNormalPlaytimeUs;
- accessUnit->meta()->setInt64("timeUs", nptUs);
+ */
+ accessUnit->meta()->setInt64("timeUs", ALooper::GetNowUs());
source->queueAccessUnit(accessUnit);
}
@@ -490,6 +500,10 @@
}
}
+void NuPlayer::RTPSource::setTargetBitrate(int32_t bitrate) {
+ mRTPConn->setTargetBitrate(bitrate);
+}
+
void NuPlayer::RTPSource::onTimeUpdate(int32_t trackIndex, uint32_t rtpTime, uint64_t ntpTime) {
ALOGV("onTimeUpdate track %d, rtpTime = 0x%08x, ntpTime = %#016llx",
trackIndex, rtpTime, (long long)ntpTime);
@@ -656,6 +670,7 @@
newTrackInfo.mIsAudio = isAudioKey;
mTracks.push(newTrackInfo);
info = &mTracks.editTop();
+ info->mJbTimeMs = 300;
}
if (key == "rtp-param-mime-type") {
@@ -698,6 +713,8 @@
} else if (key == "rtp-param-set-socket-network") {
int64_t networkHandle = atoll(value);
setSocketNetwork(networkHandle);
+ } else if (key == "rtp-param-jitter-buffer-time") {
+ info->mJbTimeMs = atoi(value);
}
return OK;
diff --git a/media/libmediaplayerservice/nuplayer/RTPSource.h b/media/libmediaplayerservice/nuplayer/RTPSource.h
index 5085a7e..fb2d3b9 100644
--- a/media/libmediaplayerservice/nuplayer/RTPSource.h
+++ b/media/libmediaplayerservice/nuplayer/RTPSource.h
@@ -52,6 +52,9 @@
const String8& rtpParams);
enum {
+ RTP_FIRST_PACKET = 100,
+ RTCP_FIRST_PACKET = 101,
+ RTP_QUALITY = 102,
RTCP_TSFB = 205,
RTCP_PSFB = 206,
RTP_CVO = 300,
@@ -77,8 +80,12 @@
int64_t seekTimeUs,
MediaPlayerSeekMode mode = MediaPlayerSeekMode::SEEK_PREVIOUS_SYNC) override;
+ virtual bool isRealTime() const;
+
void onMessageReceived(const sp<AMessage> &msg);
+ virtual void setTargetBitrate(int32_t bitrate) override;
+
protected:
virtual ~RTPSource();
@@ -95,7 +102,6 @@
};
const int64_t kBufferingPollIntervalUs = 1000000ll;
- const int32_t kMinVideoBitrate = 192000; /* bps */
enum State {
DISCONNECTED,
@@ -123,6 +129,8 @@
int32_t mTimeScale;
int32_t mAS;
+ /* RTP jitter buffer time in milliseconds */
+ uint32_t mJbTimeMs;
/* Unique ID indicates itself */
uint32_t mSelfID;
/* extmap:<value> for CVO will be set to here */
diff --git a/media/libmediatranscoding/Android.bp b/media/libmediatranscoding/Android.bp
index 29ed65a..b7bad7f 100644
--- a/media/libmediatranscoding/Android.bp
+++ b/media/libmediatranscoding/Android.bp
@@ -14,11 +14,8 @@
* limitations under the License.
*/
-// AIDL interfaces of MediaTranscoding.
-aidl_interface {
- name: "mediatranscoding_aidl_interface",
- unstable: true,
- local_include_dir: "aidl",
+filegroup {
+ name: "libmediatranscoding_aidl",
srcs: [
"aidl/android/media/IMediaTranscodingService.aidl",
"aidl/android/media/ITranscodingClient.aidl",
@@ -34,6 +31,15 @@
"aidl/android/media/TranscodingResultParcel.aidl",
"aidl/android/media/TranscodingTestConfig.aidl",
],
+ path: "aidl",
+}
+
+// AIDL interfaces of MediaTranscoding.
+aidl_interface {
+ name: "mediatranscoding_aidl_interface",
+ unstable: true,
+ local_include_dir: "aidl",
+ srcs: [":libmediatranscoding_aidl"],
backend:
{
java: {
diff --git a/media/libmediatranscoding/transcoder/MediaSampleReaderNDK.cpp b/media/libmediatranscoding/transcoder/MediaSampleReaderNDK.cpp
index 6a00a10..53d567e 100644
--- a/media/libmediatranscoding/transcoder/MediaSampleReaderNDK.cpp
+++ b/media/libmediatranscoding/transcoder/MediaSampleReaderNDK.cpp
@@ -22,7 +22,6 @@
#include <algorithm>
#include <cmath>
-#include <vector>
namespace android {
@@ -47,12 +46,6 @@
}
auto sampleReader = std::shared_ptr<MediaSampleReaderNDK>(new MediaSampleReaderNDK(extractor));
- status = sampleReader->init();
- if (status != AMEDIA_OK) {
- LOG(ERROR) << "MediaSampleReaderNDK::init returned error: " << status;
- return nullptr;
- }
-
return sampleReader;
}
@@ -60,39 +53,42 @@
: mExtractor(extractor), mTrackCount(AMediaExtractor_getTrackCount(mExtractor)) {
if (mTrackCount > 0) {
mTrackCursors.resize(mTrackCount);
- mTrackCursors.resize(mTrackCount);
}
}
-media_status_t MediaSampleReaderNDK::init() {
- for (size_t trackIndex = 0; trackIndex < mTrackCount; trackIndex++) {
- media_status_t status = AMediaExtractor_selectTrack(mExtractor, trackIndex);
- if (status != AMEDIA_OK) {
- LOG(ERROR) << "AMediaExtractor_selectTrack returned error: " << status;
- return status;
- }
- }
-
- mExtractorTrackIndex = AMediaExtractor_getSampleTrackIndex(mExtractor);
- if (mExtractorTrackIndex >= 0) {
- mTrackCursors[mExtractorTrackIndex].current.set(mExtractorSampleIndex,
- AMediaExtractor_getSampleTime(mExtractor));
- } else if (mTrackCount > 0) {
- // The extractor track index is only allowed to be invalid if there are no tracks.
- LOG(ERROR) << "Track index " << mExtractorTrackIndex << " is invalid for track count "
- << mTrackCount;
- return AMEDIA_ERROR_MALFORMED;
- }
-
- return AMEDIA_OK;
-}
-
MediaSampleReaderNDK::~MediaSampleReaderNDK() {
if (mExtractor != nullptr) {
AMediaExtractor_delete(mExtractor);
}
}
+void MediaSampleReaderNDK::advanceTrack_l(int trackIndex) {
+ if (!mEnforceSequentialAccess) {
+ // Note: Positioning the extractor before advancing the track is needed for two reasons:
+ // 1. To enable multiple advances without explicitly letting the extractor catch up.
+ // 2. To prevent the extractor from being farther than "next".
+ (void)moveToTrack_l(trackIndex);
+ }
+
+ SampleCursor& cursor = mTrackCursors[trackIndex];
+ cursor.previous = cursor.current;
+ cursor.current = cursor.next;
+ cursor.next.reset();
+
+ if (mEnforceSequentialAccess && trackIndex == mExtractorTrackIndex) {
+ while (advanceExtractor_l()) {
+ SampleCursor& cursor = mTrackCursors[mExtractorTrackIndex];
+ if (cursor.current.isSet && cursor.current.index == mExtractorSampleIndex) {
+ if (mExtractorTrackIndex != trackIndex) {
+ mTrackSignals[mExtractorTrackIndex].notify_all();
+ }
+ break;
+ }
+ }
+ }
+ return;
+}
+
bool MediaSampleReaderNDK::advanceExtractor_l() {
// Reset the "next" sample time whenever the extractor advances past a sample that is current,
// to ensure that "next" is appropriately updated when the extractor advances over the next
@@ -103,6 +99,10 @@
}
if (!AMediaExtractor_advance(mExtractor)) {
+ mEosReached = true;
+ for (auto it = mTrackSignals.begin(); it != mTrackSignals.end(); ++it) {
+ it->second.notify_all();
+ }
return false;
}
@@ -117,6 +117,7 @@
cursor.next.set(mExtractorSampleIndex, AMediaExtractor_getSampleTime(mExtractor));
}
}
+
return true;
}
@@ -150,38 +151,15 @@
return AMEDIA_OK;
}
-void MediaSampleReaderNDK::advanceTrack(int trackIndex) {
- std::scoped_lock lock(mExtractorMutex);
-
- if (trackIndex < 0 || trackIndex >= mTrackCount) {
- LOG(ERROR) << "Invalid trackIndex " << trackIndex << " for trackCount " << mTrackCount;
- return;
- }
-
- // Note: Positioning the extractor before advancing the track is needed for two reasons:
- // 1. To enable multiple advances without explicitly letting the extractor catch up.
- // 2. To prevent the extractor from being farther than "next".
- (void)positionExtractorForTrack_l(trackIndex);
-
- SampleCursor& cursor = mTrackCursors[trackIndex];
- cursor.previous = cursor.current;
- cursor.current = cursor.next;
- cursor.next.reset();
-}
-
-media_status_t MediaSampleReaderNDK::positionExtractorForTrack_l(int trackIndex) {
- media_status_t status = AMEDIA_OK;
- const SampleCursor& cursor = mTrackCursors[trackIndex];
-
- // Seek backwards if the extractor is ahead of the current time.
- if (cursor.current.isSet && mExtractorSampleIndex > cursor.current.index) {
- status = seekExtractorBackwards_l(cursor.current.timeStampUs, trackIndex,
- cursor.current.index);
+media_status_t MediaSampleReaderNDK::moveToSample_l(SamplePosition& pos, int trackIndex) {
+ // Seek backwards if the extractor is ahead of the sample.
+ if (pos.isSet && mExtractorSampleIndex > pos.index) {
+ media_status_t status = seekExtractorBackwards_l(pos.timeStampUs, trackIndex, pos.index);
if (status != AMEDIA_OK) return status;
}
- // Advance until extractor points to the current sample.
- while (!(cursor.current.isSet && cursor.current.index == mExtractorSampleIndex)) {
+ // Advance until extractor points to the sample.
+ while (!(pos.isSet && pos.index == mExtractorSampleIndex)) {
if (!advanceExtractor_l()) {
return AMEDIA_ERROR_END_OF_STREAM;
}
@@ -190,28 +168,129 @@
return AMEDIA_OK;
}
-media_status_t MediaSampleReaderNDK::getEstimatedBitrateForTrack(int trackIndex, int32_t* bitrate) {
+media_status_t MediaSampleReaderNDK::moveToTrack_l(int trackIndex) {
+ return moveToSample_l(mTrackCursors[trackIndex].current, trackIndex);
+}
+
+media_status_t MediaSampleReaderNDK::waitForTrack_l(int trackIndex,
+ std::unique_lock<std::mutex>& lockHeld) {
+ while (trackIndex != mExtractorTrackIndex && !mEosReached && mEnforceSequentialAccess) {
+ mTrackSignals[trackIndex].wait(lockHeld);
+ }
+
+ if (mEosReached) {
+ return AMEDIA_ERROR_END_OF_STREAM;
+ }
+ return AMEDIA_OK;
+}
+
+media_status_t MediaSampleReaderNDK::primeExtractorForTrack_l(
+ int trackIndex, std::unique_lock<std::mutex>& lockHeld) {
+ if (mExtractorTrackIndex < 0) {
+ mExtractorTrackIndex = AMediaExtractor_getSampleTrackIndex(mExtractor);
+ if (mExtractorTrackIndex < 0) {
+ return AMEDIA_ERROR_END_OF_STREAM;
+ }
+ mTrackCursors[mExtractorTrackIndex].current.set(mExtractorSampleIndex,
+ AMediaExtractor_getSampleTime(mExtractor));
+ }
+
+ if (mEnforceSequentialAccess) {
+ return waitForTrack_l(trackIndex, lockHeld);
+ } else {
+ return moveToTrack_l(trackIndex);
+ }
+}
+
+media_status_t MediaSampleReaderNDK::selectTrack(int trackIndex) {
std::scoped_lock lock(mExtractorMutex);
- media_status_t status = AMEDIA_OK;
if (trackIndex < 0 || trackIndex >= mTrackCount) {
LOG(ERROR) << "Invalid trackIndex " << trackIndex << " for trackCount " << mTrackCount;
return AMEDIA_ERROR_INVALID_PARAMETER;
+ } else if (mTrackSignals.find(trackIndex) != mTrackSignals.end()) {
+ LOG(ERROR) << "TrackIndex " << trackIndex << " already selected";
+ return AMEDIA_ERROR_INVALID_PARAMETER;
+ } else if (mExtractorTrackIndex >= 0) {
+ LOG(ERROR) << "Tracks must be selected before sample reading begins.";
+ return AMEDIA_ERROR_UNSUPPORTED;
+ }
+
+ media_status_t status = AMediaExtractor_selectTrack(mExtractor, trackIndex);
+ if (status != AMEDIA_OK) {
+ LOG(ERROR) << "AMediaExtractor_selectTrack returned error: " << status;
+ return status;
+ }
+
+ mTrackSignals.emplace(std::piecewise_construct, std::forward_as_tuple(trackIndex),
+ std::forward_as_tuple());
+ return AMEDIA_OK;
+}
+
+media_status_t MediaSampleReaderNDK::setEnforceSequentialAccess(bool enforce) {
+ std::scoped_lock lock(mExtractorMutex);
+
+ if (mEnforceSequentialAccess && !enforce) {
+ // If switching from enforcing to not enforcing sequential access there may be threads
+ // waiting that needs to be woken up.
+ for (auto it = mTrackSignals.begin(); it != mTrackSignals.end(); ++it) {
+ it->second.notify_all();
+ }
+ } else if (!mEnforceSequentialAccess && enforce && mExtractorTrackIndex >= 0) {
+ // If switching from not enforcing to enforcing sequential access the extractor needs to be
+ // positioned for the track farthest behind so that it won't get stuck waiting.
+ struct {
+ SamplePosition* pos = nullptr;
+ int trackIndex = -1;
+ } earliestSample;
+
+ for (int trackIndex = 0; trackIndex < mTrackCount; ++trackIndex) {
+ SamplePosition& lastKnownTrackPosition = mTrackCursors[trackIndex].current.isSet
+ ? mTrackCursors[trackIndex].current
+ : mTrackCursors[trackIndex].previous;
+
+ if (lastKnownTrackPosition.isSet) {
+ if (earliestSample.pos == nullptr ||
+ earliestSample.pos->index > lastKnownTrackPosition.index) {
+ earliestSample.pos = &lastKnownTrackPosition;
+ earliestSample.trackIndex = trackIndex;
+ }
+ }
+ }
+
+ if (earliestSample.pos == nullptr) {
+ LOG(ERROR) << "No known sample position found";
+ return AMEDIA_ERROR_UNKNOWN;
+ }
+
+ media_status_t status = moveToSample_l(*earliestSample.pos, earliestSample.trackIndex);
+ if (status != AMEDIA_OK) return status;
+
+ while (!(mTrackCursors[mExtractorTrackIndex].current.isSet &&
+ mTrackCursors[mExtractorTrackIndex].current.index == mExtractorSampleIndex)) {
+ if (!advanceExtractor_l()) {
+ return AMEDIA_ERROR_END_OF_STREAM;
+ }
+ }
+ }
+
+ mEnforceSequentialAccess = enforce;
+ return AMEDIA_OK;
+}
+
+media_status_t MediaSampleReaderNDK::getEstimatedBitrateForTrack(int trackIndex, int32_t* bitrate) {
+ std::scoped_lock lock(mExtractorMutex);
+ media_status_t status = AMEDIA_OK;
+
+ if (mTrackSignals.find(trackIndex) == mTrackSignals.end()) {
+ LOG(ERROR) << "Track is not selected.";
+ return AMEDIA_ERROR_INVALID_PARAMETER;
} else if (bitrate == nullptr) {
LOG(ERROR) << "bitrate pointer is NULL.";
return AMEDIA_ERROR_INVALID_PARAMETER;
- }
-
- // Rewind the extractor and sample from the beginning of the file.
- if (mExtractorSampleIndex > 0) {
- status = AMediaExtractor_seekTo(mExtractor, 0, AMEDIAEXTRACTOR_SEEK_PREVIOUS_SYNC);
- if (status != AMEDIA_OK) {
- LOG(ERROR) << "Unable to reset extractor: " << status;
- return status;
- }
-
- mExtractorTrackIndex = AMediaExtractor_getSampleTrackIndex(mExtractor);
- mExtractorSampleIndex = 0;
+ } else if (mExtractorTrackIndex >= 0) {
+ LOG(ERROR) << "getEstimatedBitrateForTrack must be called before sample reading begins.";
+ return AMEDIA_ERROR_UNSUPPORTED;
}
// Sample the track.
@@ -222,7 +301,7 @@
int64_t lastSampleTimeUs = 0;
do {
- if (mExtractorTrackIndex == trackIndex) {
+ if (AMediaExtractor_getSampleTrackIndex(mExtractor) == trackIndex) {
lastSampleTimeUs = AMediaExtractor_getSampleTime(mExtractor);
if (totalSampleSize == 0) {
firstSampleTimeUs = lastSampleTimeUs;
@@ -231,7 +310,15 @@
lastSampleSize = AMediaExtractor_getSampleSize(mExtractor);
totalSampleSize += lastSampleSize;
}
- } while ((lastSampleTimeUs - firstSampleTimeUs) < kSamplingDurationUs && advanceExtractor_l());
+ } while ((lastSampleTimeUs - firstSampleTimeUs) < kSamplingDurationUs &&
+ AMediaExtractor_advance(mExtractor));
+
+ // Reset the extractor to the beginning.
+ status = AMediaExtractor_seekTo(mExtractor, 0, AMEDIAEXTRACTOR_SEEK_PREVIOUS_SYNC);
+ if (status != AMEDIA_OK) {
+ LOG(ERROR) << "Unable to reset extractor: " << status;
+ return status;
+ }
int64_t durationUs = 0;
const int64_t sampledDurationUs = lastSampleTimeUs - firstSampleTimeUs;
@@ -263,17 +350,17 @@
}
media_status_t MediaSampleReaderNDK::getSampleInfoForTrack(int trackIndex, MediaSampleInfo* info) {
- std::scoped_lock lock(mExtractorMutex);
+ std::unique_lock<std::mutex> lock(mExtractorMutex);
- if (trackIndex < 0 || trackIndex >= mTrackCount) {
- LOG(ERROR) << "Invalid trackIndex " << trackIndex << " for trackCount " << mTrackCount;
+ if (mTrackSignals.find(trackIndex) == mTrackSignals.end()) {
+ LOG(ERROR) << "Track not selected.";
return AMEDIA_ERROR_INVALID_PARAMETER;
} else if (info == nullptr) {
LOG(ERROR) << "MediaSampleInfo pointer is NULL.";
return AMEDIA_ERROR_INVALID_PARAMETER;
}
- media_status_t status = positionExtractorForTrack_l(trackIndex);
+ media_status_t status = primeExtractorForTrack_l(trackIndex, lock);
if (status == AMEDIA_OK) {
info->presentationTimeUs = AMediaExtractor_getSampleTime(mExtractor);
info->flags = AMediaExtractor_getSampleFlags(mExtractor);
@@ -283,24 +370,25 @@
info->flags = SAMPLE_FLAG_END_OF_STREAM;
info->size = 0;
}
-
return status;
}
media_status_t MediaSampleReaderNDK::readSampleDataForTrack(int trackIndex, uint8_t* buffer,
size_t bufferSize) {
- std::scoped_lock lock(mExtractorMutex);
+ std::unique_lock<std::mutex> lock(mExtractorMutex);
- if (trackIndex < 0 || trackIndex >= mTrackCount) {
- LOG(ERROR) << "Invalid trackIndex " << trackIndex << " for trackCount " << mTrackCount;
+ if (mTrackSignals.find(trackIndex) == mTrackSignals.end()) {
+ LOG(ERROR) << "Track not selected.";
return AMEDIA_ERROR_INVALID_PARAMETER;
} else if (buffer == nullptr) {
LOG(ERROR) << "buffer pointer is NULL";
return AMEDIA_ERROR_INVALID_PARAMETER;
}
- media_status_t status = positionExtractorForTrack_l(trackIndex);
- if (status != AMEDIA_OK) return status;
+ media_status_t status = primeExtractorForTrack_l(trackIndex, lock);
+ if (status != AMEDIA_OK) {
+ return status;
+ }
ssize_t sampleSize = AMediaExtractor_getSampleSize(mExtractor);
if (bufferSize < sampleSize) {
@@ -314,9 +402,21 @@
return AMEDIA_ERROR_INVALID_PARAMETER;
}
+ advanceTrack_l(trackIndex);
+
return AMEDIA_OK;
}
+void MediaSampleReaderNDK::advanceTrack(int trackIndex) {
+ std::scoped_lock lock(mExtractorMutex);
+
+ if (mTrackSignals.find(trackIndex) != mTrackSignals.end()) {
+ advanceTrack_l(trackIndex);
+ } else {
+ LOG(ERROR) << "Trying to advance a track that is not selected (#" << trackIndex << ")";
+ }
+}
+
AMediaFormat* MediaSampleReaderNDK::getFileFormat() {
return AMediaExtractor_getFileFormat(mExtractor);
}
diff --git a/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp b/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp
index 58e2066..92ce60a 100644
--- a/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp
@@ -92,6 +92,7 @@
if (mState == STARTED) {
abortTranscodeLoop();
+ mMediaSampleReader->setEnforceSequentialAccess(false);
mTranscodingThread.join();
mOutputQueue->abort(); // Wake up any threads waiting for samples.
mState = STOPPED;
diff --git a/media/libmediatranscoding/transcoder/MediaTranscoder.cpp b/media/libmediatranscoding/transcoder/MediaTranscoder.cpp
index ed702db..fbed5c2 100644
--- a/media/libmediatranscoding/transcoder/MediaTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/MediaTranscoder.cpp
@@ -133,6 +133,12 @@
mTracksAdded.insert(transcoder);
if (mTracksAdded.size() == mTrackTranscoders.size()) {
+ // Enable sequential access mode on the sample reader to achieve optimal read performance.
+ // This has to wait until all tracks have delivered their output formats and the sample
+ // writer is started. Otherwise the tracks will not get their output sample queues drained
+ // and the transcoder could hang due to one track running out of buffers and blocking the
+ // other tracks from reading source samples before they could output their formats.
+ mSampleReader->setEnforceSequentialAccess(true);
LOG(INFO) << "Starting sample writer.";
bool started = mSampleWriter->start();
if (!started) {
@@ -229,6 +235,12 @@
return AMEDIA_ERROR_INVALID_PARAMETER;
}
+ media_status_t status = mSampleReader->selectTrack(trackIndex);
+ if (status != AMEDIA_OK) {
+ LOG(ERROR) << "Unable to select track " << trackIndex;
+ return status;
+ }
+
std::shared_ptr<MediaTrackTranscoder> transcoder;
std::shared_ptr<AMediaFormat> format;
@@ -270,7 +282,7 @@
format = std::shared_ptr<AMediaFormat>(mergedFormat, &AMediaFormat_delete);
}
- media_status_t status = transcoder->configure(mSampleReader, trackIndex, format);
+ transcoder->configure(mSampleReader, trackIndex, format);
if (status != AMEDIA_OK) {
LOG(ERROR) << "Configure track transcoder for track #" << trackIndex << " returned error "
<< status;
@@ -344,6 +356,7 @@
}
mSampleWriter->stop();
+ mSampleReader->setEnforceSequentialAccess(false);
for (auto& transcoder : mTrackTranscoders) {
transcoder->stop();
}
diff --git a/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp b/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp
index e2cc6b6..e7c0271 100644
--- a/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp
@@ -142,8 +142,6 @@
LOG(ERROR) << "Output queue aborted";
return AMEDIA_ERROR_IO;
}
-
- mMediaSampleReader->advanceTrack(mTrackIndex);
}
if (mStopRequested && !mEosFromSource) {
diff --git a/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp b/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
index 5702627..b0bf59f 100644
--- a/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
@@ -326,8 +326,6 @@
mStatus = status;
return;
}
-
- mMediaSampleReader->advanceTrack(mTrackIndex);
} else {
LOG(DEBUG) << "EOS from source.";
mEosFromSource = true;
diff --git a/media/libmediatranscoding/transcoder/benchmark/MediaSampleReaderBenchmark.cpp b/media/libmediatranscoding/transcoder/benchmark/MediaSampleReaderBenchmark.cpp
index a651fa2..f0b9304 100644
--- a/media/libmediatranscoding/transcoder/benchmark/MediaSampleReaderBenchmark.cpp
+++ b/media/libmediatranscoding/transcoder/benchmark/MediaSampleReaderBenchmark.cpp
@@ -42,8 +42,8 @@
using namespace android;
static void ReadMediaSamples(benchmark::State& state, const std::string& srcFileName,
- bool readAudio) {
- // Asset directory
+ bool readAudio, bool sequentialAccess = false) {
+ // Asset directory.
static const std::string kAssetDirectory = "/data/local/tmp/TranscodingBenchmark/";
int srcFd = 0;
@@ -59,9 +59,13 @@
for (auto _ : state) {
auto sampleReader = MediaSampleReaderNDK::createFromFd(srcFd, 0, fileSize);
+ if (sampleReader->setEnforceSequentialAccess(sequentialAccess) != AMEDIA_OK) {
+ state.SkipWithError("setEnforceSequentialAccess failed");
+ return;
+ }
- std::vector<std::thread> trackThreads;
-
+ // Select tracks.
+ std::vector<int> trackIndices;
for (int trackIndex = 0; trackIndex < sampleReader->getTrackCount(); ++trackIndex) {
const char* mime = nullptr;
@@ -78,6 +82,13 @@
continue;
}
+ trackIndices.push_back(trackIndex);
+ sampleReader->selectTrack(trackIndex);
+ }
+
+ // Start threads.
+ std::vector<std::thread> trackThreads;
+ for (auto trackIndex : trackIndices) {
trackThreads.emplace_back([trackIndex, sampleReader, &state] {
LOG(INFO) << "Track " << trackIndex << " started";
MediaSampleInfo info;
@@ -102,14 +113,13 @@
state.SkipWithError("Error reading sample data");
break;
}
-
- sampleReader->advanceTrack(trackIndex);
}
LOG(INFO) << "Track " << trackIndex << " finished";
});
}
+ // Join threads.
for (auto& thread : trackThreads) {
thread.join();
}
@@ -122,17 +132,23 @@
#define TRANSCODER_BENCHMARK(func) \
BENCHMARK(func)->UseRealTime()->MeasureProcessCPUTime()->Unit(benchmark::kMillisecond)
-static void BM_MediaSampleReader_AudioVideo(benchmark::State& state) {
+static void BM_MediaSampleReader_AudioVideo_Parallel(benchmark::State& state) {
ReadMediaSamples(state, "video_1920x1080_3648frame_h264_22Mbps_30fps_aac.mp4",
true /* readAudio */);
}
+static void BM_MediaSampleReader_AudioVideo_Sequential(benchmark::State& state) {
+ ReadMediaSamples(state, "video_1920x1080_3648frame_h264_22Mbps_30fps_aac.mp4",
+ true /* readAudio */, true /* sequentialAccess */);
+}
+
static void BM_MediaSampleReader_Video(benchmark::State& state) {
ReadMediaSamples(state, "video_1920x1080_3648frame_h264_22Mbps_30fps_aac.mp4",
false /* readAudio */);
}
-TRANSCODER_BENCHMARK(BM_MediaSampleReader_AudioVideo);
+TRANSCODER_BENCHMARK(BM_MediaSampleReader_AudioVideo_Parallel);
+TRANSCODER_BENCHMARK(BM_MediaSampleReader_AudioVideo_Sequential);
TRANSCODER_BENCHMARK(BM_MediaSampleReader_Video);
BENCHMARK_MAIN();
diff --git a/media/libmediatranscoding/transcoder/benchmark/MediaTranscoderBenchmark.cpp b/media/libmediatranscoding/transcoder/benchmark/MediaTranscoderBenchmark.cpp
index b31b675..ccd9353 100644
--- a/media/libmediatranscoding/transcoder/benchmark/MediaTranscoderBenchmark.cpp
+++ b/media/libmediatranscoding/transcoder/benchmark/MediaTranscoderBenchmark.cpp
@@ -77,7 +77,8 @@
};
static void TranscodeMediaFile(benchmark::State& state, const std::string& srcFileName,
- const std::string& dstFileName, bool includeAudio) {
+ const std::string& dstFileName, bool includeAudio,
+ bool transcodeVideo = true) {
// Default bitrate
static constexpr int32_t kVideoBitRate = 20 * 1000 * 1000; // 20Mbs
// Write-only, create file if non-existent.
@@ -132,8 +133,10 @@
}
if (strncmp(mime, "video/", 6) == 0) {
- dstFormat = AMediaFormat_new();
- AMediaFormat_setInt32(dstFormat, AMEDIAFORMAT_KEY_BIT_RATE, kVideoBitRate);
+ if (transcodeVideo) {
+ dstFormat = AMediaFormat_new();
+ AMediaFormat_setInt32(dstFormat, AMEDIAFORMAT_KEY_BIT_RATE, kVideoBitRate);
+ }
int32_t frameCount;
if (AMediaFormat_getInt32(srcFormat, AMEDIAFORMAT_KEY_FRAME_COUNT, &frameCount)) {
@@ -198,8 +201,21 @@
false /* includeAudio */);
}
+static void BM_TranscodeAudioVideoPassthrough(benchmark::State& state) {
+ TranscodeMediaFile(state, "video_1920x1080_3648frame_h264_22Mbps_30fps_aac.mp4",
+ "video_1920x1080_3648frame_h264_22Mbps_30fps_aac_passthrough_AV.mp4",
+ true /* includeAudio */, false /* transcodeVideo */);
+}
+static void BM_TranscodeVideoPassthrough(benchmark::State& state) {
+ TranscodeMediaFile(state, "video_1920x1080_3648frame_h264_22Mbps_30fps.mp4",
+ "video_1920x1080_3648frame_h264_22Mbps_30fps_passthrough_AV.mp4",
+ false /* includeAudio */, false /* transcodeVideo */);
+}
+
TRANSCODER_BENCHMARK(BM_TranscodeAvc2AvcAudioVideo2AudioVideo);
TRANSCODER_BENCHMARK(BM_TranscodeAvc2AvcAudioVideo2Video);
TRANSCODER_BENCHMARK(BM_TranscodeAvc2AvcVideo2Video);
+TRANSCODER_BENCHMARK(BM_TranscodeAudioVideoPassthrough);
+TRANSCODER_BENCHMARK(BM_TranscodeVideoPassthrough);
BENCHMARK_MAIN();
diff --git a/media/libmediatranscoding/transcoder/include/media/MediaSampleReader.h b/media/libmediatranscoding/transcoder/include/media/MediaSampleReader.h
index 25df7ab..7b6fbef 100644
--- a/media/libmediatranscoding/transcoder/include/media/MediaSampleReader.h
+++ b/media/libmediatranscoding/transcoder/include/media/MediaSampleReader.h
@@ -24,12 +24,15 @@
namespace android {
/**
- * MediaSampleReader is an interface for reading media samples from a container.
- * MediaSampleReader allows for reading samples from multiple tracks independently of each other
- * while preserving the order of samples within each individual track.
- * MediaSampleReader implementations are thread safe and can be used by multiple threads
- * concurrently. But note that MediaSampleReader only maintains one state per track so concurrent
- * usage of the same track from multiple threads has no benefit.
+ * MediaSampleReader is an interface for reading media samples from a container. MediaSampleReader
+ * allows for reading samples from multiple tracks on individual threads independently of each other
+ * while preserving the order of samples. Due to poor non-sequential access performance of the
+ * underlying extractor, MediaSampleReader can optionally enforce sequential sample access by
+ * blocking requests for tracks that the underlying extractor does not currently point to. Waiting
+ * threads are serviced once the reader advances to a sample from the specified track. Due to this
+ * it is important to read samples and advance the reader from all selected tracks to avoid hanging
+ * other tracks. MediaSampleReader implementations are thread safe and sample access should be done
+ * on one thread per selected track.
*/
class MediaSampleReader {
public:
@@ -57,6 +60,24 @@
virtual AMediaFormat* getTrackFormat(int trackIndex) = 0;
/**
+ * Select a track for sample access. Tracks must be selected in order for sample information and
+ * sample data to be available for that track. Samples for selected tracks must be accessed on
+ * its own thread to avoid blocking other tracks.
+ * @param trackIndex The track to select.
+ * @return AMEDIA_OK on success.
+ */
+ virtual media_status_t selectTrack(int trackIndex) = 0;
+
+ /**
+ * Toggles sequential access enforcement on or off. When the reader enforces sequential access
+ * calls to read sample information will block unless the underlying extractor points to the
+ * specified track.
+ * @param enforce True to enforce sequential access.
+ * @return AMEDIA_OK on success.
+ */
+ virtual media_status_t setEnforceSequentialAccess(bool enforce) = 0;
+
+ /**
* Estimates the bitrate of a source track by sampling sample sizes. The bitrate is returned in
* megabits per second (Mbps). This method will fail if the track only contains a single sample
* and does not have an associated duration.
@@ -67,7 +88,9 @@
virtual media_status_t getEstimatedBitrateForTrack(int trackIndex, int32_t* bitrate);
/**
- * Returns the sample information for the current sample in the specified track.
+ * Returns the sample information for the current sample in the specified track. Note that this
+ * method will block until the reader advances to a sample belonging to the requested track if
+ * the reader is in sequential access mode.
* @param trackIndex The track index (zero-based).
* @param info Pointer to a MediaSampleInfo object where the sample information is written.
* @return AMEDIA_OK on success, AMEDIA_ERROR_END_OF_STREAM if there are no more samples to read
@@ -77,7 +100,10 @@
virtual media_status_t getSampleInfoForTrack(int trackIndex, MediaSampleInfo* info) = 0;
/**
- * Reads the current sample's data into the supplied buffer.
+ * Returns the sample data for the current sample in the specified track into the supplied
+ * buffer. Note that this method will block until the reader advances to a sample belonging to
+ * the requested track if the reader is in sequential access mode. Upon successful return this
+ * method will also advance the specified track to the next sample.
* @param trackIndex The track index (zero-based).
* @param buffer The buffer to write the sample's data to.
* @param bufferSize The size of the supplied buffer.
@@ -90,7 +116,9 @@
size_t bufferSize) = 0;
/**
- * Advance the specified track to the next sample.
+ * Advance the specified track to the next sample. If the reader is in sequential access mode
+ * and the current sample belongs to the specified track, the reader will also advance to the
+ * next sample and wake up any threads waiting on the new track.
* @param trackIndex The track index (zero-based).
*/
virtual void advanceTrack(int trackIndex) = 0;
diff --git a/media/libmediatranscoding/transcoder/include/media/MediaSampleReaderNDK.h b/media/libmediatranscoding/transcoder/include/media/MediaSampleReaderNDK.h
index 6c5019d..5f9822d 100644
--- a/media/libmediatranscoding/transcoder/include/media/MediaSampleReaderNDK.h
+++ b/media/libmediatranscoding/transcoder/include/media/MediaSampleReaderNDK.h
@@ -20,6 +20,7 @@
#include <media/MediaSampleReader.h>
#include <media/NdkMediaExtractor.h>
+#include <map>
#include <memory>
#include <mutex>
#include <vector>
@@ -46,6 +47,8 @@
AMediaFormat* getFileFormat() override;
size_t getTrackCount() const override;
AMediaFormat* getTrackFormat(int trackIndex) override;
+ media_status_t selectTrack(int trackIndex) override;
+ media_status_t setEnforceSequentialAccess(bool enforce) override;
media_status_t getEstimatedBitrateForTrack(int trackIndex, int32_t* bitrate) override;
media_status_t getSampleInfoForTrack(int trackIndex, MediaSampleInfo* info) override;
media_status_t readSampleDataForTrack(int trackIndex, uint8_t* buffer,
@@ -55,20 +58,6 @@
virtual ~MediaSampleReaderNDK() override;
private:
- /**
- * Creates a new MediaSampleReaderNDK object from an AMediaExtractor. The extractor needs to be
- * initialized with a valid data source before attempting to create a MediaSampleReaderNDK.
- * @param extractor The initialized media extractor.
- */
- MediaSampleReaderNDK(AMediaExtractor* extractor);
- media_status_t init();
-
- AMediaExtractor* mExtractor = nullptr;
- std::mutex mExtractorMutex;
- const size_t mTrackCount;
-
- int mExtractorTrackIndex = -1;
- uint64_t mExtractorSampleIndex = 0;
/**
* SamplePosition describes the position of a single sample in the media file using its
@@ -100,13 +89,52 @@
SamplePosition next;
};
- /** Samples cursor for each track in the file. */
- std::vector<SampleCursor> mTrackCursors;
+ /**
+ * Creates a new MediaSampleReaderNDK object from an AMediaExtractor. The extractor needs to be
+ * initialized with a valid data source before attempting to create a MediaSampleReaderNDK.
+ * @param extractor The initialized media extractor.
+ */
+ MediaSampleReaderNDK(AMediaExtractor* extractor);
+ /** Advances the track to next sample. */
+ void advanceTrack_l(int trackIndex);
+
+ /** Advances the extractor to next sample. */
bool advanceExtractor_l();
- media_status_t positionExtractorForTrack_l(int trackIndex);
+
+ /** Moves the extractor backwards to the specified sample. */
media_status_t seekExtractorBackwards_l(int64_t targetTimeUs, int targetTrackIndex,
uint64_t targetSampleIndex);
+
+ /** Moves the extractor to the specified sample. */
+ media_status_t moveToSample_l(SamplePosition& pos, int trackIndex);
+
+ /** Moves the extractor to the next sample of the specified track. */
+ media_status_t moveToTrack_l(int trackIndex);
+
+ /** In sequential mode, waits for the extractor to reach the next sample for the track. */
+ media_status_t waitForTrack_l(int trackIndex, std::unique_lock<std::mutex>& lockHeld);
+
+ /**
+ * Ensures the extractor is ready for the next sample of the track regardless of access mode.
+ */
+ media_status_t primeExtractorForTrack_l(int trackIndex, std::unique_lock<std::mutex>& lockHeld);
+
+ AMediaExtractor* mExtractor = nullptr;
+ std::mutex mExtractorMutex;
+ const size_t mTrackCount;
+
+ int mExtractorTrackIndex = -1;
+ uint64_t mExtractorSampleIndex = 0;
+
+ bool mEosReached = false;
+ bool mEnforceSequentialAccess = false;
+
+ // Maps selected track indices to condition variables for sequential sample access control.
+ std::map<int, std::condition_variable> mTrackSignals;
+
+ // Samples cursor for each track in the file.
+ std::vector<SampleCursor> mTrackCursors;
};
} // namespace android
diff --git a/media/libmediatranscoding/transcoder/tests/MediaSampleReaderNDKTests.cpp b/media/libmediatranscoding/transcoder/tests/MediaSampleReaderNDKTests.cpp
index 323e5ae..e8acd48 100644
--- a/media/libmediatranscoding/transcoder/tests/MediaSampleReaderNDKTests.cpp
+++ b/media/libmediatranscoding/transcoder/tests/MediaSampleReaderNDKTests.cpp
@@ -26,10 +26,14 @@
#include <gtest/gtest.h>
#include <media/MediaSampleReaderNDK.h>
#include <utils/Timers.h>
-
#include <cmath>
+#include <mutex>
+#include <thread>
// TODO(b/153453392): Test more asset types and validate sample data from readSampleDataForTrack.
+// TODO(b/153453392): Test for sequential and parallel (single thread and multi thread) access.
+// TODO(b/153453392): Test for switching between sequential and parallel access in different points
+// of time.
namespace android {
@@ -121,48 +125,47 @@
MediaSampleReaderNDK::createFromFd(mSourceFd, 0, mFileSize);
ASSERT_TRUE(sampleReader);
- MediaSampleInfo info;
- int trackEosCount = 0;
- std::vector<bool> trackReachedEos(mTrackCount, false);
- std::vector<std::vector<int64_t>> readerTimestamps(mTrackCount);
+ for (int trackIndex = 0; trackIndex < mTrackCount; trackIndex++) {
+ EXPECT_EQ(sampleReader->selectTrack(trackIndex), AMEDIA_OK);
+ }
// Initialize the extractor timestamps.
initExtractorTimestamps();
- // Read 5s of each track at a time.
- const int64_t chunkDurationUs = SEC_TO_USEC(5);
- int64_t chunkEndTimeUs = chunkDurationUs;
+ std::mutex timestampMutex;
+ std::vector<std::thread> trackThreads;
+ std::vector<std::vector<int64_t>> readerTimestamps(mTrackCount);
- // Loop until all tracks have reached End Of Stream.
- while (trackEosCount < mTrackCount) {
- for (int trackIndex = 0; trackIndex < mTrackCount; trackIndex++) {
- if (trackReachedEos[trackIndex]) continue;
-
- // Advance current track to next chunk end time.
- do {
+ for (int trackIndex = 0; trackIndex < mTrackCount; trackIndex++) {
+ trackThreads.emplace_back([sampleReader, trackIndex, ×tampMutex, &readerTimestamps] {
+ MediaSampleInfo info;
+ while (true) {
media_status_t status = sampleReader->getSampleInfoForTrack(trackIndex, &info);
if (status != AMEDIA_OK) {
- ASSERT_EQ(status, AMEDIA_ERROR_END_OF_STREAM);
- ASSERT_TRUE((info.flags & SAMPLE_FLAG_END_OF_STREAM) != 0);
- trackReachedEos[trackIndex] = true;
- trackEosCount++;
+ EXPECT_EQ(status, AMEDIA_ERROR_END_OF_STREAM);
+ EXPECT_TRUE((info.flags & SAMPLE_FLAG_END_OF_STREAM) != 0);
break;
}
ASSERT_TRUE((info.flags & SAMPLE_FLAG_END_OF_STREAM) == 0);
+ timestampMutex.lock();
readerTimestamps[trackIndex].push_back(info.presentationTimeUs);
+ timestampMutex.unlock();
sampleReader->advanceTrack(trackIndex);
- } while (info.presentationTimeUs < chunkEndTimeUs);
- }
- chunkEndTimeUs += chunkDurationUs;
+ }
+ });
+ }
+
+ for (auto& thread : trackThreads) {
+ thread.join();
}
for (int trackIndex = 0; trackIndex < mTrackCount; trackIndex++) {
LOG(DEBUG) << "Track " << trackIndex << ", comparing "
<< readerTimestamps[trackIndex].size() << " samples.";
- ASSERT_EQ(readerTimestamps[trackIndex].size(), mExtractorTimestamps[trackIndex].size());
+ EXPECT_EQ(readerTimestamps[trackIndex].size(), mExtractorTimestamps[trackIndex].size());
for (size_t sampleIndex = 0; sampleIndex < readerTimestamps[trackIndex].size();
sampleIndex++) {
- ASSERT_EQ(readerTimestamps[trackIndex][sampleIndex],
+ EXPECT_EQ(readerTimestamps[trackIndex][sampleIndex],
mExtractorTimestamps[trackIndex][sampleIndex]);
}
}
@@ -178,6 +181,8 @@
std::vector<int32_t> actualTrackBitrates = getTrackBitrates();
for (int trackIndex = 0; trackIndex < mTrackCount; ++trackIndex) {
+ EXPECT_EQ(sampleReader->selectTrack(trackIndex), AMEDIA_OK);
+
int32_t bitrate;
EXPECT_EQ(sampleReader->getEstimatedBitrateForTrack(trackIndex, &bitrate), AMEDIA_OK);
EXPECT_GT(bitrate, 0);
diff --git a/media/libmediatranscoding/transcoder/tests/MediaTrackTranscoderTests.cpp b/media/libmediatranscoding/transcoder/tests/MediaTrackTranscoderTests.cpp
index 804571c..a46c2bd 100644
--- a/media/libmediatranscoding/transcoder/tests/MediaTrackTranscoderTests.cpp
+++ b/media/libmediatranscoding/transcoder/tests/MediaTrackTranscoderTests.cpp
@@ -111,6 +111,7 @@
}
ASSERT_NE(mSourceFormat, nullptr);
+ EXPECT_EQ(mMediaSampleReader->selectTrack(mTrackIndex), AMEDIA_OK);
}
// Drains the transcoder's output queue in a loop.
diff --git a/media/libmediatranscoding/transcoder/tests/PassthroughTrackTranscoderTests.cpp b/media/libmediatranscoding/transcoder/tests/PassthroughTrackTranscoderTests.cpp
index b79f58c..a2ffbe4 100644
--- a/media/libmediatranscoding/transcoder/tests/PassthroughTrackTranscoderTests.cpp
+++ b/media/libmediatranscoding/transcoder/tests/PassthroughTrackTranscoderTests.cpp
@@ -159,6 +159,7 @@
MediaSampleReaderNDK::createFromFd(mSourceFd, 0, mSourceFileSize);
EXPECT_NE(mediaSampleReader, nullptr);
+ EXPECT_EQ(mediaSampleReader->selectTrack(mTrackIndex), AMEDIA_OK);
EXPECT_EQ(transcoder.configure(mediaSampleReader, mTrackIndex, nullptr /* destinationFormat */),
AMEDIA_OK);
ASSERT_TRUE(transcoder.start());
diff --git a/media/libmediatranscoding/transcoder/tests/VideoTrackTranscoderTests.cpp b/media/libmediatranscoding/transcoder/tests/VideoTrackTranscoderTests.cpp
index b432553..e809cbd 100644
--- a/media/libmediatranscoding/transcoder/tests/VideoTrackTranscoderTests.cpp
+++ b/media/libmediatranscoding/transcoder/tests/VideoTrackTranscoderTests.cpp
@@ -97,6 +97,7 @@
std::shared_ptr<TestCallback> callback = std::make_shared<TestCallback>();
auto transcoder = VideoTrackTranscoder::create(callback);
+ EXPECT_EQ(mMediaSampleReader->selectTrack(mTrackIndex), AMEDIA_OK);
EXPECT_EQ(transcoder->configure(mMediaSampleReader, mTrackIndex, mDestinationFormat),
AMEDIA_OK);
ASSERT_TRUE(transcoder->start());
@@ -152,6 +153,11 @@
mSourceFormat.get(), false /* includeBitrate*/);
EXPECT_NE(destFormat, nullptr);
+ EXPECT_EQ(mMediaSampleReader->selectTrack(mTrackIndex), AMEDIA_OK);
+
+ int32_t srcBitrate;
+ EXPECT_EQ(mMediaSampleReader->getEstimatedBitrateForTrack(mTrackIndex, &srcBitrate), AMEDIA_OK);
+
ASSERT_EQ(transcoder->configure(mMediaSampleReader, mTrackIndex, destFormat), AMEDIA_OK);
ASSERT_TRUE(transcoder->start());
@@ -166,9 +172,6 @@
int32_t outBitrate;
EXPECT_TRUE(AMediaFormat_getInt32(outputFormat.get(), AMEDIAFORMAT_KEY_BIT_RATE, &outBitrate));
- int32_t srcBitrate;
- EXPECT_EQ(mMediaSampleReader->getEstimatedBitrateForTrack(mTrackIndex, &srcBitrate), AMEDIA_OK);
-
EXPECT_EQ(srcBitrate, outBitrate);
}
@@ -206,6 +209,7 @@
auto callback = std::make_shared<TestCallback>();
auto transcoder = VideoTrackTranscoder::create(callback);
+ EXPECT_EQ(mMediaSampleReader->selectTrack(mTrackIndex), AMEDIA_OK);
EXPECT_EQ(transcoder->configure(mMediaSampleReader, mTrackIndex, mDestinationFormat),
AMEDIA_OK);
ASSERT_TRUE(transcoder->start());
@@ -214,7 +218,7 @@
std::vector<std::shared_ptr<MediaSample>> samples;
std::thread sampleConsumerThread([&outputQueue, &samples, &semaphore] {
std::shared_ptr<MediaSample> sample;
- while (samples.size() < 10 && !outputQueue->dequeue(&sample)) {
+ while (samples.size() < 4 && !outputQueue->dequeue(&sample)) {
ASSERT_NE(sample, nullptr);
samples.push_back(sample);
diff --git a/media/libstagefright/HevcUtils.cpp b/media/libstagefright/HevcUtils.cpp
index aac656a..5f9c20e 100644
--- a/media/libstagefright/HevcUtils.cpp
+++ b/media/libstagefright/HevcUtils.cpp
@@ -380,6 +380,54 @@
return reader.overRead() ? ERROR_MALFORMED : OK;
}
+void HevcParameterSets::FindHEVCDimensions(const sp<ABuffer> &SpsBuffer, int32_t *width, int32_t *height)
+{
+ ALOGD("FindHEVCDimensions");
+ // See Rec. ITU-T H.265 v3 (04/2015) Chapter 7.3.2.2 for reference
+ ABitReader reader(SpsBuffer->data() + 1, SpsBuffer->size() - 1);
+ // Skip sps_video_parameter_set_id
+ reader.skipBits(4);
+ uint8_t maxSubLayersMinus1 = reader.getBitsWithFallback(3, 0);
+ // Skip sps_temporal_id_nesting_flag;
+ reader.skipBits(1);
+ // Skip general profile
+ reader.skipBits(96);
+ if (maxSubLayersMinus1 > 0) {
+ bool subLayerProfilePresentFlag[8];
+ bool subLayerLevelPresentFlag[8];
+ for (int i = 0; i < maxSubLayersMinus1; ++i) {
+ subLayerProfilePresentFlag[i] = reader.getBitsWithFallback(1, 0);
+ subLayerLevelPresentFlag[i] = reader.getBitsWithFallback(1, 0);
+ }
+ // Skip reserved
+ reader.skipBits(2 * (8 - maxSubLayersMinus1));
+ for (int i = 0; i < maxSubLayersMinus1; ++i) {
+ if (subLayerProfilePresentFlag[i]) {
+ // Skip profile
+ reader.skipBits(88);
+ }
+ if (subLayerLevelPresentFlag[i]) {
+ // Skip sub_layer_level_idc[i]
+ reader.skipBits(8);
+ }
+ }
+ }
+ // Skip sps_seq_parameter_set_id
+ skipUE(&reader);
+ uint8_t chromaFormatIdc = parseUEWithFallback(&reader, 0);
+ if (chromaFormatIdc == 3) {
+ // Skip separate_colour_plane_flag
+ reader.skipBits(1);
+ }
+ skipUE(&reader);
+ skipUE(&reader);
+
+ // pic_width_in_luma_samples
+ *width = parseUEWithFallback(&reader, 0);
+ // pic_height_in_luma_samples
+ *height = parseUEWithFallback(&reader, 0);
+}
+
status_t HevcParameterSets::parsePps(
const uint8_t* data UNUSED_PARAM, size_t size UNUSED_PARAM) {
return OK;
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index da39476..5015787 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -2044,20 +2044,25 @@
} else if (mFlags & kFlagOutputBuffersChanged) {
PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED);
mFlags &= ~kFlagOutputBuffersChanged;
- } else if (mFlags & kFlagOutputFormatChanged) {
- PostReplyWithError(replyID, INFO_FORMAT_CHANGED);
- mFlags &= ~kFlagOutputFormatChanged;
} else {
sp<AMessage> response = new AMessage;
- ssize_t index = dequeuePortBuffer(kPortIndexOutput);
-
- if (index < 0) {
- CHECK_EQ(index, -EAGAIN);
+ BufferInfo *info = peekNextPortBuffer(kPortIndexOutput);
+ if (!info) {
return false;
}
- const sp<MediaCodecBuffer> &buffer =
- mPortBuffers[kPortIndexOutput][index].mData;
+ // In synchronous mode, output format change should be handled
+ // at dequeue to put the event at the correct order.
+
+ const sp<MediaCodecBuffer> &buffer = info->mData;
+ handleOutputFormatChangeIfNeeded(buffer);
+ if (mFlags & kFlagOutputFormatChanged) {
+ PostReplyWithError(replyID, INFO_FORMAT_CHANGED);
+ mFlags &= ~kFlagOutputFormatChanged;
+ return true;
+ }
+
+ ssize_t index = dequeuePortBuffer(kPortIndexOutput);
response->setSize("index", index);
response->setSize("offset", buffer->offset());
@@ -2542,107 +2547,13 @@
break;
}
- sp<RefBase> obj;
- CHECK(msg->findObject("buffer", &obj));
- sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
-
- if (mOutputFormat != buffer->format()) {
- if (mFlags & kFlagUseBlockModel) {
- sp<AMessage> diff1 = mOutputFormat->changesFrom(buffer->format());
- sp<AMessage> diff2 = buffer->format()->changesFrom(mOutputFormat);
- std::set<std::string> keys;
- size_t numEntries = diff1->countEntries();
- AMessage::Type type;
- for (size_t i = 0; i < numEntries; ++i) {
- keys.emplace(diff1->getEntryNameAt(i, &type));
- }
- numEntries = diff2->countEntries();
- for (size_t i = 0; i < numEntries; ++i) {
- keys.emplace(diff2->getEntryNameAt(i, &type));
- }
- sp<WrapperObject<std::set<std::string>>> changedKeys{
- new WrapperObject<std::set<std::string>>{std::move(keys)}};
- buffer->meta()->setObject("changedKeys", changedKeys);
- }
- mOutputFormat = buffer->format();
- ALOGV("[%s] output format changed to: %s",
- mComponentName.c_str(), mOutputFormat->debugString(4).c_str());
-
- if (mSoftRenderer == NULL &&
- mSurface != NULL &&
- (mFlags & kFlagUsesSoftwareRenderer)) {
- AString mime;
- CHECK(mOutputFormat->findString("mime", &mime));
-
- // TODO: propagate color aspects to software renderer to allow better
- // color conversion to RGB. For now, just mark dataspace for YUV
- // rendering.
- int32_t dataSpace;
- if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) {
- ALOGD("[%s] setting dataspace on output surface to #%x",
- mComponentName.c_str(), dataSpace);
- int err = native_window_set_buffers_data_space(
- mSurface.get(), (android_dataspace)dataSpace);
- ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err);
- }
- if (mOutputFormat->contains("hdr-static-info")) {
- HDRStaticInfo info;
- if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {
- setNativeWindowHdrMetadata(mSurface.get(), &info);
- }
- }
-
- sp<ABuffer> hdr10PlusInfo;
- if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo)
- && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
- native_window_set_buffers_hdr10_plus_metadata(mSurface.get(),
- hdr10PlusInfo->size(), hdr10PlusInfo->data());
- }
-
- if (mime.startsWithIgnoreCase("video/")) {
- mSurface->setDequeueTimeout(-1);
- mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);
- }
- }
-
- requestCpuBoostIfNeeded();
-
- if (mFlags & kFlagIsEncoder) {
- // Before we announce the format change we should
- // collect codec specific data and amend the output
- // format as necessary.
- int32_t flags = 0;
- (void) buffer->meta()->findInt32("flags", &flags);
- if ((flags & BUFFER_FLAG_CODECCONFIG) && !(mFlags & kFlagIsSecure)) {
- status_t err =
- amendOutputFormatWithCodecSpecificData(buffer);
-
- if (err != OK) {
- ALOGE("Codec spit out malformed codec "
- "specific data!");
- }
- }
- }
- if (mFlags & kFlagIsAsync) {
- onOutputFormatChanged();
- } else {
- mFlags |= kFlagOutputFormatChanged;
- postActivityNotificationIfPossible();
- }
-
- // Notify mCrypto of video resolution changes
- if (mCrypto != NULL) {
- int32_t left, top, right, bottom, width, height;
- if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
- mCrypto->notifyResolution(right - left + 1, bottom - top + 1);
- } else if (mOutputFormat->findInt32("width", &width)
- && mOutputFormat->findInt32("height", &height)) {
- mCrypto->notifyResolution(width, height);
- }
- }
- }
-
if (mFlags & kFlagIsAsync) {
+ sp<RefBase> obj;
+ CHECK(msg->findObject("buffer", &obj));
+ sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
+
+ // In asynchronous mode, output format change is processed immediately.
+ handleOutputFormatChangeIfNeeded(buffer);
onOutputBufferAvailable();
} else if (mFlags & kFlagDequeueOutputPending) {
CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
@@ -3473,6 +3384,106 @@
}
}
+void MediaCodec::handleOutputFormatChangeIfNeeded(const sp<MediaCodecBuffer> &buffer) {
+ sp<AMessage> format = buffer->format();
+ if (mOutputFormat == format) {
+ return;
+ }
+ if (mFlags & kFlagUseBlockModel) {
+ sp<AMessage> diff1 = mOutputFormat->changesFrom(format);
+ sp<AMessage> diff2 = format->changesFrom(mOutputFormat);
+ std::set<std::string> keys;
+ size_t numEntries = diff1->countEntries();
+ AMessage::Type type;
+ for (size_t i = 0; i < numEntries; ++i) {
+ keys.emplace(diff1->getEntryNameAt(i, &type));
+ }
+ numEntries = diff2->countEntries();
+ for (size_t i = 0; i < numEntries; ++i) {
+ keys.emplace(diff2->getEntryNameAt(i, &type));
+ }
+ sp<WrapperObject<std::set<std::string>>> changedKeys{
+ new WrapperObject<std::set<std::string>>{std::move(keys)}};
+ buffer->meta()->setObject("changedKeys", changedKeys);
+ }
+ mOutputFormat = format;
+ ALOGV("[%s] output format changed to: %s",
+ mComponentName.c_str(), mOutputFormat->debugString(4).c_str());
+
+ if (mSoftRenderer == NULL &&
+ mSurface != NULL &&
+ (mFlags & kFlagUsesSoftwareRenderer)) {
+ AString mime;
+ CHECK(mOutputFormat->findString("mime", &mime));
+
+ // TODO: propagate color aspects to software renderer to allow better
+ // color conversion to RGB. For now, just mark dataspace for YUV
+ // rendering.
+ int32_t dataSpace;
+ if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) {
+ ALOGD("[%s] setting dataspace on output surface to #%x",
+ mComponentName.c_str(), dataSpace);
+ int err = native_window_set_buffers_data_space(
+ mSurface.get(), (android_dataspace)dataSpace);
+ ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err);
+ }
+ if (mOutputFormat->contains("hdr-static-info")) {
+ HDRStaticInfo info;
+ if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {
+ setNativeWindowHdrMetadata(mSurface.get(), &info);
+ }
+ }
+
+ sp<ABuffer> hdr10PlusInfo;
+ if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo)
+ && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
+ native_window_set_buffers_hdr10_plus_metadata(mSurface.get(),
+ hdr10PlusInfo->size(), hdr10PlusInfo->data());
+ }
+
+ if (mime.startsWithIgnoreCase("video/")) {
+ mSurface->setDequeueTimeout(-1);
+ mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);
+ }
+ }
+
+ requestCpuBoostIfNeeded();
+
+ if (mFlags & kFlagIsEncoder) {
+ // Before we announce the format change we should
+ // collect codec specific data and amend the output
+ // format as necessary.
+ int32_t flags = 0;
+ (void) buffer->meta()->findInt32("flags", &flags);
+ if ((flags & BUFFER_FLAG_CODECCONFIG) && !(mFlags & kFlagIsSecure)) {
+ status_t err =
+ amendOutputFormatWithCodecSpecificData(buffer);
+
+ if (err != OK) {
+ ALOGE("Codec spit out malformed codec "
+ "specific data!");
+ }
+ }
+ }
+ if (mFlags & kFlagIsAsync) {
+ onOutputFormatChanged();
+ } else {
+ mFlags |= kFlagOutputFormatChanged;
+ postActivityNotificationIfPossible();
+ }
+
+ // Notify mCrypto of video resolution changes
+ if (mCrypto != NULL) {
+ int32_t left, top, right, bottom, width, height;
+ if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
+ mCrypto->notifyResolution(right - left + 1, bottom - top + 1);
+ } else if (mOutputFormat->findInt32("width", &width)
+ && mOutputFormat->findInt32("height", &height)) {
+ mCrypto->notifyResolution(width, height);
+ }
+ }
+}
+
void MediaCodec::extractCSD(const sp<AMessage> &format) {
mCSD.clear();
@@ -3938,19 +3949,31 @@
return OK;
}
-ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
+MediaCodec::BufferInfo *MediaCodec::peekNextPortBuffer(int32_t portIndex) {
CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
if (availBuffers->empty()) {
+ return nullptr;
+ }
+
+ return &mPortBuffers[portIndex][*availBuffers->begin()];
+}
+
+ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
+ CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
+
+ BufferInfo *info = peekNextPortBuffer(portIndex);
+ if (!info) {
return -EAGAIN;
}
+ List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
size_t index = *availBuffers->begin();
+ CHECK_EQ(info, &mPortBuffers[portIndex][index]);
availBuffers->erase(availBuffers->begin());
- BufferInfo *info = &mPortBuffers[portIndex][index];
CHECK(!info->mOwnedByClient);
{
Mutex::Autolock al(mBufferLock);
diff --git a/media/libstagefright/OWNERS b/media/libstagefright/OWNERS
new file mode 100644
index 0000000..819389d
--- /dev/null
+++ b/media/libstagefright/OWNERS
@@ -0,0 +1,7 @@
+set noparent
+chz@google.com
+essick@google.com
+lajos@google.com
+marcone@google.com
+taklee@google.com
+wonsik@google.com
\ No newline at end of file
diff --git a/media/libstagefright/include/HevcUtils.h b/media/libstagefright/include/HevcUtils.h
index d2a86eb..6a4a168 100644
--- a/media/libstagefright/include/HevcUtils.h
+++ b/media/libstagefright/include/HevcUtils.h
@@ -94,6 +94,8 @@
// Note that this method does not write the start code.
bool write(size_t index, uint8_t* dest, size_t size);
status_t makeHvcc(uint8_t *hvcc, size_t *hvccSize, size_t nalSizeLength);
+ void FindHEVCDimensions(
+ const sp<ABuffer> &SpsBuffer, int32_t *width, int32_t *height);
Info getInfo() const { return mInfo; }
static bool IsHevcIDR(const uint8_t *data, size_t size);
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index 1f8e780..c4026ec 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -452,6 +452,7 @@
size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
status_t onQueueInputBuffer(const sp<AMessage> &msg);
status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
+ BufferInfo *peekNextPortBuffer(int32_t portIndex);
ssize_t dequeuePortBuffer(int32_t portIndex);
status_t getBufferAndFormat(
@@ -483,6 +484,7 @@
status_t onSetParameters(const sp<AMessage> ¶ms);
status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);
+ void handleOutputFormatChangeIfNeeded(const sp<MediaCodecBuffer> &buffer);
bool isExecuting() const;
uint64_t getGraphicBufferSize();
diff --git a/media/libstagefright/include/media/stagefright/MetaDataBase.h b/media/libstagefright/include/media/stagefright/MetaDataBase.h
index 173b701..2f34094 100644
--- a/media/libstagefright/include/media/stagefright/MetaDataBase.h
+++ b/media/libstagefright/include/media/stagefright/MetaDataBase.h
@@ -248,8 +248,9 @@
// Treat empty track as malformed for MediaRecorder.
kKeyEmptyTrackMalFormed = 'nemt', // bool (int32_t)
- kKeySps = 'sSps', // int32_t, indicates that a buffer is sps (value ignored).
- kKeyPps = 'sPps', // int32_t, indicates that a buffer is pps (value ignored).
+ kKeyVps = 'sVps', // int32_t, indicates that a buffer has vps.
+ kKeySps = 'sSps', // int32_t, indicates that a buffer has sps.
+ kKeyPps = 'sPps', // int32_t, indicates that a buffer has pps.
kKeySelfID = 'sfid', // int32_t, source ID to identify itself on RTP protocol.
kKeyPayloadType = 'pTyp', // int32_t, SDP negotiated payload type.
kKeyRtpExtMap = 'extm', // int32_t, rtp extension ID for cvo on RTP protocol.
diff --git a/media/libstagefright/rtsp/AAVCAssembler.cpp b/media/libstagefright/rtsp/AAVCAssembler.cpp
index 0164040..a0b66a7 100644
--- a/media/libstagefright/rtsp/AAVCAssembler.cpp
+++ b/media/libstagefright/rtsp/AAVCAssembler.cpp
@@ -37,12 +37,73 @@
mAccessUnitRTPTime(0),
mNextExpectedSeqNoValid(false),
mNextExpectedSeqNo(0),
- mAccessUnitDamaged(false) {
+ mAccessUnitDamaged(false),
+ mFirstIFrameProvided(false),
+ mLastIFrameProvidedAtMs(0) {
}
AAVCAssembler::~AAVCAssembler() {
}
+int32_t AAVCAssembler::addNack(
+ const sp<ARTPSource> &source) {
+ List<sp<ABuffer>> *queue = source->queue();
+ int32_t nackCount = 0;
+
+ List<sp<ABuffer> >::iterator it = queue->begin();
+
+ if (it == queue->end()) {
+ return nackCount /* 0 */;
+ }
+
+ uint16_t queueHeadSeqNum = (*it)->int32Data();
+
+ // move to the packet after which RTCP:NACK was sent.
+ for (; it != queue->end(); ++it) {
+ int32_t seqNum = (*it)->int32Data();
+ if (seqNum >= source->mHighestNackNumber) {
+ break;
+ }
+ }
+
+ int32_t nackStartAt = -1;
+
+ while (it != queue->end()) {
+ int32_t seqBeforeLast = (*it)->int32Data();
+ // increase iterator.
+ if ((++it) == queue->end()) {
+ break;
+ }
+ int32_t seqLast = (*it)->int32Data();
+
+ if ((seqLast - seqBeforeLast) < 0) {
+ ALOGD("addNack: found end of seqNum from(%d) to(%d)", seqBeforeLast, seqLast);
+ source->mHighestNackNumber = 0;
+ }
+
+ // missed packet found
+ if (seqLast > (seqBeforeLast + 1) &&
+ // we didn't send RTCP:NACK for this packet yet.
+ (seqLast - 1) > source->mHighestNackNumber) {
+ source->mHighestNackNumber = seqLast - 1;
+ nackStartAt = seqBeforeLast + 1;
+ break;
+ }
+
+ }
+
+ if (nackStartAt != -1) {
+ nackCount = source->mHighestNackNumber - nackStartAt + 1;
+ ALOGD("addNack: nackCount=%d, nackFrom=%d, nackTo=%d", nackCount,
+ nackStartAt, source->mHighestNackNumber);
+
+ uint16_t mask = (uint16_t)(0xffff) >> (16 - nackCount + 1);
+ source->setSeqNumToNACK(nackStartAt, mask, queueHeadSeqNum);
+ }
+
+ return nackCount;
+}
+
ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit(
const sp<ARTPSource> &source) {
List<sp<ABuffer> > *queue = source->queue();
@@ -52,78 +113,54 @@
}
sp<ABuffer> buffer = *queue->begin();
- int32_t rtpTime;
- CHECK(buffer->meta()->findInt32("rtp-time", &rtpTime));
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
int64_t startTime = source->mFirstSysTime / 1000;
int64_t nowTime = ALooper::GetNowUs() / 1000;
int64_t playedTime = nowTime - startTime;
- int32_t playedTimeRtp = source->mFirstRtpTime +
- (((uint32_t)playedTime) * (source->mClockRate / 1000));
- const int32_t jitterTime = source->mClockRate / 5; // 200ms
- int32_t expiredTimeInJb = rtpTime + jitterTime;
+ int64_t playedTimeRtp =
+ source->mFirstRtpTime + (((uint32_t)playedTime) * (source->mClockRate / 1000));
+ const uint32_t jitterTime =
+ (uint32_t)(source->mClockRate / ((float)1000 / (source->mJbTimeMs)));
+ uint32_t expiredTimeInJb = rtpTime + jitterTime;
bool isExpired = expiredTimeInJb <= (playedTimeRtp);
bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);
bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));
if (mShowQueue && mShowQueueCnt < 20) {
showCurrentQueue(queue);
- ALOGD("start=%lld, now=%lld, played=%lld", (long long)startTime,
- (long long)nowTime, (long long)playedTime);
- ALOGD("rtp-time(JB)=%d, played-rtp-time(JB)=%d, expired-rtp-time(JB)=%d isExpired=%d",
- rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+ printNowTimeUs(startTime, nowTime, playedTime);
+ printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
mShowQueueCnt++;
}
- ALOGV("start=%lld, now=%lld, played=%lld", (long long)startTime,
- (long long)nowTime, (long long)playedTime);
- ALOGV("rtp-time(JB)=%d, played-rtp-time(JB)=%d, expired-rtp-time(JB)=%d isExpired=%d",
- rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+ AAVCAssembler::addNack(source);
if (!isExpired) {
ALOGV("buffering in jitter buffer.");
return NOT_ENOUGH_DATA;
}
- if (isTooLate200)
+ if (isTooLate200) {
ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");
+ }
if (isTooLate300) {
- ALOGW("buffer arrived too late. 300ms..");
- ALOGW("start=%lld, now=%lld, played=%lld", (long long)startTime,
- (long long)nowTime, (long long)playedTime);
- ALOGW("rtp-time(JB)=%d, plyed-rtp-time(JB)=%d, exp-rtp-time(JB)=%d diff=%lld isExpired=%d",
- rtpTime, playedTimeRtp, expiredTimeInJb,
- ((long long)playedTimeRtp) - expiredTimeInJb, isExpired);
- ALOGW("expected Seq. NO =%d", buffer->int32Data());
+ ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",
+ ((long long)playedTimeRtp) - expiredTimeInJb, buffer->int32Data());
+ printNowTimeUs(startTime, nowTime, playedTime);
+ printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
- List<sp<ABuffer> >::iterator it = queue->begin();
- while (it != queue->end()) {
- CHECK((*it)->meta()->findInt32("rtp-time", &rtpTime));
- if (rtpTime + jitterTime >= playedTimeRtp) {
- mNextExpectedSeqNo = (*it)->int32Data();
- break;
- }
- it++;
- }
- source->noticeAbandonBuffer();
+ mNextExpectedSeqNo = pickProperSeq(queue, jitterTime, playedTimeRtp);
}
if (mNextExpectedSeqNoValid) {
int32_t size = queue->size();
- int32_t cnt = 0;
- List<sp<ABuffer> >::iterator it = queue->begin();
- while (it != queue->end()) {
- if ((uint32_t)(*it)->int32Data() >= mNextExpectedSeqNo) {
- break;
- }
+ int32_t cntRemove = deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
- it = queue->erase(it);
- cnt++;
- }
-
- if (cnt > 0) {
- source->noticeAbandonBuffer(cnt);
- ALOGW("delete %d of %d buffers", cnt, size);
+ if (cntRemove > 0) {
+ source->noticeAbandonBuffer(cntRemove);
+ ALOGW("delete %d of %d buffers", cntRemove, size);
}
if (queue->empty()) {
return NOT_ENOUGH_DATA;
@@ -187,12 +224,30 @@
}
}
+void AAVCAssembler::checkIFrameProvided(const sp<ABuffer> &buffer) {
+ if (buffer->size() == 0) {
+ return;
+ }
+ const uint8_t *data = buffer->data();
+ unsigned nalType = data[0] & 0x1f;
+ if (nalType == 0x5) {
+ mFirstIFrameProvided = true;
+ mLastIFrameProvidedAtMs = ALooper::GetNowUs() / 1000;
+
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ ALOGD("got First I-frame to be decoded. rtpTime=%u, size=%zu", rtpTime, buffer->size());
+ }
+}
+
void AAVCAssembler::addSingleNALUnit(const sp<ABuffer> &buffer) {
ALOGV("addSingleNALUnit of size %zu", buffer->size());
#if !LOG_NDEBUG
hexdump(buffer->data(), buffer->size());
#endif
+ checkIFrameProvided(buffer);
+
uint32_t rtpTime;
CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
@@ -280,6 +335,11 @@
size_t totalCount = 1;
bool complete = false;
+ uint32_t rtpTimeStartAt;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTimeStartAt));
+ uint32_t startSeqNo = buffer->int32Data();
+ bool pFrame = nalType == 0x1;
+
if (data[1] & 0x40) {
// Huh? End bit also set on the first buffer.
@@ -288,6 +348,8 @@
complete = true;
} else {
List<sp<ABuffer> >::iterator it = ++queue->begin();
+ int32_t connected = 1;
+ bool snapped = false;
while (it != queue->end()) {
ALOGV("sequence length %zu", totalCount);
@@ -297,26 +359,32 @@
size_t size = buffer->size();
if ((uint32_t)buffer->int32Data() != expectedSeqNo) {
- ALOGV("sequence not complete, expected seqNo %d, got %d",
- expectedSeqNo, (uint32_t)buffer->int32Data());
+ ALOGD("sequence not complete, expected seqNo %u, got %u, nalType %u",
+ expectedSeqNo, (unsigned)buffer->int32Data(), nalType);
+ snapped = true;
- return WRONG_SEQUENCE_NUMBER;
+ if (!pFrame) {
+ return WRONG_SEQUENCE_NUMBER;
+ }
}
+ if (!snapped) {
+ connected++;
+ }
+
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
if (size < 2
|| data[0] != indicator
|| (data[1] & 0x1f) != nalType
- || (data[1] & 0x80)) {
+ || (data[1] & 0x80)
+ || rtpTime != rtpTimeStartAt) {
ALOGV("Ignoring malformed FU buffer.");
// Delete the whole start of the FU.
- it = queue->begin();
- for (size_t i = 0; i <= totalCount; ++i) {
- it = queue->erase(it);
- }
-
mNextExpectedSeqNo = expectedSeqNo + 1;
+ deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
return MALFORMED_PACKET;
}
@@ -324,9 +392,17 @@
totalSize += size - 2;
++totalCount;
- expectedSeqNo = expectedSeqNo + 1;
+ expectedSeqNo = (uint32_t)buffer->int32Data() + 1;
if (data[1] & 0x40) {
+ if (pFrame && !recycleUnit(startSeqNo, expectedSeqNo,
+ connected, totalCount, 0.5f)) {
+ mNextExpectedSeqNo = expectedSeqNo;
+ deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
+
+ return MALFORMED_PACKET;
+ }
+
// This is the last fragment.
complete = true;
break;
@@ -433,22 +509,78 @@
msg->post();
}
+int32_t AAVCAssembler::pickProperSeq(const Queue *queue, uint32_t jit, int64_t play) {
+ sp<ABuffer> buffer = *(queue->begin());
+ uint32_t rtpTime;
+ int32_t nextSeqNo = buffer->int32Data();
+
+ Queue::const_iterator it = queue->begin();
+ while (it != queue->end()) {
+ CHECK((*it)->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ // if pkt in time exists, that should be the next pivot
+ if (rtpTime + jit >= play) {
+ nextSeqNo = (*it)->int32Data();
+ break;
+ }
+ it++;
+ }
+ return nextSeqNo;
+}
+
+bool AAVCAssembler::recycleUnit(uint32_t start, uint32_t end, uint32_t connected,
+ size_t avail, float goodRatio) {
+ float total = end - start;
+ float valid = connected;
+ float exist = avail;
+ bool isRecycle = (valid / total) >= goodRatio;
+
+ ALOGV("checking p-frame losses.. recvBufs %f valid %f diff %f recycle? %d",
+ exist, valid, total, isRecycle);
+
+ return isRecycle;
+}
+
+int32_t AAVCAssembler::deleteUnitUnderSeq(Queue *queue, uint32_t seq) {
+ int32_t initSize = queue->size();
+ Queue::iterator it = queue->begin();
+ while (it != queue->end()) {
+ if ((uint32_t)(*it)->int32Data() >= seq) {
+ break;
+ }
+ it++;
+ }
+ queue->erase(queue->begin(), it);
+ return initSize - queue->size();
+}
+
+inline void AAVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {
+ ALOGD("start=%lld, now=%lld, played=%lld",
+ (long long)start, (long long)now, (long long)play);
+}
+
+inline void AAVCAssembler::printRTPTime(uint32_t rtp, int64_t play, uint32_t exp, bool isExp) {
+ ALOGD("rtp-time(JB)=%u, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%u isExpired=%d",
+ rtp, (long long)play, exp, isExp);
+}
+
ARTPAssembler::AssemblyStatus AAVCAssembler::assembleMore(
const sp<ARTPSource> &source) {
AssemblyStatus status = addNALUnit(source);
if (status == MALFORMED_PACKET) {
- mAccessUnitDamaged = true;
+ uint64_t msecsSinceLastIFrame = (ALooper::GetNowUs() / 1000) - mLastIFrameProvidedAtMs;
+ if (msecsSinceLastIFrame > 1000) {
+ ALOGV("request FIR to get a new I-Frame, time since "
+ "last I-Frame %llu ms", (unsigned long long)msecsSinceLastIFrame);
+ source->onIssueFIRByAssembler();
+ }
}
return status;
}
void AAVCAssembler::packetLost() {
CHECK(mNextExpectedSeqNoValid);
- ALOGV("packetLost (expected %d)", mNextExpectedSeqNo);
-
+ ALOGD("packetLost (expected %u)", mNextExpectedSeqNo);
++mNextExpectedSeqNo;
-
- mAccessUnitDamaged = true;
}
void AAVCAssembler::onByeReceived() {
diff --git a/media/libstagefright/rtsp/AAVCAssembler.h b/media/libstagefright/rtsp/AAVCAssembler.h
index e19480c..913a868 100644
--- a/media/libstagefright/rtsp/AAVCAssembler.h
+++ b/media/libstagefright/rtsp/AAVCAssembler.h
@@ -31,6 +31,7 @@
struct AAVCAssembler : public ARTPAssembler {
explicit AAVCAssembler(const sp<AMessage> ¬ify);
+ typedef List<sp<ABuffer> > Queue;
protected:
virtual ~AAVCAssembler();
@@ -45,8 +46,12 @@
bool mNextExpectedSeqNoValid;
uint32_t mNextExpectedSeqNo;
bool mAccessUnitDamaged;
+ bool mFirstIFrameProvided;
+ uint64_t mLastIFrameProvidedAtMs;
List<sp<ABuffer> > mNALUnits;
+ int32_t addNack(const sp<ARTPSource> &source);
+ void checkIFrameProvided(const sp<ABuffer> &buffer);
AssemblyStatus addNALUnit(const sp<ARTPSource> &source);
void addSingleNALUnit(const sp<ABuffer> &buffer);
AssemblyStatus addFragmentedNALUnit(List<sp<ABuffer> > *queue);
@@ -54,6 +59,13 @@
void submitAccessUnit();
+ int32_t pickProperSeq(const Queue *q, uint32_t jit, int64_t play);
+ bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,
+ size_t avail, float goodRatio);
+ int32_t deleteUnitUnderSeq(Queue *q, uint32_t seq);
+ void printNowTimeUs(int64_t start, int64_t now, int64_t play);
+ void printRTPTime(uint32_t rtp, int64_t play, uint32_t exp, bool isExp);
+
DISALLOW_EVIL_CONSTRUCTORS(AAVCAssembler);
};
diff --git a/media/libstagefright/rtsp/AHEVCAssembler.cpp b/media/libstagefright/rtsp/AHEVCAssembler.cpp
index 93869fb..148a0ba 100644
--- a/media/libstagefright/rtsp/AHEVCAssembler.cpp
+++ b/media/libstagefright/rtsp/AHEVCAssembler.cpp
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-#define LOG_NDEBUG 0
+//#define LOG_NDEBUG 0
#define LOG_TAG "AHEVCAssembler"
#include <utils/Log.h>
@@ -25,6 +25,7 @@
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
+#include <include/HevcUtils.h>
#include <media/stagefright/foundation/hexdump.h>
#include <stdint.h>
@@ -46,7 +47,11 @@
mAccessUnitRTPTime(0),
mNextExpectedSeqNoValid(false),
mNextExpectedSeqNo(0),
- mAccessUnitDamaged(false) {
+ mAccessUnitDamaged(false),
+ mFirstIFrameProvided(false),
+ mLastIFrameProvidedAtMs(0),
+ mWidth(0),
+ mHeight(0) {
ALOGV("Constructor");
}
@@ -54,6 +59,66 @@
AHEVCAssembler::~AHEVCAssembler() {
}
+int32_t AHEVCAssembler::addNack(
+ const sp<ARTPSource> &source) {
+ List<sp<ABuffer>> *queue = source->queue();
+ int32_t nackCount = 0;
+
+ List<sp<ABuffer> >::iterator it = queue->begin();
+
+ if (it == queue->end()) {
+ return nackCount /* 0 */;
+ }
+
+ uint16_t queueHeadSeqNum = (*it)->int32Data();
+
+ // move to the packet after which RTCP:NACK was sent.
+ for (; it != queue->end(); ++it) {
+ int32_t seqNum = (*it)->int32Data();
+ if (seqNum >= source->mHighestNackNumber) {
+ break;
+ }
+ }
+
+ int32_t nackStartAt = -1;
+
+ while (it != queue->end()) {
+ int32_t seqBeforeLast = (*it)->int32Data();
+ // increase iterator.
+ if ((++it) == queue->end()) {
+ break;
+ }
+
+ int32_t seqLast = (*it)->int32Data();
+
+ if ((seqLast - seqBeforeLast) < 0) {
+ ALOGD("addNack: found end of seqNum from(%d) to(%d)", seqBeforeLast, seqLast);
+ source->mHighestNackNumber = 0;
+ }
+
+ // missed packet found
+ if (seqLast > (seqBeforeLast + 1) &&
+ // we didn't send RTCP:NACK for this packet yet.
+ (seqLast - 1) > source->mHighestNackNumber) {
+ source->mHighestNackNumber = seqLast -1;
+ nackStartAt = seqBeforeLast + 1;
+ break;
+ }
+
+ }
+
+ if (nackStartAt != -1) {
+ nackCount = source->mHighestNackNumber - nackStartAt + 1;
+ ALOGD("addNack: nackCount=%d, nackFrom=%d, nackTo=%d", nackCount,
+ nackStartAt, source->mHighestNackNumber);
+
+ uint16_t mask = (uint16_t)(0xffff) >> (16 - nackCount + 1);
+ source->setSeqNumToNACK(nackStartAt, mask, queueHeadSeqNum);
+ }
+
+ return nackCount;
+}
+
ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit(
const sp<ARTPSource> &source) {
List<sp<ABuffer> > *queue = source->queue();
@@ -63,33 +128,54 @@
}
sp<ABuffer> buffer = *queue->begin();
- int32_t rtpTime;
- CHECK(buffer->meta()->findInt32("rtp-time", &rtpTime));
+ buffer->meta()->setObject("source", source);
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
int64_t startTime = source->mFirstSysTime / 1000;
int64_t nowTime = ALooper::GetNowUs() / 1000;
int64_t playedTime = nowTime - startTime;
- int32_t playedTimeRtp = source->mFirstRtpTime +
+ int64_t playedTimeRtp = source->mFirstRtpTime +
(((uint32_t)playedTime) * (source->mClockRate / 1000));
- int32_t expiredTimeInJb = rtpTime + (source->mClockRate / 5);
+ const uint32_t jitterTime = (uint32_t)(source->mClockRate / ((float)1000 / (source->mJbTimeMs)));
+ uint32_t expiredTimeInJb = rtpTime + jitterTime;
bool isExpired = expiredTimeInJb <= (playedTimeRtp);
- ALOGV("start=%lld, now=%lld, played=%lld", (long long)startTime,
- (long long)nowTime, (long long)playedTime);
- ALOGV("rtp-time(JB)=%d, played-rtp-time(JB)=%d, expired-rtp-time(JB)=%d isExpired=%d",
- rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+ bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);
+ bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));
+
+ if (mShowQueueCnt < 20) {
+ showCurrentQueue(queue);
+ printNowTimeUs(startTime, nowTime, playedTime);
+ printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+ mShowQueueCnt++;
+ }
+
+ AHEVCAssembler::addNack(source);
if (!isExpired) {
ALOGV("buffering in jitter buffer.");
return NOT_ENOUGH_DATA;
}
- if (mNextExpectedSeqNoValid) {
- List<sp<ABuffer> >::iterator it = queue->begin();
- while (it != queue->end()) {
- if ((uint32_t)(*it)->int32Data() >= mNextExpectedSeqNo) {
- break;
- }
+ if (isTooLate200) {
+ ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");
+ }
- it = queue->erase(it);
+ if (isTooLate300) {
+ ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",
+ ((long long)playedTimeRtp) - expiredTimeInJb, buffer->int32Data());
+ printNowTimeUs(startTime, nowTime, playedTime);
+ printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+
+ mNextExpectedSeqNo = pickProperSeq(queue, jitterTime, playedTimeRtp);
+ }
+
+ if (mNextExpectedSeqNoValid) {
+ int32_t size = queue->size();
+ int32_t cntRemove = deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
+
+ if (cntRemove > 0) {
+ source->noticeAbandonBuffer(cntRemove);
+ ALOGW("delete %d of %d buffers", cntRemove, size);
}
if (queue->empty()) {
@@ -154,15 +240,74 @@
}
}
+void AHEVCAssembler::checkSpsUpdated(const sp<ABuffer> &buffer) {
+ if (buffer->size() == 0) {
+ return;
+ }
+ const uint8_t *data = buffer->data();
+ HevcParameterSets paramSets;
+ unsigned nalType = (data[0] >> 1) & H265_NALU_MASK;
+ if (nalType == H265_NALU_SPS) {
+ int32_t width = 0, height = 0;
+ paramSets.FindHEVCDimensions(buffer, &width, &height);
+ ALOGV("existing resolution (%u x %u)", mWidth, mHeight);
+ if (width != mWidth || height != mHeight) {
+ mFirstIFrameProvided = false;
+ mWidth = width;
+ mHeight = height;
+ ALOGD("found a new resolution (%u x %u)", mWidth, mHeight);
+ }
+ }
+}
+
+void AHEVCAssembler::checkIFrameProvided(const sp<ABuffer> &buffer) {
+ if (buffer->size() == 0) {
+ return;
+ }
+ const uint8_t *data = buffer->data();
+ unsigned nalType = (data[0] >> 1) & H265_NALU_MASK;
+ if (nalType > 0x0F && nalType < 0x18) {
+ mLastIFrameProvidedAtMs = ALooper::GetNowUs() / 1000;
+ if (!mFirstIFrameProvided) {
+ mFirstIFrameProvided = true;
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ ALOGD("got First I-frame to be decoded. rtpTime=%d, size=%zu", rtpTime, buffer->size());
+ }
+ }
+}
+
+bool AHEVCAssembler::dropFramesUntilIframe(const sp<ABuffer> &buffer) {
+ if (buffer->size() == 0) {
+ return false;
+ }
+ const uint8_t *data = buffer->data();
+ unsigned nalType = (data[0] >> 1) & H265_NALU_MASK;
+ return !mFirstIFrameProvided && nalType < 0x10;
+}
+
void AHEVCAssembler::addSingleNALUnit(const sp<ABuffer> &buffer) {
ALOGV("addSingleNALUnit of size %zu", buffer->size());
#if !LOG_NDEBUG
hexdump(buffer->data(), buffer->size());
#endif
+ checkSpsUpdated(buffer);
+ checkIFrameProvided(buffer);
uint32_t rtpTime;
CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ if (dropFramesUntilIframe(buffer)) {
+ sp<ARTPSource> source = nullptr;
+ buffer->meta()->findObject("source", (sp<android::RefBase>*)&source);
+ if (source != nullptr) {
+ ALOGD("Issued FIR to get the I-frame");
+ source->onIssueFIRByAssembler();
+ }
+ ALOGD("drop P-frames till an I-frame provided. rtpTime %u", rtpTime);
+ return;
+ }
+
if (!mNALUnits.empty() && rtpTime != mAccessUnitRTPTime) {
submitAccessUnit();
}
@@ -260,6 +405,11 @@
size_t totalCount = 1;
bool complete = false;
+ uint32_t rtpTimeStartAt;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTimeStartAt));
+ uint32_t startSeqNo = buffer->int32Data();
+ bool pFrame = (nalType < 0x10);
+
if (data[2] & 0x40) {
// Huh? End bit also set on the first buffer.
@@ -268,6 +418,8 @@
complete = true;
} else {
List<sp<ABuffer> >::iterator it = ++queue->begin();
+ int32_t connected = 1;
+ bool snapped = false;
while (it != queue->end()) {
ALOGV("sequence length %zu", totalCount);
@@ -277,26 +429,32 @@
size_t size = buffer->size();
if ((uint32_t)buffer->int32Data() != expectedSeqNo) {
- ALOGV("sequence not complete, expected seqNo %d, got %d",
- expectedSeqNo, (uint32_t)buffer->int32Data());
+ ALOGV("sequence not complete, expected seqNo %u, got %u, nalType %u",
+ expectedSeqNo, (uint32_t)buffer->int32Data(), nalType);
+ snapped = true;
- return WRONG_SEQUENCE_NUMBER;
+ if (!pFrame) {
+ return WRONG_SEQUENCE_NUMBER;
+ }
}
+ if (!snapped) {
+ connected++;
+ }
+
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
if (size < 3
|| ((data[0] >> 1) & H265_NALU_MASK) != indicator
|| (data[2] & H265_NALU_MASK) != nalType
- || (data[2] & 0x80)) {
+ || (data[2] & 0x80)
+ || rtpTime != rtpTimeStartAt) {
ALOGV("Ignoring malformed FU buffer.");
// Delete the whole start of the FU.
- it = queue->begin();
- for (size_t i = 0; i <= totalCount; ++i) {
- it = queue->erase(it);
- }
-
mNextExpectedSeqNo = expectedSeqNo + 1;
+ deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
return MALFORMED_PACKET;
}
@@ -304,9 +462,16 @@
totalSize += size - 3;
++totalCount;
- expectedSeqNo = expectedSeqNo + 1;
+ expectedSeqNo = (uint32_t)buffer->int32Data() + 1;
if (data[2] & 0x40) {
+ if (pFrame && !recycleUnit(startSeqNo, expectedSeqNo,
+ connected, totalCount, 0.5f)) {
+ mNextExpectedSeqNo = expectedSeqNo;
+ deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
+
+ return MALFORMED_PACKET;
+ }
// This is the last fragment.
complete = true;
break;
@@ -335,6 +500,7 @@
unit->data()[1] = tid;
size_t offset = 2;
+ int32_t cvo = -1;
List<sp<ABuffer> >::iterator it = queue->begin();
for (size_t i = 0; i < totalCount; ++i) {
const sp<ABuffer> &buffer = *it;
@@ -345,6 +511,7 @@
#endif
memcpy(unit->data() + offset, buffer->data() + 3, buffer->size() - 3);
+ buffer->meta()->findInt32("cvo", &cvo);
offset += buffer->size() - 3;
it = queue->erase(it);
@@ -352,6 +519,10 @@
unit->setRange(0, totalSize);
+ if (cvo >= 0) {
+ unit->meta()->setInt32("cvo", cvo);
+ }
+
addSingleNALUnit(unit);
ALOGV("successfully assembled a NAL unit from fragments.");
@@ -372,6 +543,7 @@
sp<ABuffer> accessUnit = new ABuffer(totalSize);
size_t offset = 0;
+ int32_t cvo = -1;
for (List<sp<ABuffer> >::iterator it = mNALUnits.begin();
it != mNALUnits.end(); ++it) {
memcpy(accessUnit->data() + offset, "\x00\x00\x00\x01", 4);
@@ -380,6 +552,7 @@
sp<ABuffer> nal = *it;
memcpy(accessUnit->data() + offset, nal->data(), nal->size());
offset += nal->size();
+ nal->meta()->findInt32("cvo", &cvo);
}
CopyTimes(accessUnit, *mNALUnits.begin());
@@ -388,6 +561,9 @@
printf(mAccessUnitDamaged ? "X" : ".");
fflush(stdout);
#endif
+ if (cvo >= 0) {
+ accessUnit->meta()->setInt32("cvo", cvo);
+ }
if (mAccessUnitDamaged) {
accessUnit->meta()->setInt32("damaged", true);
@@ -401,22 +577,80 @@
msg->post();
}
+int32_t AHEVCAssembler::pickProperSeq(const Queue *queue, uint32_t jit, int64_t play) {
+ sp<ABuffer> buffer = *(queue->begin());
+ uint32_t rtpTime;
+ int32_t nextSeqNo = buffer->int32Data();
+
+ Queue::const_iterator it = queue->begin();
+ while (it != queue->end()) {
+ CHECK((*it)->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ // if pkt in time exists, that should be the next pivot
+ if (rtpTime + jit >= play) {
+ nextSeqNo = (*it)->int32Data();
+ break;
+ }
+ it++;
+ }
+ return nextSeqNo;
+}
+
+bool AHEVCAssembler::recycleUnit(uint32_t start, uint32_t end, uint32_t connected,
+ size_t avail, float goodRatio) {
+ float total = end - start;
+ float valid = connected;
+ float exist = avail;
+ bool isRecycle = (valid / total) >= goodRatio;
+
+ ALOGV("checking p-frame losses.. recvBufs %f valid %f diff %f recycle? %d",
+ exist, valid, total, isRecycle);
+
+ return isRecycle;
+}
+
+int32_t AHEVCAssembler::deleteUnitUnderSeq(Queue *queue, uint32_t seq) {
+ int32_t initSize = queue->size();
+ Queue::iterator it = queue->begin();
+ while (it != queue->end()) {
+ if ((uint32_t)(*it)->int32Data() >= seq) {
+ break;
+ }
+ it++;
+ }
+ queue->erase(queue->begin(), it);
+ return initSize - queue->size();
+}
+
+inline void AHEVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {
+ ALOGD("start=%lld, now=%lld, played=%lld",
+ (long long)start, (long long)now, (long long)play);
+}
+
+inline void AHEVCAssembler::printRTPTime(uint32_t rtp, int64_t play, uint32_t exp, bool isExp) {
+ ALOGD("rtp-time(JB)=%u, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%u isExpired=%d",
+ rtp, (long long)play, exp, isExp);
+}
+
+
ARTPAssembler::AssemblyStatus AHEVCAssembler::assembleMore(
const sp<ARTPSource> &source) {
AssemblyStatus status = addNALUnit(source);
if (status == MALFORMED_PACKET) {
- mAccessUnitDamaged = true;
+ uint64_t msecsSinceLastIFrame = (ALooper::GetNowUs() / 1000) - mLastIFrameProvidedAtMs;
+ if (msecsSinceLastIFrame > 1000) {
+ ALOGV("request FIR to get a new I-Frame, time after "
+ "last I-Frame in %llu ms", (unsigned long long)msecsSinceLastIFrame);
+ source->onIssueFIRByAssembler();
+ }
}
return status;
}
void AHEVCAssembler::packetLost() {
CHECK(mNextExpectedSeqNoValid);
- ALOGV("packetLost (expected %d)", mNextExpectedSeqNo);
+ ALOGD("packetLost (expected %u)", mNextExpectedSeqNo);
++mNextExpectedSeqNo;
-
- mAccessUnitDamaged = true;
}
void AHEVCAssembler::onByeReceived() {
diff --git a/media/libstagefright/rtsp/AHEVCAssembler.h b/media/libstagefright/rtsp/AHEVCAssembler.h
index cc20622..16fc1c8 100644
--- a/media/libstagefright/rtsp/AHEVCAssembler.h
+++ b/media/libstagefright/rtsp/AHEVCAssembler.h
@@ -31,6 +31,8 @@
struct AHEVCAssembler : public ARTPAssembler {
AHEVCAssembler(const sp<AMessage> ¬ify);
+ typedef List<sp<ABuffer> > Queue;
+
protected:
virtual ~AHEVCAssembler();
@@ -45,8 +47,16 @@
bool mNextExpectedSeqNoValid;
uint32_t mNextExpectedSeqNo;
bool mAccessUnitDamaged;
+ bool mFirstIFrameProvided;
+ uint64_t mLastIFrameProvidedAtMs;
+ int32_t mWidth;
+ int32_t mHeight;
List<sp<ABuffer> > mNALUnits;
+ int32_t addNack(const sp<ARTPSource> &source);
+ void checkSpsUpdated(const sp<ABuffer> &buffer);
+ void checkIFrameProvided(const sp<ABuffer> &buffer);
+ bool dropFramesUntilIframe(const sp<ABuffer> &buffer);
AssemblyStatus addNALUnit(const sp<ARTPSource> &source);
void addSingleNALUnit(const sp<ABuffer> &buffer);
AssemblyStatus addFragmentedNALUnit(List<sp<ABuffer> > *queue);
@@ -54,6 +64,13 @@
void submitAccessUnit();
+ int32_t pickProperSeq(const Queue *queue, uint32_t jit, int64_t play);
+ bool recycleUnit(uint32_t start, uint32_t end, uint32_t conneceted,
+ size_t avail, float goodRatio);
+ int32_t deleteUnitUnderSeq(Queue *queue, uint32_t seq);
+ void printNowTimeUs(int64_t start, int64_t now, int64_t play);
+ void printRTPTime(uint32_t rtp, int64_t play, uint32_t exp, bool isExp);
+
DISALLOW_EVIL_CONSTRUCTORS(AHEVCAssembler);
};
diff --git a/media/libstagefright/rtsp/ARTPConnection.cpp b/media/libstagefright/rtsp/ARTPConnection.cpp
index 1346c9a..f57077c 100644
--- a/media/libstagefright/rtsp/ARTPConnection.cpp
+++ b/media/libstagefright/rtsp/ARTPConnection.cpp
@@ -78,7 +78,9 @@
: mFlags(flags),
mPollEventPending(false),
mLastReceiverReportTimeUs(-1),
- mLastBitrateReportTimeUs(-1) {
+ mLastBitrateReportTimeUs(-1),
+ mTargetBitrate(-1),
+ mJbTimeMs(300) {
}
ARTPConnection::~ARTPConnection() {
@@ -439,6 +441,24 @@
continue;
}
+ // add NACK and FIR that needs to be sent immediately.
+ sp<ABuffer> buffer = new ABuffer(kMaxUDPSize);
+ for (size_t i = 0; i < it->mSources.size(); ++i) {
+ buffer->setRange(0, 0);
+ int cnt = it->mSources.valueAt(i)->addNACK(buffer);
+ if (cnt > 0) {
+ ALOGV("Send NACK for lost %d Packets", cnt);
+ send(&*it, buffer);
+ }
+
+ buffer->setRange(0, 0);
+ it->mSources.valueAt(i)->addFIR(buffer);
+ if (buffer->size() > 0) {
+ ALOGD("Send FIR immediately for lost Packets");
+ send(&*it, buffer);
+ }
+ }
+
++it;
}
}
@@ -524,8 +544,9 @@
(!receiveRTP && s->mNumRTCPPacketsReceived == 0)
? sizeSockSt : 0;
- if (mFlags & kViLTEConnection)
+ if (mFlags & kViLTEConnection) {
remoteAddrLen = 0;
+ }
ssize_t nbytes;
do {
@@ -1012,8 +1033,12 @@
source = new ARTPSource(
srcId, info->mSessionDesc, info->mIndex, info->mNotifyMsg);
+ if (mFlags & kViLTEConnection) {
+ source->setPeriodicFIR(false);
+ }
+
source->setSelfID(mSelfID);
- source->setMinMaxBitrate(mMinBitrate, mMaxBitrate);
+ source->setJbTime(mJbTimeMs > 0 ? mJbTimeMs : 300);
info->mSources.add(srcId, source);
} else {
source = info->mSources.valueAt(index);
@@ -1033,9 +1058,12 @@
mSelfID = selfID;
}
-void ARTPConnection::setMinMaxBitrate(int32_t min, int32_t max) {
- mMinBitrate = min;
- mMaxBitrate = max;
+void ARTPConnection::setJbTime(const uint32_t jbTimeMs) {
+ mJbTimeMs = jbTimeMs;
+}
+
+void ARTPConnection::setTargetBitrate(int32_t targetBitrate) {
+ mTargetBitrate = targetBitrate;
}
void ARTPConnection::checkRxBitrate(int64_t nowUs) {
@@ -1068,17 +1096,8 @@
for (size_t i = 0; i < s->mSources.size(); ++i) {
sp<ARTPSource> source = s->mSources.valueAt(i);
- source->setBitrateData(bitrate, nowUs);
- source->setTargetBitrate();
- source->addTMMBR(buffer);
- if (source->isNeedToDowngrade()) {
- sp<AMessage> notify = s->mNotifyMsg->dup();
- notify->setInt32("rtcp-event", 1);
- notify->setInt32("payload-type", 400);
- notify->setInt32("feedback-type", 1);
- notify->setInt32("sender", source->getSelfID());
- notify->post();
- }
+ source->notifyPktInfo(bitrate, nowUs);
+ source->addTMMBR(buffer, mTargetBitrate);
}
if (buffer->size() > 0) {
ALOGV("Sending TMMBR...");
diff --git a/media/libstagefright/rtsp/ARTPConnection.h b/media/libstagefright/rtsp/ARTPConnection.h
index 712eec5..7c8218f 100644
--- a/media/libstagefright/rtsp/ARTPConnection.h
+++ b/media/libstagefright/rtsp/ARTPConnection.h
@@ -46,7 +46,8 @@
void injectPacket(int index, const sp<ABuffer> &buffer);
void setSelfID(const uint32_t selfID);
- void setMinMaxBitrate(int32_t min, int32_t max);
+ void setJbTime(const uint32_t jbTimeMs);
+ void setTargetBitrate(int32_t targetBitrate);
// Creates a pair of UDP datagram sockets bound to adjacent ports
// (the rtpSocket is bound to an even port, the rtcpSocket to the
@@ -85,9 +86,10 @@
int64_t mLastBitrateReportTimeUs;
int32_t mSelfID;
+ int32_t mTargetBitrate;
- int32_t mMinBitrate;
- int32_t mMaxBitrate;
+ uint32_t mJbTimeMs;
+
int32_t mCumulativeBytes;
void onAddStream(const sp<AMessage> &msg);
diff --git a/media/libstagefright/rtsp/ARTPSource.cpp b/media/libstagefright/rtsp/ARTPSource.cpp
index bbe9d94..6303fc4 100644
--- a/media/libstagefright/rtsp/ARTPSource.cpp
+++ b/media/libstagefright/rtsp/ARTPSource.cpp
@@ -46,15 +46,21 @@
mFirstRtpTime(0),
mFirstSysTime(0),
mClockRate(0),
+ mJbTimeMs(300), // default jitter buffer time is 300ms.
+ mFirstSsrc(0),
+ mHighestNackNumber(0),
mID(id),
mHighestSeqNumber(0),
mPrevExpected(0),
mBaseSeqNumber(0),
mNumBuffersReceived(0),
mPrevNumBuffersReceived(0),
+ mPrevExpectedForRR(0),
+ mPrevNumBuffersReceivedForRR(0),
mLastNTPTime(0),
mLastNTPTimeUpdateUs(0),
mIssueFIRRequests(false),
+ mIssueFIRByAssembler(false),
mLastFIRRequestUs(-1),
mNextFIRSeqNo((rand() * 256.0) / RAND_MAX),
mNotify(notify) {
@@ -120,20 +126,29 @@
bool ARTPSource::queuePacket(const sp<ABuffer> &buffer) {
uint32_t seqNum = (uint32_t)buffer->int32Data();
+ int32_t ssrc = 0;
+ buffer->meta()->findInt32("ssrc", &ssrc);
+
if (mNumBuffersReceived++ == 0 && mFirstSysTime == 0) {
- int32_t firstRtpTime;
- CHECK(buffer->meta()->findInt32("rtp-time", &firstRtpTime));
+ uint32_t firstRtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&firstRtpTime));
mFirstSysTime = ALooper::GetNowUs();
mHighestSeqNumber = seqNum;
mBaseSeqNumber = seqNum;
mFirstRtpTime = firstRtpTime;
- ALOGV("first-rtp arrived: first-rtp-time=%d, sys-time=%lld, seq-num=%u",
- mFirstRtpTime, (long long)mFirstSysTime, mHighestSeqNumber);
+ mFirstSsrc = ssrc;
+ ALOGD("first-rtp arrived: first-rtp-time=%d, sys-time=%lld, seq-num=%u, ssrc=%d",
+ mFirstRtpTime, (long long)mFirstSysTime, mHighestSeqNumber, mFirstSsrc);
mClockRate = 90000;
mQueue.push_back(buffer);
return true;
}
+ if (mFirstSsrc != ssrc) {
+ ALOGW("Discarding a buffer due to unexpected ssrc");
+ return false;
+ }
+
// Only the lower 16-bit of the sequence numbers are transmitted,
// derive the high-order bits by choosing the candidate closest
// to the highest sequence number (extended to 32 bits) received so far.
@@ -196,20 +211,34 @@
}
void ARTPSource::addFIR(const sp<ABuffer> &buffer) {
- if (!mIssueFIRRequests) {
+ if (!mIssueFIRRequests && !mIssueFIRByAssembler) {
return;
}
+ bool send = false;
int64_t nowUs = ALooper::GetNowUs();
- if (mLastFIRRequestUs >= 0 && mLastFIRRequestUs + 5000000LL > nowUs) {
- // Send FIR requests at most every 5 secs.
+ int64_t usecsSinceLastFIR = nowUs - mLastFIRRequestUs;
+ if (mLastFIRRequestUs < 0) {
+ // A first FIR, just send it.
+ send = true;
+ } else if (mIssueFIRByAssembler && (usecsSinceLastFIR > 1000000)) {
+ // A FIR issued by Assembler.
+ // Send it if last FIR is not sent within a sec.
+ send = true;
+ } else if (mIssueFIRRequests && (usecsSinceLastFIR > 5000000)) {
+ // A FIR issued periodically reagardless packet loss.
+ // Send it if last FIR is not sent within 5 secs.
+ send = true;
+ }
+
+ if (!send) {
return;
}
mLastFIRRequestUs = nowUs;
if (buffer->size() + 20 > buffer->capacity()) {
- ALOGW("RTCP buffer too small to accomodate FIR.");
+ ALOGW("RTCP buffer too small to accommodate FIR.");
return;
}
@@ -218,7 +247,7 @@
data[0] = 0x80 | 4;
data[1] = 206; // PSFB
data[2] = 0;
- data[3] = 4;
+ data[3] = 4; // total (4+1) * sizeof(int32_t) = 20 bytes
data[4] = kSourceID >> 24;
data[5] = (kSourceID >> 16) & 0xff;
data[6] = (kSourceID >> 8) & 0xff;
@@ -240,14 +269,16 @@
data[18] = 0x00;
data[19] = 0x00;
- buffer->setRange(buffer->offset(), buffer->size() + 20);
+ buffer->setRange(buffer->offset(), buffer->size() + (data[3] + 1) * sizeof(int32_t));
+
+ mIssueFIRByAssembler = false;
ALOGV("Added FIR request.");
}
void ARTPSource::addReceiverReport(const sp<ABuffer> &buffer) {
if (buffer->size() + 32 > buffer->capacity()) {
- ALOGW("RTCP buffer too small to accomodate RR.");
+ ALOGW("RTCP buffer too small to accommodate RR.");
return;
}
@@ -255,16 +286,16 @@
// According to appendix A.3 in RFC 3550
uint32_t expected = mHighestSeqNumber - mBaseSeqNumber + 1;
- int64_t intervalExpected = expected - mPrevExpected;
- int64_t intervalReceived = mNumBuffersReceived - mPrevNumBuffersReceived;
+ int64_t intervalExpected = expected - mPrevExpectedForRR;
+ int64_t intervalReceived = mNumBuffersReceived - mPrevNumBuffersReceivedForRR;
int64_t intervalPacketLost = intervalExpected - intervalReceived;
if (intervalExpected > 0 && intervalPacketLost > 0) {
fraction = (intervalPacketLost << 8) / intervalExpected;
}
- mPrevExpected = expected;
- mPrevNumBuffersReceived = mNumBuffersReceived;
+ mPrevExpectedForRR = expected;
+ mPrevNumBuffersReceivedForRR = mNumBuffersReceived;
int32_t cumulativePacketLost = (int32_t)expected - mNumBuffersReceived;
uint8_t *data = buffer->data() + buffer->size();
@@ -272,7 +303,7 @@
data[0] = 0x80 | 1;
data[1] = 201; // RR
data[2] = 0;
- data[3] = 7;
+ data[3] = 7; // total (7+1) * sizeof(int32_t) = 32 bytes
data[4] = kSourceID >> 24;
data[5] = (kSourceID >> 16) & 0xff;
data[6] = (kSourceID >> 8) & 0xff;
@@ -318,18 +349,18 @@
data[30] = (DLSR >> 8) & 0xff;
data[31] = DLSR & 0xff;
- buffer->setRange(buffer->offset(), buffer->size() + 32);
+ buffer->setRange(buffer->offset(), buffer->size() + (data[3] + 1) * sizeof(int32_t));
}
-void ARTPSource::addTMMBR(const sp<ABuffer> &buffer) {
+void ARTPSource::addTMMBR(const sp<ABuffer> &buffer, int32_t targetBitrate) {
if (buffer->size() + 20 > buffer->capacity()) {
ALOGW("RTCP buffer too small to accommodate RR.");
return;
}
- int32_t targetBitrate = mQualManager.getTargetBitrate();
- if (targetBitrate <= 0)
+ if (targetBitrate <= 0) {
return;
+ }
uint8_t *data = buffer->data() + buffer->size();
@@ -363,52 +394,145 @@
data[18] = (mantissa & 0x0007f) << 1;
data[19] = 40; // 40 bytes overhead;
- buffer->setRange(buffer->offset(), buffer->size() + 20);
+ buffer->setRange(buffer->offset(), buffer->size() + (data[3] + 1) * sizeof(int32_t));
+}
+
+int ARTPSource::addNACK(const sp<ABuffer> &buffer) {
+ constexpr size_t kMaxFCIs = 10; // max number of FCIs
+ if (buffer->size() + (3 + kMaxFCIs) * sizeof(int32_t) > buffer->capacity()) {
+ ALOGW("RTCP buffer too small to accommodate NACK.");
+ return -1;
+ }
+
+ uint8_t *data = buffer->data() + buffer->size();
+
+ data[0] = 0x80 | 1; // Generic NACK
+ data[1] = 205; // TSFB
+ data[2] = 0;
+ data[3] = 0; // will be decided later
+ data[4] = kSourceID >> 24;
+ data[5] = (kSourceID >> 16) & 0xff;
+ data[6] = (kSourceID >> 8) & 0xff;
+ data[7] = kSourceID & 0xff;
+
+ data[8] = mID >> 24;
+ data[9] = (mID >> 16) & 0xff;
+ data[10] = (mID >> 8) & 0xff;
+ data[11] = mID & 0xff;
+
+ List<int> list;
+ List<int>::iterator it;
+ getSeqNumToNACK(list, kMaxFCIs);
+ size_t cnt = 0;
+
+ int *FCI = (int *)(data + 12);
+ for (it = list.begin(); it != list.end() && cnt < kMaxFCIs; it++) {
+ *(FCI + cnt) = *it;
+ cnt++;
+ }
+
+ data[3] = (3 + cnt) - 1; // total (3 + #ofFCI) * sizeof(int32_t) byte
+
+ buffer->setRange(buffer->offset(), buffer->size() + (data[3] + 1) * sizeof(int32_t));
+
+ return cnt;
+}
+
+int ARTPSource::getSeqNumToNACK(List<int>& list, int size) {
+ AutoMutex _l(mMapLock);
+ int cnt = 0;
+
+ std::map<uint16_t, infoNACK>::iterator it;
+ for(it = mNACKMap.begin(); it != mNACKMap.end() && cnt < size; it++) {
+ infoNACK &info_it = it->second;
+ if (info_it.needToNACK) {
+ info_it.needToNACK = false;
+ // switch LSB to MSB for sending N/W
+ uint32_t FCI;
+ uint8_t *temp = (uint8_t *)&FCI;
+ temp[0] = (info_it.seqNum >> 8) & 0xff;
+ temp[1] = (info_it.seqNum) & 0xff;
+ temp[2] = (info_it.mask >> 8) & 0xff;
+ temp[3] = (info_it.mask) & 0xff;
+
+ list.push_back(FCI);
+ cnt++;
+ }
+ }
+
+ return cnt;
+}
+
+void ARTPSource::setSeqNumToNACK(uint16_t seqNum, uint16_t mask, uint16_t nowJitterHeadSeqNum) {
+ AutoMutex _l(mMapLock);
+ infoNACK info = {seqNum, mask, nowJitterHeadSeqNum, true};
+ std::map<uint16_t, infoNACK>::iterator it;
+
+ it = mNACKMap.find(seqNum);
+ if (it != mNACKMap.end()) {
+ infoNACK &info_it = it->second;
+ // renew if (mask or head seq) is changed
+ if ((info_it.mask != mask) || (info_it.nowJitterHeadSeqNum != nowJitterHeadSeqNum)) {
+ info_it = info;
+ }
+ } else {
+ mNACKMap[seqNum] = info;
+ }
+
+ // delete all NACK far from current Jitter's first sequence number
+ it = mNACKMap.begin();
+ while (it != mNACKMap.end()) {
+ infoNACK &info_it = it->second;
+
+ int diff = nowJitterHeadSeqNum - info_it.nowJitterHeadSeqNum;
+ if (diff > 100) {
+ ALOGV("Delete %d pkt from NACK map ", info_it.seqNum);
+ it = mNACKMap.erase(it);
+ } else {
+ it++;
+ }
+ }
+
}
uint32_t ARTPSource::getSelfID() {
return kSourceID;
}
+
void ARTPSource::setSelfID(const uint32_t selfID) {
kSourceID = selfID;
}
-void ARTPSource::setMinMaxBitrate(int32_t min, int32_t max) {
- mQualManager.setMinMaxBitrate(min, max);
+void ARTPSource::setJbTime(const uint32_t jbTimeMs) {
+ mJbTimeMs = jbTimeMs;
}
-void ARTPSource::setBitrateData(int32_t bitrate, int64_t time) {
- mQualManager.setBitrateData(bitrate, time);
+void ARTPSource::setPeriodicFIR(bool enable) {
+ ALOGD("setPeriodicFIR %d", enable);
+ mIssueFIRRequests = enable;
}
-void ARTPSource::setTargetBitrate() {
- uint8_t fraction = 0;
+void ARTPSource::notifyPktInfo(int32_t bitrate, int64_t /*time*/) {
+ sp<AMessage> notify = mNotify->dup();
+ notify->setInt32("rtcp-event", 1);
+ notify->setInt32("payload-type", 102);
+ notify->setInt32("feedback-type", 0);
+ // sending target bitrate up to application to share rtp quality.
+ notify->setInt32("bit-rate", bitrate);
+ notify->setInt32("highest-seq-num", mHighestSeqNumber);
+ notify->setInt32("base-seq-num", mBaseSeqNumber);
+ notify->setInt32("prev-expected", mPrevExpected);
+ notify->setInt32("num-buf-recv", mNumBuffersReceived);
+ notify->setInt32("prev-num-buf-recv", mPrevNumBuffersReceived);
+ notify->post();
- // According to appendix A.3 in RFC 3550
uint32_t expected = mHighestSeqNumber - mBaseSeqNumber + 1;
- int64_t intervalExpected = expected - mPrevExpected;
- int64_t intervalReceived = mNumBuffersReceived - mPrevNumBuffersReceived;
- int64_t intervalPacketLost = intervalExpected - intervalReceived;
-
- ALOGI("UID %p expectedPkts %lld lostPkts %lld", this, (long long)intervalExpected, (long long)intervalPacketLost);
-
- if (intervalPacketLost < 0 || intervalExpected == 0)
- fraction = 0;
- else if (intervalExpected <= intervalPacketLost)
- fraction = 255;
- else
- fraction = (intervalPacketLost << 8) / intervalExpected;
-
- mQualManager.setTargetBitrate(fraction, ALooper::GetNowUs(), intervalExpected < 5);
+ mPrevExpected = expected;
+ mPrevNumBuffersReceived = mNumBuffersReceived;
}
-bool ARTPSource::isNeedToReport() {
- int64_t intervalReceived = mNumBuffersReceived - mPrevNumBuffersReceived;
- return (intervalReceived > 0) ? true : false;
-}
-
-bool ARTPSource::isNeedToDowngrade() {
- return mQualManager.isNeedToDowngrade();
+void ARTPSource::onIssueFIRByAssembler() {
+ mIssueFIRByAssembler = true;
}
void ARTPSource::noticeAbandonBuffer(int cnt) {
diff --git a/media/libstagefright/rtsp/ARTPSource.h b/media/libstagefright/rtsp/ARTPSource.h
index 652e753..ea683a0 100644
--- a/media/libstagefright/rtsp/ARTPSource.h
+++ b/media/libstagefright/rtsp/ARTPSource.h
@@ -23,7 +23,9 @@
#include <media/stagefright/foundation/ABase.h>
#include <utils/List.h>
#include <utils/RefBase.h>
-#include <QualManager.h>
+#include <utils/Thread.h>
+
+#include <map>
namespace android {
@@ -46,23 +48,28 @@
void addReceiverReport(const sp<ABuffer> &buffer);
void addFIR(const sp<ABuffer> &buffer);
- void addTMMBR(const sp<ABuffer> &buffer);
+ void addTMMBR(const sp<ABuffer> &buffer, int32_t targetBitrate);
+ int addNACK(const sp<ABuffer> &buffer);
+ void setSeqNumToNACK(uint16_t seqNum, uint16_t mask, uint16_t nowJitterHeadSeqNum);
uint32_t getSelfID();
void setSelfID(const uint32_t selfID);
- void setMinMaxBitrate(int32_t min, int32_t max);
- void setBitrateData(int32_t bitrate, int64_t time);
- void setTargetBitrate();
-
- bool isNeedToReport();
- bool isNeedToDowngrade();
+ void setJbTime(const uint32_t jbTimeMs);
+ void setPeriodicFIR(bool enable);
+ void notifyPktInfo(int32_t bitrate, int64_t time);
+ // FIR needs to be sent by missing packet or broken video image.
+ void onIssueFIRByAssembler();
void noticeAbandonBuffer(int cnt=1);
int32_t mFirstSeqNumber;
- int32_t mFirstRtpTime;
+ uint32_t mFirstRtpTime;
int64_t mFirstSysTime;
int32_t mClockRate;
+ uint32_t mJbTimeMs;
+ int32_t mFirstSsrc;
+ int32_t mHighestNackNumber;
+
private:
uint32_t mID;
@@ -71,21 +78,33 @@
uint32_t mBaseSeqNumber;
int32_t mNumBuffersReceived;
int32_t mPrevNumBuffersReceived;
+ uint32_t mPrevExpectedForRR;
+ int32_t mPrevNumBuffersReceivedForRR;
List<sp<ABuffer> > mQueue;
sp<ARTPAssembler> mAssembler;
+ typedef struct infoNACK {
+ uint16_t seqNum;
+ uint16_t mask;
+ uint16_t nowJitterHeadSeqNum;
+ bool needToNACK;
+ } infoNACK;
+
+ Mutex mMapLock;
+ std::map<uint16_t, infoNACK> mNACKMap;
+ int getSeqNumToNACK(List<int>& list, int size);
+
uint64_t mLastNTPTime;
int64_t mLastNTPTimeUpdateUs;
bool mIssueFIRRequests;
+ bool mIssueFIRByAssembler;
int64_t mLastFIRRequestUs;
uint8_t mNextFIRSeqNo;
sp<AMessage> mNotify;
- QualManager mQualManager;
-
bool queuePacket(const sp<ABuffer> &buffer);
DISALLOW_EVIL_CONSTRUCTORS(ARTPSource);
diff --git a/media/libstagefright/rtsp/ARTPWriter.cpp b/media/libstagefright/rtsp/ARTPWriter.cpp
index 70d34de..76afb04 100644
--- a/media/libstagefright/rtsp/ARTPWriter.cpp
+++ b/media/libstagefright/rtsp/ARTPWriter.cpp
@@ -42,21 +42,24 @@
#define H264_NALU_PFRAME 0x1
#define H265_NALU_MASK 0x3F
-#define H265_NALU_VPS 0x40
-#define H265_NALU_SPS 0x42
-#define H265_NALU_PPS 0x44
+#define H265_NALU_VPS 0x20
+#define H265_NALU_SPS 0x21
+#define H265_NALU_PPS 0x22
+#define LINK_HEADER_SIZE 14
+#define IP_HEADER_SIZE 20
#define UDP_HEADER_SIZE 8
+#define TCPIP_HEADER_SIZE (LINK_HEADER_SIZE + IP_HEADER_SIZE + UDP_HEADER_SIZE)
#define RTP_HEADER_SIZE 12
-#define RTP_HEADER_EXT_SIZE 1
+#define RTP_HEADER_EXT_SIZE 8
#define RTP_FU_HEADER_SIZE 2
-#define RTP_PAYLOAD_ROOM_SIZE 140
+#define RTP_PAYLOAD_ROOM_SIZE 100 // ROOM size for IPv6 header, ESP and etc.
namespace android {
// static const size_t kMaxPacketSize = 65507; // maximum payload in UDP over IP
-static const size_t kMaxPacketSize = 1500;
+static const size_t kMaxPacketSize = 1280;
static char kCNAME[255] = "someone@somewhere";
static int UniformRand(int limit) {
@@ -67,7 +70,8 @@
: mFlags(0),
mFd(dup(fd)),
mLooper(new ALooper),
- mReflector(new AHandlerReflector<ARTPWriter>(this)) {
+ mReflector(new AHandlerReflector<ARTPWriter>(this)),
+ mTrafficRec(new TrafficRecorder<uint32_t, size_t>(128)) {
CHECK_GE(fd, 0);
mIsIPv6 = false;
@@ -117,7 +121,8 @@
: mFlags(0),
mFd(dup(fd)),
mLooper(new ALooper),
- mReflector(new AHandlerReflector<ARTPWriter>(this)) {
+ mReflector(new AHandlerReflector<ARTPWriter>(this)),
+ mTrafficRec(new TrafficRecorder<uint32_t, size_t>(128)) {
CHECK_GE(fd, 0);
mIsIPv6 = false;
@@ -126,6 +131,7 @@
mLooper->start();
makeSocketPairAndBind(localIp, localPort, remoteIp , remotePort);
+ mVPSBuf = NULL;
mSPSBuf = NULL;
mPPSBuf = NULL;
@@ -147,6 +153,11 @@
}
ARTPWriter::~ARTPWriter() {
+ if (mVPSBuf != NULL) {
+ mVPSBuf->release();
+ mVPSBuf = NULL;
+ }
+
if (mSPSBuf != NULL) {
mSPSBuf->release();
mSPSBuf = NULL;
@@ -277,12 +288,9 @@
return OK;
}
-// return size of SPS if there is more NAL unit found following to SPS.
-static uint32_t StripStartcode(MediaBufferBase *buffer) {
- uint32_t nalSize = 0;
-
+static void StripStartcode(MediaBufferBase *buffer) {
if (buffer->range_length() < 4) {
- return 0;
+ return;
}
const uint8_t *ptr =
@@ -292,55 +300,129 @@
buffer->set_range(
buffer->range_offset() + 4, buffer->range_length() - 4);
}
-
- ptr = (const uint8_t *)buffer->data() + buffer->range_offset();
-
- if (buffer->range_length() > 0 && (*ptr & H264_NALU_MASK) == H264_NALU_SPS) {
- for (uint32_t i = 1; i + 4 <= buffer->range_length(); i++) {
-
- if (!memcmp(ptr + i, "\x00\x00\x00\x01", 4)) {
- // Now, we found one more NAL unit in the media buffer.
- // Mostly, it will be a PPS.
- nalSize = i;
- ALOGV("SPS found. size=%d", nalSize);
- }
- }
- }
-
- return nalSize;
}
-static void SpsPpsParser(MediaBufferBase *mediaBuffer,
- MediaBufferBase **spsBuffer, MediaBufferBase **ppsBuffer, uint32_t spsSize) {
+static const uint8_t SPCSize = 4; // Start Prefix Code Size
+static const uint8_t startPrefixCode[SPCSize] = {0, 0, 0, 1};
+static const uint8_t spcKMPidx[SPCSize] = {0, 0, 2, 0};
+static void SpsPpsParser(MediaBufferBase *buffer,
+ MediaBufferBase **spsBuffer, MediaBufferBase **ppsBuffer) {
- if (mediaBuffer == NULL || mediaBuffer->range_length() < 4)
- return;
+ while (buffer->range_length() > 0) {
+ const uint8_t *NALPtr = (const uint8_t *)buffer->data() + buffer->range_offset();
- if ((*spsBuffer) != NULL) {
- (*spsBuffer)->release();
- (*spsBuffer) = NULL;
+ MediaBufferBase **targetPtr = NULL;
+ if ((*NALPtr & H264_NALU_MASK) == H264_NALU_SPS) {
+ targetPtr = spsBuffer;
+ } else if ((*NALPtr & H264_NALU_MASK) == H264_NALU_PPS) {
+ targetPtr = ppsBuffer;
+ } else {
+ return;
+ }
+ ALOGV("SPS(7) or PPS(8) found. Type %d", *NALPtr & H264_NALU_MASK);
+
+ uint32_t bufferSize = buffer->range_length();
+ MediaBufferBase *&target = *targetPtr;
+ uint32_t i = 0, j = 0;
+ bool isBoundFound = false;
+ for (i = 0; i < bufferSize; i++) {
+ while (j > 0 && NALPtr[i] != startPrefixCode[j]) {
+ j = spcKMPidx[j - 1];
+ }
+ if (NALPtr[i] == startPrefixCode[j]) {
+ j++;
+ if (j == SPCSize) {
+ isBoundFound = true;
+ break;
+ }
+ }
+ }
+
+ uint32_t targetSize;
+ if (target != NULL) {
+ target->release();
+ }
+ // note that targetSize is never 0 as the first byte is never part
+ // of a start prefix
+ if (isBoundFound) {
+ targetSize = i - SPCSize + 1;
+ target = MediaBufferBase::Create(targetSize);
+ memcpy(target->data(),
+ (const uint8_t *)buffer->data() + buffer->range_offset(),
+ targetSize);
+ buffer->set_range(buffer->range_offset() + targetSize + SPCSize,
+ buffer->range_length() - targetSize - SPCSize);
+ } else {
+ targetSize = bufferSize;
+ target = MediaBufferBase::Create(targetSize);
+ memcpy(target->data(),
+ (const uint8_t *)buffer->data() + buffer->range_offset(),
+ targetSize);
+ buffer->set_range(buffer->range_offset() + bufferSize, 0);
+ return;
+ }
}
+}
- if ((*ppsBuffer) != NULL) {
- (*ppsBuffer)->release();
- (*ppsBuffer) = NULL;
- }
+static void VpsSpsPpsParser(MediaBufferBase *buffer,
+ MediaBufferBase **vpsBuffer, MediaBufferBase **spsBuffer, MediaBufferBase **ppsBuffer) {
- // we got sps/pps but startcode of sps is striped.
- (*spsBuffer) = MediaBufferBase::Create(spsSize);
- memcpy((*spsBuffer)->data(),
- (const uint8_t *)mediaBuffer->data() + mediaBuffer->range_offset(),
- spsSize);
+ while (buffer->range_length() > 0) {
+ const uint8_t *NALPtr = (const uint8_t *)buffer->data() + buffer->range_offset();
+ uint8_t nalType = ((*NALPtr) >> 1) & H265_NALU_MASK;
- int32_t ppsSize = mediaBuffer->range_length() - spsSize - 4 /*startcode*/;
- if (ppsSize > 0) {
- (*ppsBuffer) = MediaBufferBase::Create(ppsSize);
- ALOGV("PPS found. size=%d", (int)ppsSize);
- mediaBuffer->set_range(mediaBuffer->range_offset() + spsSize + 4 /*startcode*/,
- mediaBuffer->range_length() - spsSize - 4 /*startcode*/);
- memcpy((*ppsBuffer)->data(),
- (const uint8_t *)mediaBuffer->data() + mediaBuffer->range_offset(),
- ppsSize);
+ MediaBufferBase **targetPtr = NULL;
+ if (nalType == H265_NALU_VPS) {
+ targetPtr = vpsBuffer;
+ } else if (nalType == H265_NALU_SPS) {
+ targetPtr = spsBuffer;
+ } else if (nalType == H265_NALU_PPS) {
+ targetPtr = ppsBuffer;
+ } else {
+ return;
+ }
+ ALOGV("VPS(32) SPS(33) or PPS(34) found. Type %d", nalType);
+
+ uint32_t bufferSize = buffer->range_length();
+ MediaBufferBase *&target = *targetPtr;
+ uint32_t i = 0, j = 0;
+ bool isBoundFound = false;
+ for (i = 0; i < bufferSize; i++) {
+ while (j > 0 && NALPtr[i] != startPrefixCode[j]) {
+ j = spcKMPidx[j - 1];
+ }
+ if (NALPtr[i] == startPrefixCode[j]) {
+ j++;
+ if (j == SPCSize) {
+ isBoundFound = true;
+ break;
+ }
+ }
+ }
+
+ if (target != NULL) {
+ target->release();
+ }
+ uint32_t targetSize;
+ // note that targetSize is never 0 as the first byte is never part
+ // of a start prefix
+ if (isBoundFound) {
+ targetSize = i - SPCSize + 1;
+ target = MediaBufferBase::Create(j);
+ memcpy(target->data(),
+ (const uint8_t *)buffer->data() + buffer->range_offset(),
+ j);
+ buffer->set_range(buffer->range_offset() + targetSize + SPCSize,
+ buffer->range_length() - targetSize - SPCSize);
+ } else {
+ targetSize = bufferSize;
+ target = MediaBufferBase::Create(targetSize);
+ memcpy(target->data(),
+ (const uint8_t *)buffer->data() + buffer->range_offset(),
+ targetSize);
+ buffer->set_range(buffer->range_offset() + bufferSize, 0);
+ return;
+ }
}
}
@@ -451,15 +533,17 @@
ALOGV("read buffer of size %zu", mediaBuf->range_length());
if (mMode == H264) {
- uint32_t spsSize = 0;
- if ((spsSize = StripStartcode(mediaBuf)) > 0) {
- SpsPpsParser(mediaBuf, &mSPSBuf, &mPPSBuf, spsSize);
- } else {
+ StripStartcode(mediaBuf);
+ SpsPpsParser(mediaBuf, &mSPSBuf, &mPPSBuf);
+ if (mediaBuf->range_length() > 0) {
sendAVCData(mediaBuf);
}
} else if (mMode == H265) {
StripStartcode(mediaBuf);
- sendHEVCData(mediaBuf);
+ VpsSpsPpsParser(mediaBuf, &mVPSBuf, &mSPSBuf, &mPPSBuf);
+ if (mediaBuf->range_length() > 0) {
+ sendHEVCData(mediaBuf);
+ }
} else if (mMode == H263) {
sendH263Data(mediaBuf);
} else if (mMode == AMR_NB || mMode == AMR_WB) {
@@ -504,11 +588,20 @@
remAddr = (struct sockaddr *)&mRTPAddr;
}
+ // Unseal code if moderator is needed (prevent overflow of instant bandwidth)
+ // Set limit bits per period through the moderator.
+ // ex) 6KByte/10ms = 48KBit/10ms = 4.8MBit/s instant limit
+ // ModerateInstantTraffic(10, 6 * 1024);
+
ssize_t n = sendto(isRTCP ? mRTCPSocket : mRTPSocket,
buffer->data(), buffer->size(), 0, remAddr, sizeSockSt);
if (n != (ssize_t)buffer->size()) {
ALOGW("packets can not be sent. ret=%d, buf=%d", (int)n, (int)buffer->size());
+ } else {
+ // Record current traffic & Print bits while last 1sec (1000ms)
+ mTrafficRec->writeBytes(buffer->size());
+ mTrafficRec->printAccuBitsForLastPeriod(1000, 1000);
}
#if LOG_TO_FILES
@@ -807,12 +900,13 @@
}
void ARTPWriter::sendSPSPPSIfIFrame(MediaBufferBase *mediaBuf, int64_t timeUs) {
+ CHECK(mediaBuf->range_length() > 0);
const uint8_t *mediaData =
(const uint8_t *)mediaBuf->data() + mediaBuf->range_offset();
- if (mediaBuf->range_length() == 0
- || (mediaData[0] & H264_NALU_MASK) != H264_NALU_IFRAME)
+ if ((mediaData[0] & H264_NALU_MASK) != H264_NALU_IFRAME) {
return;
+ }
if (mSPSBuf != NULL) {
mSPSBuf->meta_data().setInt64(kKeyTime, timeUs);
@@ -827,6 +921,35 @@
}
}
+void ARTPWriter::sendVPSSPSPPSIfIFrame(MediaBufferBase *mediaBuf, int64_t timeUs) {
+ CHECK(mediaBuf->range_length() > 0);
+ const uint8_t *mediaData =
+ (const uint8_t *)mediaBuf->data() + mediaBuf->range_offset();
+
+ int nalType = ((mediaData[0] >> 1) & H265_NALU_MASK);
+ if (!(nalType >= 16 && nalType <= 21) /*H265_NALU_IFRAME*/) {
+ return;
+ }
+
+ if (mVPSBuf != NULL) {
+ mVPSBuf->meta_data().setInt64(kKeyTime, timeUs);
+ mVPSBuf->meta_data().setInt32(kKeyVps, 1);
+ sendHEVCData(mVPSBuf);
+ }
+
+ if (mSPSBuf != NULL) {
+ mSPSBuf->meta_data().setInt64(kKeyTime, timeUs);
+ mSPSBuf->meta_data().setInt32(kKeySps, 1);
+ sendHEVCData(mSPSBuf);
+ }
+
+ if (mPPSBuf != NULL) {
+ mPPSBuf->meta_data().setInt64(kKeyTime, timeUs);
+ mPPSBuf->meta_data().setInt32(kKeyPps, 1);
+ sendHEVCData(mPPSBuf);
+ }
+}
+
void ARTPWriter::sendHEVCData(MediaBufferBase *mediaBuf) {
// 12 bytes RTP header + 2 bytes for the FU-indicator and FU-header.
CHECK_GE(kMaxPacketSize, 12u + 2u);
@@ -834,21 +957,33 @@
int64_t timeUs;
CHECK(mediaBuf->meta_data().findInt64(kKeyTime, &timeUs));
- sendSPSPPSIfIFrame(mediaBuf, timeUs);
+ sendVPSSPSPPSIfIFrame(mediaBuf, timeUs);
uint32_t rtpTime = mRTPTimeBase + (timeUs * 9 / 100ll);
+ CHECK(mediaBuf->range_length() > 0);
const uint8_t *mediaData =
(const uint8_t *)mediaBuf->data() + mediaBuf->range_offset();
+ int32_t isNonVCL = 0;
+ if (mediaBuf->meta_data().findInt32(kKeyVps, &isNonVCL) ||
+ mediaBuf->meta_data().findInt32(kKeySps, &isNonVCL) ||
+ mediaBuf->meta_data().findInt32(kKeyPps, &isNonVCL)) {
+ isNonVCL = 1;
+ }
+
sp<ABuffer> buffer = new ABuffer(kMaxPacketSize);
- if (mediaBuf->range_length() + UDP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_PAYLOAD_ROOM_SIZE
- <= buffer->capacity()) {
+ if (mediaBuf->range_length() + TCPIP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_HEADER_EXT_SIZE
+ + RTP_PAYLOAD_ROOM_SIZE <= buffer->capacity()) {
// The data fits into a single packet
uint8_t *data = buffer->data();
data[0] = 0x80;
- data[1] = (1 << 7) | mPayloadType; // M-bit
+ if (isNonVCL) {
+ data[1] = mPayloadType; // Marker bit should not be set in case of Non-VCL
+ } else {
+ data[1] = (1 << 7) | mPayloadType; // M-bit
+ }
data[2] = (mSeqNo >> 8) & 0xff;
data[3] = mSeqNo & 0xff;
data[4] = rtpTime >> 24;
@@ -881,11 +1016,11 @@
while (offset < mediaBuf->range_length()) {
size_t size = mediaBuf->range_length() - offset;
bool lastPacket = true;
- if (size + UDP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_FU_HEADER_SIZE +
- RTP_PAYLOAD_ROOM_SIZE > buffer->capacity()) {
+ if (size + TCPIP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_HEADER_EXT_SIZE +
+ RTP_FU_HEADER_SIZE + RTP_PAYLOAD_ROOM_SIZE > buffer->capacity()) {
lastPacket = false;
- size = buffer->capacity() - UDP_HEADER_SIZE - RTP_HEADER_SIZE -
- RTP_FU_HEADER_SIZE - RTP_PAYLOAD_ROOM_SIZE;
+ size = buffer->capacity() - TCPIP_HEADER_SIZE - RTP_HEADER_SIZE -
+ RTP_HEADER_EXT_SIZE - RTP_FU_HEADER_SIZE - RTP_PAYLOAD_ROOM_SIZE;
}
uint8_t *data = buffer->data();
@@ -963,6 +1098,7 @@
uint32_t rtpTime = mRTPTimeBase + (timeUs * 9 / 100LL);
+ CHECK(mediaBuf->range_length() > 0);
const uint8_t *mediaData =
(const uint8_t *)mediaBuf->data() + mediaBuf->range_offset();
@@ -973,9 +1109,10 @@
isSpsPps = true;
}
+ mTrafficRec->updateClock(ALooper::GetNowUs() / 1000);
sp<ABuffer> buffer = new ABuffer(kMaxPacketSize);
- if (mediaBuf->range_length() + UDP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_PAYLOAD_ROOM_SIZE
- <= buffer->capacity()) {
+ if (mediaBuf->range_length() + TCPIP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_HEADER_EXT_SIZE
+ + RTP_PAYLOAD_ROOM_SIZE <= buffer->capacity()) {
// The data fits into a single packet
uint8_t *data = buffer->data();
data[0] = 0x80;
@@ -1051,11 +1188,11 @@
while (offset < mediaBuf->range_length()) {
size_t size = mediaBuf->range_length() - offset;
bool lastPacket = true;
- if (size + UDP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_FU_HEADER_SIZE +
- RTP_PAYLOAD_ROOM_SIZE > buffer->capacity()) {
+ if (size + TCPIP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_HEADER_EXT_SIZE +
+ RTP_FU_HEADER_SIZE + RTP_PAYLOAD_ROOM_SIZE > buffer->capacity()) {
lastPacket = false;
- size = buffer->capacity() - UDP_HEADER_SIZE - RTP_HEADER_SIZE -
- RTP_FU_HEADER_SIZE - RTP_PAYLOAD_ROOM_SIZE;
+ size = buffer->capacity() - TCPIP_HEADER_SIZE - RTP_HEADER_SIZE -
+ RTP_HEADER_EXT_SIZE - RTP_FU_HEADER_SIZE - RTP_PAYLOAD_ROOM_SIZE;
}
uint8_t *data = buffer->data();
@@ -1408,4 +1545,15 @@
}
}
+// TODO : Develop more advanced moderator based on AS & TMMBR value
+void ARTPWriter::ModerateInstantTraffic(uint32_t samplePeriod, uint32_t limitBytes) {
+ unsigned int bytes = mTrafficRec->readBytesForLastPeriod(samplePeriod);
+ if (bytes > limitBytes) {
+ ALOGI("Nuclear moderator. #seq = %d \t\t %d bits / 10ms",
+ mSeqNo, bytes * 8);
+ usleep(4000);
+ mTrafficRec->updateClock(ALooper::GetNowUs() / 1000);
+ }
+}
+
} // namespace android
diff --git a/media/libstagefright/rtsp/ARTPWriter.h b/media/libstagefright/rtsp/ARTPWriter.h
index f7e2204..6f25a66 100644
--- a/media/libstagefright/rtsp/ARTPWriter.h
+++ b/media/libstagefright/rtsp/ARTPWriter.h
@@ -28,6 +28,7 @@
#include <sys/socket.h>
#include <android/multinetwork.h>
+#include "TrafficRecorder.h"
#define LOG_TO_FILES 0
@@ -102,6 +103,7 @@
AString mSeqParamSet;
AString mPicParamSet;
+ MediaBufferBase *mVPSBuf;
MediaBufferBase *mSPSBuf;
MediaBufferBase *mPPSBuf;
@@ -116,6 +118,7 @@
uint32_t mOpponentID;
uint32_t mBitrate;
+ sp<TrafficRecorder<uint32_t, size_t> > mTrafficRec;
int32_t mNumSRsSent;
int32_t mRTPCVOExtMap;
@@ -143,6 +146,7 @@
void dumpSessionDesc();
void sendBye();
+ void sendVPSSPSPPSIfIFrame(MediaBufferBase *mediaBuf, int64_t timeUs);
void sendSPSPPSIfIFrame(MediaBufferBase *mediaBuf, int64_t timeUs);
void sendHEVCData(MediaBufferBase *mediaBuf);
void sendAVCData(MediaBufferBase *mediaBuf);
@@ -152,6 +156,7 @@
void send(const sp<ABuffer> &buffer, bool isRTCP);
void makeSocketPairAndBind(String8& localIp, int localPort, String8& remoteIp, int remotePort);
+ void ModerateInstantTraffic(uint32_t samplePeriod, uint32_t limitBytes);
DISALLOW_EVIL_CONSTRUCTORS(ARTPWriter);
};
diff --git a/media/libstagefright/rtsp/Android.bp b/media/libstagefright/rtsp/Android.bp
index 6179142..f990ecf 100644
--- a/media/libstagefright/rtsp/Android.bp
+++ b/media/libstagefright/rtsp/Android.bp
@@ -18,7 +18,6 @@
"ARTSPConnection.cpp",
"ASessionDescription.cpp",
"SDPLoader.cpp",
- "QualManager.cpp",
],
shared_libs: [
diff --git a/media/libstagefright/rtsp/TrafficRecorder.h b/media/libstagefright/rtsp/TrafficRecorder.h
new file mode 100644
index 0000000..f8e7c03
--- /dev/null
+++ b/media/libstagefright/rtsp/TrafficRecorder.h
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef A_TRAFFIC_RECORDER_H_
+
+#define A_TRAFFIC_RECORDER_H_
+
+#include <android-base/logging.h>
+#include <utils/RefBase.h>
+
+namespace android {
+
+// Circular array to save recent amount of bytes
+template <class Time, class Bytes>
+class TrafficRecorder : public RefBase {
+private:
+ size_t mSize;
+ size_t mSizeMask;
+ Time *mTimeArray = NULL;
+ Bytes *mBytesArray = NULL;
+ size_t mHeadIdx = 0;
+ size_t mTailIdx = 0;
+
+ Time mClock = 0;
+ Time mLastTimeOfPrint = 0;
+ Bytes mAccuBytesOfPrint = 0;
+public:
+ TrafficRecorder();
+ TrafficRecorder(size_t size);
+ virtual ~TrafficRecorder();
+
+ void init();
+
+ void updateClock(Time now);
+
+ Bytes readBytesForLastPeriod(Time period);
+ void writeBytes(Bytes bytes);
+
+ void printAccuBitsForLastPeriod(Time period, Time unit);
+};
+
+template <class Time, class Bytes>
+TrafficRecorder<Time, Bytes>::TrafficRecorder() {
+ TrafficRecorder(128);
+}
+
+template <class Time, class Bytes>
+TrafficRecorder<Time, Bytes>::TrafficRecorder(size_t size) {
+ size_t exp;
+ for (exp = 0; exp < 32; exp++) {
+ if (size <= (1ul << exp)) {
+ break;
+ }
+ }
+ mSize = (1ul << exp); // size = 2^exp
+ mSizeMask = mSize - 1;
+
+ LOG(VERBOSE) << "TrafficRecorder Init size " << mSize;
+ mTimeArray = new Time[mSize];
+ mBytesArray = new Bytes[mSize];
+
+ init();
+}
+
+template <class Time, class Bytes>
+TrafficRecorder<Time, Bytes>::~TrafficRecorder() {
+ delete[] mTimeArray;
+ delete[] mBytesArray;
+}
+
+template <class Time, class Bytes>
+void TrafficRecorder<Time, Bytes>::init() {
+ mHeadIdx = 0;
+ mTailIdx = 0;
+ mTimeArray[0] = 0;
+ mBytesArray[0] = 0;
+}
+
+template <class Time, class Bytes>
+void TrafficRecorder<Time, Bytes>::updateClock(Time now) {
+ mClock = now;
+}
+
+template <class Time, class Bytes>
+Bytes TrafficRecorder<Time, Bytes>::readBytesForLastPeriod(Time period) {
+ Bytes bytes = 0;
+
+ size_t i = mTailIdx;
+ while (i != mHeadIdx) {
+ LOG(VERBOSE) << "READ " << i << " time " << mTimeArray[i] << " \t EndOfPeriod " << mClock - period;
+ if (mTimeArray[i] < mClock - period) {
+ break;
+ }
+ bytes += mBytesArray[i];
+ i = (i + mSize - 1) & mSizeMask;
+ }
+ mHeadIdx = i;
+ return bytes;
+}
+
+template <class Time, class Bytes>
+void TrafficRecorder<Time, Bytes>::writeBytes(Bytes bytes) {
+ size_t writeIdx;
+ if (mClock == mTimeArray[mTailIdx]) {
+ writeIdx = mTailIdx;
+ mBytesArray[writeIdx] += bytes;
+ } else {
+ writeIdx = (mTailIdx + 1) % mSize;
+ mTimeArray[writeIdx] = mClock;
+ mBytesArray[writeIdx] = bytes;
+ }
+
+ LOG(VERBOSE) << "WRITE " << writeIdx << " time " << mClock;
+ if (writeIdx == mHeadIdx) {
+ LOG(WARNING) << "Traffic recorder size exceeded at " << mHeadIdx;
+ mHeadIdx = (mHeadIdx + 1) & mSizeMask;
+ }
+
+ mTailIdx = writeIdx;
+ mAccuBytesOfPrint += bytes;
+}
+
+template <class Time, class Bytes>
+void TrafficRecorder<Time, Bytes>::printAccuBitsForLastPeriod(Time period, Time unit) {
+ Time duration = mClock - mLastTimeOfPrint;
+ float numOfUnit = (float)duration / unit;
+ if (duration > period) {
+ ALOGD("Actual Tx period %.0f ms \t %.0f Bits/Unit",
+ numOfUnit * 1000.f, mAccuBytesOfPrint * 8.f / numOfUnit);
+ mLastTimeOfPrint = mClock;
+ mAccuBytesOfPrint = 0;
+ init();
+ }
+}
+
+} // namespace android
+
+#endif // A_TRAFFIC_RECORDER_H_
diff --git a/services/audioflinger/AudioFlinger.cpp b/services/audioflinger/AudioFlinger.cpp
index 34bdac5..fe45221 100644
--- a/services/audioflinger/AudioFlinger.cpp
+++ b/services/audioflinger/AudioFlinger.cpp
@@ -855,7 +855,8 @@
input.notificationsPerBuffer, input.speed,
input.sharedBuffer, sessionId, &output.flags,
callingPid, input.clientInfo.clientTid, clientUid,
- &lStatus, portId, input.audioTrackCallback);
+ &lStatus, portId, input.audioTrackCallback,
+ input.opPackageName);
LOG_ALWAYS_FATAL_IF((lStatus == NO_ERROR) && (track == 0));
// we don't abort yet if lStatus != NO_ERROR; there is still work to be done regardless
@@ -2071,8 +2072,8 @@
Mutex::Autolock _l(mLock);
RecordThread *thread = checkRecordThread_l(output.inputId);
if (thread == NULL) {
- ALOGE("createRecord() checkRecordThread_l failed, input handle %d", output.inputId);
- lStatus = BAD_VALUE;
+ ALOGW("createRecord() checkRecordThread_l failed, input handle %d", output.inputId);
+ lStatus = FAILED_TRANSACTION;
goto Exit;
}
diff --git a/services/audioflinger/DeviceEffectManager.cpp b/services/audioflinger/DeviceEffectManager.cpp
index dfbefd9..cecd52b 100644
--- a/services/audioflinger/DeviceEffectManager.cpp
+++ b/services/audioflinger/DeviceEffectManager.cpp
@@ -117,10 +117,19 @@
status_t AudioFlinger::DeviceEffectManager::checkEffectCompatibility(
const effect_descriptor_t *desc) {
+ sp<EffectsFactoryHalInterface> effectsFactory = mAudioFlinger.getEffectsFactory();
+ if (effectsFactory == nullptr) {
+ return BAD_VALUE;
+ }
- if ((desc->flags & EFFECT_FLAG_TYPE_MASK) != EFFECT_FLAG_TYPE_PRE_PROC
- && (desc->flags & EFFECT_FLAG_TYPE_MASK) != EFFECT_FLAG_TYPE_POST_PROC) {
- ALOGW("%s() non pre/post processing device effect %s", __func__, desc->name);
+ static const float sMinDeviceEffectHalVersion = 6.0;
+ float halVersion = effectsFactory->getHalVersion();
+
+ if (((desc->flags & EFFECT_FLAG_TYPE_MASK) != EFFECT_FLAG_TYPE_PRE_PROC
+ && (desc->flags & EFFECT_FLAG_TYPE_MASK) != EFFECT_FLAG_TYPE_POST_PROC)
+ || halVersion < sMinDeviceEffectHalVersion) {
+ ALOGW("%s() non pre/post processing device effect %s or incompatible API version %f",
+ __func__, desc->name, halVersion);
return BAD_VALUE;
}
diff --git a/services/audioflinger/PlaybackTracks.h b/services/audioflinger/PlaybackTracks.h
index d05c8b8..a4b8650 100644
--- a/services/audioflinger/PlaybackTracks.h
+++ b/services/audioflinger/PlaybackTracks.h
@@ -26,10 +26,11 @@
bool hasOpPlayAudio() const;
static sp<OpPlayAudioMonitor> createIfNeeded(
- uid_t uid, const audio_attributes_t& attr, int id, audio_stream_type_t streamType);
+ uid_t uid, const audio_attributes_t& attr, int id, audio_stream_type_t streamType,
+ const std::string& opPackageName);
private:
- OpPlayAudioMonitor(uid_t uid, audio_usage_t usage, int id);
+ OpPlayAudioMonitor(uid_t uid, audio_usage_t usage, int id, const String16& opPackageName);
void onFirstRef() override;
static void getPackagesForUid(uid_t uid, Vector<String16>& packages);
@@ -49,10 +50,10 @@
void checkPlayAudioForUsage();
std::atomic_bool mHasOpPlayAudio;
- Vector<String16> mPackages;
const uid_t mUid;
const int32_t mUsage; // on purpose not audio_usage_t because always checked in appOps as int32_t
const int mId; // for logging purposes only
+ const String16 mOpPackageName;
};
// playback track
@@ -77,7 +78,8 @@
audio_port_handle_t portId = AUDIO_PORT_HANDLE_NONE,
/** default behaviour is to start when there are as many frames
* ready as possible (aka. Buffer is full). */
- size_t frameCountToBeReady = SIZE_MAX);
+ size_t frameCountToBeReady = SIZE_MAX,
+ const std::string opPackageName = "");
virtual ~Track();
virtual status_t initCheck() const;
diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp
index 6302fc4..2a6aeac 100644
--- a/services/audioflinger/Threads.cpp
+++ b/services/audioflinger/Threads.cpp
@@ -1946,7 +1946,7 @@
// here instead of constructor of PlaybackThread so that the onFirstRef
// callback would not be made on an incompletely constructed object.
if (mOutput->stream->setEventCallback(this) != OK) {
- ALOGE("Failed to add event callback");
+ ALOGD("Failed to add event callback");
}
}
run(mThreadName, ANDROID_PRIORITY_URGENT_AUDIO);
@@ -2080,7 +2080,8 @@
uid_t uid,
status_t *status,
audio_port_handle_t portId,
- const sp<media::IAudioTrackCallback>& callback)
+ const sp<media::IAudioTrackCallback>& callback,
+ const std::string& opPackageName)
{
size_t frameCount = *pFrameCount;
size_t notificationFrameCount = *pNotificationFrameCount;
@@ -2371,7 +2372,8 @@
track = new Track(this, client, streamType, attr, sampleRate, format,
channelMask, frameCount,
nullptr /* buffer */, (size_t)0 /* bufferSize */, sharedBuffer,
- sessionId, creatorPid, uid, trackFlags, TrackBase::TYPE_DEFAULT, portId);
+ sessionId, creatorPid, uid, trackFlags, TrackBase::TYPE_DEFAULT, portId,
+ SIZE_MAX /*frameCountToBeReady*/, opPackageName);
lStatus = track != 0 ? track->initCheck() : (status_t) NO_MEMORY;
if (lStatus != NO_ERROR) {
@@ -7911,7 +7913,8 @@
AutoMutex lock(mLock);
if (recordTrack->isInvalid()) {
recordTrack->clearSyncStartEvent();
- return INVALID_OPERATION;
+ ALOGW("%s track %d: invalidated before startInput", __func__, recordTrack->portId());
+ return DEAD_OBJECT;
}
if (mActiveTracks.indexOf(recordTrack) >= 0) {
if (recordTrack->mState == TrackBase::PAUSING) {
@@ -7941,7 +7944,8 @@
recordTrack->mState = TrackBase::STARTING_2;
// STARTING_2 forces destroy to call stopInput.
}
- return INVALID_OPERATION;
+ ALOGW("%s track %d: invalidated after startInput", __func__, recordTrack->portId());
+ return DEAD_OBJECT;
}
if (recordTrack->mState != TrackBase::STARTING_1) {
ALOGW("%s(%d): unsynchronized mState:%d change",
diff --git a/services/audioflinger/Threads.h b/services/audioflinger/Threads.h
index ac41e82..d59b702 100644
--- a/services/audioflinger/Threads.h
+++ b/services/audioflinger/Threads.h
@@ -884,7 +884,8 @@
uid_t uid,
status_t *status /*non-NULL*/,
audio_port_handle_t portId,
- const sp<media::IAudioTrackCallback>& callback);
+ const sp<media::IAudioTrackCallback>& callback,
+ const std::string& opPackageName);
AudioStreamOut* getOutput() const;
AudioStreamOut* clearOutput();
diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp
index c92bce5..fbfe077 100644
--- a/services/audioflinger/Tracks.cpp
+++ b/services/audioflinger/Tracks.cpp
@@ -386,11 +386,12 @@
// static
sp<AudioFlinger::PlaybackThread::OpPlayAudioMonitor>
AudioFlinger::PlaybackThread::OpPlayAudioMonitor::createIfNeeded(
- uid_t uid, const audio_attributes_t& attr, int id, audio_stream_type_t streamType)
+ uid_t uid, const audio_attributes_t& attr, int id, audio_stream_type_t streamType,
+ const std::string& opPackageName)
{
+ Vector <String16> packages;
+ getPackagesForUid(uid, packages);
if (isServiceUid(uid)) {
- Vector <String16> packages;
- getPackagesForUid(uid, packages);
if (packages.isEmpty()) {
ALOGD("OpPlayAudio: not muting track:%d usage:%d for service UID %d",
id,
@@ -410,12 +411,32 @@
id, attr.flags);
return nullptr;
}
- return new OpPlayAudioMonitor(uid, attr.usage, id);
+
+ String16 opPackageNameStr(opPackageName.c_str());
+ if (opPackageName.empty()) {
+ // If no package name is provided by the client, use the first associated with the uid
+ if (!packages.isEmpty()) {
+ opPackageNameStr = packages[0];
+ }
+ } else {
+ // If the provided package name is invalid, we force app ops denial by clearing the package
+ // name passed to OpPlayAudioMonitor
+ if (std::find_if(packages.begin(), packages.end(),
+ [&opPackageNameStr](const auto& package) {
+ return opPackageNameStr == package; }) == packages.end()) {
+ ALOGW("The package name(%s) provided does not correspond to the uid %d, "
+ "force muting the track", opPackageName.c_str(), uid);
+ // Set package name as an empty string so that hasOpPlayAudio will always return false.
+ opPackageNameStr = String16("");
+ }
+ }
+ return new OpPlayAudioMonitor(uid, attr.usage, id, opPackageNameStr);
}
AudioFlinger::PlaybackThread::OpPlayAudioMonitor::OpPlayAudioMonitor(
- uid_t uid, audio_usage_t usage, int id)
- : mHasOpPlayAudio(true), mUid(uid), mUsage((int32_t) usage), mId(id)
+ uid_t uid, audio_usage_t usage, int id, const String16& opPackageName)
+ : mHasOpPlayAudio(true), mUid(uid), mUsage((int32_t) usage), mId(id),
+ mOpPackageName(opPackageName)
{
}
@@ -429,11 +450,10 @@
void AudioFlinger::PlaybackThread::OpPlayAudioMonitor::onFirstRef()
{
- getPackagesForUid(mUid, mPackages);
checkPlayAudioForUsage();
- if (!mPackages.isEmpty()) {
+ if (mOpPackageName.size() != 0) {
mOpCallback = new PlayAudioOpCallback(this);
- mAppOpsManager.startWatchingMode(AppOpsManager::OP_PLAY_AUDIO, mPackages[0], mOpCallback);
+ mAppOpsManager.startWatchingMode(AppOpsManager::OP_PLAY_AUDIO, mOpPackageName, mOpCallback);
}
}
@@ -446,18 +466,11 @@
// - not called from PlayAudioOpCallback because the callback is not installed in this case
void AudioFlinger::PlaybackThread::OpPlayAudioMonitor::checkPlayAudioForUsage()
{
- if (mPackages.isEmpty()) {
+ if (mOpPackageName.size() == 0) {
mHasOpPlayAudio.store(false);
} else {
- bool hasIt = true;
- for (const String16& packageName : mPackages) {
- const int32_t mode = mAppOpsManager.checkAudioOpNoThrow(AppOpsManager::OP_PLAY_AUDIO,
- mUsage, mUid, packageName);
- if (mode != AppOpsManager::MODE_ALLOWED) {
- hasIt = false;
- break;
- }
- }
+ bool hasIt = mAppOpsManager.checkAudioOpNoThrow(AppOpsManager::OP_PLAY_AUDIO,
+ mUsage, mUid, mOpPackageName) == AppOpsManager::MODE_ALLOWED;
ALOGD("OpPlayAudio: track:%d usage:%d %smuted", mId, mUsage, hasIt ? "not " : "");
mHasOpPlayAudio.store(hasIt);
}
@@ -511,7 +524,8 @@
audio_output_flags_t flags,
track_type type,
audio_port_handle_t portId,
- size_t frameCountToBeReady)
+ size_t frameCountToBeReady,
+ const std::string opPackageName)
: TrackBase(thread, client, attr, sampleRate, format, channelMask, frameCount,
// TODO: Using unsecurePointer() has some associated security pitfalls
// (see declaration for details).
@@ -534,7 +548,8 @@
mPresentationCompleteFrames(0),
mFrameMap(16 /* sink-frame-to-track-frame map memory */),
mVolumeHandler(new media::VolumeHandler(sampleRate)),
- mOpPlayAudioMonitor(OpPlayAudioMonitor::createIfNeeded(uid, attr, id(), streamType)),
+ mOpPlayAudioMonitor(OpPlayAudioMonitor::createIfNeeded(
+ uid, attr, id(), streamType, opPackageName)),
// mSinkTimestamp
mFrameCountToBeReady(frameCountToBeReady),
mFastIndex(-1),
@@ -601,7 +616,7 @@
// external vibration is always created for all tracks attached to haptic playback thread.
mAudioVibrationController = new AudioVibrationController(this);
mExternalVibration = new os::ExternalVibration(
- mUid, "" /* pkg */, mAttr, mAudioVibrationController);
+ mUid, opPackageName, mAttr, mAudioVibrationController);
}
// Once this item is logged by the server, the client can add properties.
@@ -2229,7 +2244,8 @@
RecordThread *recordThread = (RecordThread *)thread.get();
return recordThread->start(this, event, triggerSession);
} else {
- return BAD_VALUE;
+ ALOGW("%s track %d: thread was destroyed", __func__, portId());
+ return DEAD_OBJECT;
}
}
diff --git a/services/audiopolicy/engine/common/Android.bp b/services/audiopolicy/engine/common/Android.bp
old mode 100755
new mode 100644
diff --git a/services/audiopolicy/engine/common/include/EngineBase.h b/services/audiopolicy/engine/common/include/EngineBase.h
old mode 100755
new mode 100644
diff --git a/services/audiopolicy/engine/common/include/LastRemovableMediaDevices.h b/services/audiopolicy/engine/common/include/LastRemovableMediaDevices.h
old mode 100755
new mode 100644
diff --git a/services/audiopolicy/engine/common/src/LastRemovableMediaDevices.cpp b/services/audiopolicy/engine/common/src/LastRemovableMediaDevices.cpp
old mode 100755
new mode 100644
diff --git a/services/audiopolicy/enginedefault/src/Engine.cpp b/services/audiopolicy/enginedefault/src/Engine.cpp
old mode 100755
new mode 100644
diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
index cb3725e..830b74c 100644
--- a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
+++ b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
@@ -1789,7 +1789,8 @@
checkAndSetVolume(curves, client->volumeSource(),
curves.getVolumeIndex(outputDesc->devices().types()),
outputDesc,
- outputDesc->devices().types());
+ outputDesc->devices().types(), 0 /*delay*/,
+ outputDesc->useHwGain() /*force*/);
// update the outputs if starting an output with a stream that can affect notification
// routing
@@ -2280,7 +2281,7 @@
sp<AudioInputDescriptor> inputDesc = mInputs.getInputForClient(portId);
if (inputDesc == 0) {
ALOGW("%s no input for client %d", __FUNCTION__, portId);
- return BAD_VALUE;
+ return DEAD_OBJECT;
}
audio_io_handle_t input = inputDesc->mIoHandle;
sp<RecordClientDescriptor> client = inputDesc->getClient(portId);
@@ -3184,6 +3185,8 @@
if (mEngine->getPhoneState() == AUDIO_MODE_IN_CALL && hasPrimaryOutput()) {
DeviceVector newDevices = getNewOutputDevices(mPrimaryOutput, true /*fromCache*/);
waitMs = updateCallRouting(newDevices, delayMs);
+ // Only apply special touch sound delay once
+ delayMs = 0;
}
for (size_t i = 0; i < mOutputs.size(); i++) {
sp<SwAudioOutputDescriptor> outputDesc = mOutputs.valueAt(i);
@@ -3193,6 +3196,8 @@
// preventing the force re-routing in case of default dev that distinguishes on address.
// Let's give back to engine full device choice decision however.
waitMs = setOutputDevices(outputDesc, newDevices, !newDevices.isEmpty(), delayMs);
+ // Only apply special touch sound delay once
+ delayMs = 0;
}
if (forceVolumeReeval && !newDevices.isEmpty()) {
applyStreamVolumes(outputDesc, newDevices.types(), waitMs, true);
diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp
index 022d686..e80838b 100644
--- a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp
+++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp
@@ -21,6 +21,7 @@
#include <cutils/properties.h>
#include <utils/CameraThreadState.h>
#include <utils/Log.h>
+#include <utils/SessionConfigurationUtils.h>
#include <utils/Trace.h>
#include <gui/Surface.h>
#include <camera/camera2/CaptureRequest.h>
@@ -492,7 +493,8 @@
return STATUS_ERROR(CameraService::ERROR_DISCONNECTED, "Camera device no longer alive");
}
- res = checkOperatingMode(operatingMode, mDevice->info(), mCameraIdStr);
+ res = SessionConfigurationUtils::checkOperatingMode(operatingMode, mDevice->info(),
+ mCameraIdStr);
if (!res.isOk()) {
return res;
}
@@ -550,247 +552,6 @@
return res;
}
-binder::Status CameraDeviceClient::checkSurfaceType(size_t numBufferProducers,
- bool deferredConsumer, int surfaceType) {
- if (numBufferProducers > MAX_SURFACES_PER_STREAM) {
- ALOGE("%s: GraphicBufferProducer count %zu for stream exceeds limit of %d",
- __FUNCTION__, numBufferProducers, MAX_SURFACES_PER_STREAM);
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, "Surface count is too high");
- } else if ((numBufferProducers == 0) && (!deferredConsumer)) {
- ALOGE("%s: Number of consumers cannot be smaller than 1", __FUNCTION__);
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, "No valid consumers.");
- }
-
- bool validSurfaceType = ((surfaceType == OutputConfiguration::SURFACE_TYPE_SURFACE_VIEW) ||
- (surfaceType == OutputConfiguration::SURFACE_TYPE_SURFACE_TEXTURE));
-
- if (deferredConsumer && !validSurfaceType) {
- ALOGE("%s: Target surface has invalid surfaceType = %d.", __FUNCTION__, surfaceType);
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, "Target Surface is invalid");
- }
-
- return binder::Status::ok();
-}
-
-binder::Status CameraDeviceClient::checkPhysicalCameraId(
- const std::vector<std::string> &physicalCameraIds, const String8 &physicalCameraId,
- const String8 &logicalCameraId) {
- if (physicalCameraId.size() == 0) {
- return binder::Status::ok();
- }
- if (std::find(physicalCameraIds.begin(), physicalCameraIds.end(),
- physicalCameraId.string()) == physicalCameraIds.end()) {
- String8 msg = String8::format("Camera %s: Camera doesn't support physicalCameraId %s.",
- logicalCameraId.string(), physicalCameraId.string());
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
- }
- return binder::Status::ok();
-}
-
-binder::Status CameraDeviceClient::checkOperatingMode(int operatingMode,
- const CameraMetadata &staticInfo, const String8 &cameraId) {
- if (operatingMode < 0) {
- String8 msg = String8::format(
- "Camera %s: Invalid operating mode %d requested", cameraId.string(), operatingMode);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT,
- msg.string());
- }
-
- bool isConstrainedHighSpeed = (operatingMode == ICameraDeviceUser::CONSTRAINED_HIGH_SPEED_MODE);
- if (isConstrainedHighSpeed) {
- camera_metadata_ro_entry_t entry = staticInfo.find(ANDROID_REQUEST_AVAILABLE_CAPABILITIES);
- bool isConstrainedHighSpeedSupported = false;
- for(size_t i = 0; i < entry.count; ++i) {
- uint8_t capability = entry.data.u8[i];
- if (capability == ANDROID_REQUEST_AVAILABLE_CAPABILITIES_CONSTRAINED_HIGH_SPEED_VIDEO) {
- isConstrainedHighSpeedSupported = true;
- break;
- }
- }
- if (!isConstrainedHighSpeedSupported) {
- String8 msg = String8::format(
- "Camera %s: Try to create a constrained high speed configuration on a device"
- " that doesn't support it.", cameraId.string());
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT,
- msg.string());
- }
- }
-
- return binder::Status::ok();
-}
-
-void CameraDeviceClient::mapStreamInfo(const OutputStreamInfo &streamInfo,
- camera3_stream_rotation_t rotation, String8 physicalId,
- hardware::camera::device::V3_4::Stream *stream /*out*/) {
- if (stream == nullptr) {
- return;
- }
-
- stream->v3_2.streamType = hardware::camera::device::V3_2::StreamType::OUTPUT;
- stream->v3_2.width = streamInfo.width;
- stream->v3_2.height = streamInfo.height;
- stream->v3_2.format = Camera3Device::mapToPixelFormat(streamInfo.format);
- auto u = streamInfo.consumerUsage;
- camera3::Camera3OutputStream::applyZSLUsageQuirk(streamInfo.format, &u);
- stream->v3_2.usage = Camera3Device::mapToConsumerUsage(u);
- stream->v3_2.dataSpace = Camera3Device::mapToHidlDataspace(streamInfo.dataSpace);
- stream->v3_2.rotation = Camera3Device::mapToStreamRotation(rotation);
- stream->v3_2.id = -1; // Invalid stream id
- stream->physicalCameraId = std::string(physicalId.string());
- stream->bufferSize = 0;
-}
-
-binder::Status
-CameraDeviceClient::convertToHALStreamCombination(const SessionConfiguration& sessionConfiguration,
- const String8 &logicalCameraId, const CameraMetadata &deviceInfo,
- metadataGetter getMetadata, const std::vector<std::string> &physicalCameraIds,
- hardware::camera::device::V3_4::StreamConfiguration &streamConfiguration,
- bool *unsupported) {
- auto operatingMode = sessionConfiguration.getOperatingMode();
- binder::Status res = checkOperatingMode(operatingMode, deviceInfo, logicalCameraId);
- if (!res.isOk()) {
- return res;
- }
-
- if (unsupported == nullptr) {
- String8 msg("unsupported nullptr");
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
- }
- *unsupported = false;
- auto ret = Camera3Device::mapToStreamConfigurationMode(
- static_cast<camera3_stream_configuration_mode_t> (operatingMode),
- /*out*/ &streamConfiguration.operationMode);
- if (ret != OK) {
- String8 msg = String8::format(
- "Camera %s: Failed mapping operating mode %d requested: %s (%d)",
- logicalCameraId.string(), operatingMode, strerror(-ret), ret);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT,
- msg.string());
- }
-
- bool isInputValid = (sessionConfiguration.getInputWidth() > 0) &&
- (sessionConfiguration.getInputHeight() > 0) &&
- (sessionConfiguration.getInputFormat() > 0);
- auto outputConfigs = sessionConfiguration.getOutputConfigurations();
- size_t streamCount = outputConfigs.size();
- streamCount = isInputValid ? streamCount + 1 : streamCount;
- streamConfiguration.streams.resize(streamCount);
- size_t streamIdx = 0;
- if (isInputValid) {
- streamConfiguration.streams[streamIdx++] = {{/*streamId*/0,
- hardware::camera::device::V3_2::StreamType::INPUT,
- static_cast<uint32_t> (sessionConfiguration.getInputWidth()),
- static_cast<uint32_t> (sessionConfiguration.getInputHeight()),
- Camera3Device::mapToPixelFormat(sessionConfiguration.getInputFormat()),
- /*usage*/ 0, HAL_DATASPACE_UNKNOWN,
- hardware::camera::device::V3_2::StreamRotation::ROTATION_0},
- /*physicalId*/ nullptr, /*bufferSize*/0};
- }
-
- for (const auto &it : outputConfigs) {
- const std::vector<sp<IGraphicBufferProducer>>& bufferProducers =
- it.getGraphicBufferProducers();
- bool deferredConsumer = it.isDeferred();
- String8 physicalCameraId = String8(it.getPhysicalCameraId());
- size_t numBufferProducers = bufferProducers.size();
- bool isStreamInfoValid = false;
- OutputStreamInfo streamInfo;
-
- res = checkSurfaceType(numBufferProducers, deferredConsumer, it.getSurfaceType());
- if (!res.isOk()) {
- return res;
- }
- res = checkPhysicalCameraId(physicalCameraIds, physicalCameraId,
- logicalCameraId);
- if (!res.isOk()) {
- return res;
- }
-
- if (deferredConsumer) {
- streamInfo.width = it.getWidth();
- streamInfo.height = it.getHeight();
- streamInfo.format = HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
- streamInfo.dataSpace = android_dataspace_t::HAL_DATASPACE_UNKNOWN;
- auto surfaceType = it.getSurfaceType();
- streamInfo.consumerUsage = GraphicBuffer::USAGE_HW_TEXTURE;
- if (surfaceType == OutputConfiguration::SURFACE_TYPE_SURFACE_VIEW) {
- streamInfo.consumerUsage |= GraphicBuffer::USAGE_HW_COMPOSER;
- }
- mapStreamInfo(streamInfo, CAMERA3_STREAM_ROTATION_0, physicalCameraId,
- &streamConfiguration.streams[streamIdx++]);
- isStreamInfoValid = true;
-
- if (numBufferProducers == 0) {
- continue;
- }
- }
-
- for (auto& bufferProducer : bufferProducers) {
- sp<Surface> surface;
- const CameraMetadata &physicalDeviceInfo = getMetadata(physicalCameraId);
- res = createSurfaceFromGbp(streamInfo, isStreamInfoValid, surface, bufferProducer,
- logicalCameraId,
- physicalCameraId.size() > 0 ? physicalDeviceInfo : deviceInfo );
-
- if (!res.isOk())
- return res;
-
- if (!isStreamInfoValid) {
- bool isDepthCompositeStream =
- camera3::DepthCompositeStream::isDepthCompositeStream(surface);
- bool isHeicCompositeStream =
- camera3::HeicCompositeStream::isHeicCompositeStream(surface);
- if (isDepthCompositeStream || isHeicCompositeStream) {
- // We need to take in to account that composite streams can have
- // additional internal camera streams.
- std::vector<OutputStreamInfo> compositeStreams;
- if (isDepthCompositeStream) {
- ret = camera3::DepthCompositeStream::getCompositeStreamInfo(streamInfo,
- deviceInfo, &compositeStreams);
- } else {
- ret = camera3::HeicCompositeStream::getCompositeStreamInfo(streamInfo,
- deviceInfo, &compositeStreams);
- }
- if (ret != OK) {
- String8 msg = String8::format(
- "Camera %s: Failed adding composite streams: %s (%d)",
- logicalCameraId.string(), strerror(-ret), ret);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
- }
-
- if (compositeStreams.size() == 0) {
- // No internal streams means composite stream not
- // supported.
- *unsupported = true;
- return binder::Status::ok();
- } else if (compositeStreams.size() > 1) {
- streamCount += compositeStreams.size() - 1;
- streamConfiguration.streams.resize(streamCount);
- }
-
- for (const auto& compositeStream : compositeStreams) {
- mapStreamInfo(compositeStream,
- static_cast<camera3_stream_rotation_t> (it.getRotation()),
- physicalCameraId, &streamConfiguration.streams[streamIdx++]);
- }
- } else {
- mapStreamInfo(streamInfo,
- static_cast<camera3_stream_rotation_t> (it.getRotation()),
- physicalCameraId, &streamConfiguration.streams[streamIdx++]);
- }
- isStreamInfoValid = true;
- }
- }
- }
- return binder::Status::ok();
-}
-
binder::Status CameraDeviceClient::isSessionConfigurationSupported(
const SessionConfiguration& sessionConfiguration, bool *status /*out*/) {
ATRACE_CALL();
@@ -806,7 +567,8 @@
}
auto operatingMode = sessionConfiguration.getOperatingMode();
- res = checkOperatingMode(operatingMode, mDevice->info(), mCameraIdStr);
+ res = SessionConfigurationUtils::checkOperatingMode(operatingMode, mDevice->info(),
+ mCameraIdStr);
if (!res.isOk()) {
return res;
}
@@ -821,8 +583,9 @@
metadataGetter getMetadata = [this](const String8 &id) {return mDevice->infoPhysical(id);};
std::vector<std::string> physicalCameraIds;
mProviderManager->isLogicalCamera(mCameraIdStr.string(), &physicalCameraIds);
- res = convertToHALStreamCombination(sessionConfiguration, mCameraIdStr,
- mDevice->info(), getMetadata, physicalCameraIds, streamConfiguration, &earlyExit);
+ res = SessionConfigurationUtils::convertToHALStreamCombination(sessionConfiguration,
+ mCameraIdStr, mDevice->info(), getMetadata, physicalCameraIds, streamConfiguration,
+ &earlyExit);
if (!res.isOk()) {
return res;
}
@@ -970,7 +733,7 @@
String8 physicalCameraId = String8(outputConfiguration.getPhysicalCameraId());
bool deferredConsumerOnly = deferredConsumer && numBufferProducers == 0;
- res = checkSurfaceType(numBufferProducers, deferredConsumer,
+ res = SessionConfigurationUtils::checkSurfaceType(numBufferProducers, deferredConsumer,
outputConfiguration.getSurfaceType());
if (!res.isOk()) {
return res;
@@ -981,7 +744,8 @@
}
std::vector<std::string> physicalCameraIds;
mProviderManager->isLogicalCamera(mCameraIdStr.string(), &physicalCameraIds);
- res = checkPhysicalCameraId(physicalCameraIds, physicalCameraId, mCameraIdStr);
+ res = SessionConfigurationUtils::checkPhysicalCameraId(physicalCameraIds, physicalCameraId,
+ mCameraIdStr);
if (!res.isOk()) {
return res;
}
@@ -1009,8 +773,8 @@
}
sp<Surface> surface;
- res = createSurfaceFromGbp(streamInfo, isStreamInfoValid, surface, bufferProducer,
- mCameraIdStr, mDevice->infoPhysical(physicalCameraId));
+ res = SessionConfigurationUtils::createSurfaceFromGbp(streamInfo, isStreamInfoValid,
+ surface, bufferProducer, mCameraIdStr, mDevice->infoPhysical(physicalCameraId));
if (!res.isOk())
return res;
@@ -1313,8 +1077,9 @@
for (size_t i = 0; i < newOutputsMap.size(); i++) {
OutputStreamInfo outInfo;
sp<Surface> surface;
- res = createSurfaceFromGbp(outInfo, /*isStreamInfoValid*/ false, surface,
- newOutputsMap.valueAt(i), mCameraIdStr, mDevice->infoPhysical(physicalCameraId));
+ res = SessionConfigurationUtils::createSurfaceFromGbp(outInfo, /*isStreamInfoValid*/ false,
+ surface, newOutputsMap.valueAt(i), mCameraIdStr,
+ mDevice->infoPhysical(physicalCameraId));
if (!res.isOk())
return res;
@@ -1364,226 +1129,6 @@
return res;
}
-bool CameraDeviceClient::isPublicFormat(int32_t format)
-{
- switch(format) {
- case HAL_PIXEL_FORMAT_RGBA_8888:
- case HAL_PIXEL_FORMAT_RGBX_8888:
- case HAL_PIXEL_FORMAT_RGB_888:
- case HAL_PIXEL_FORMAT_RGB_565:
- case HAL_PIXEL_FORMAT_BGRA_8888:
- case HAL_PIXEL_FORMAT_YV12:
- case HAL_PIXEL_FORMAT_Y8:
- case HAL_PIXEL_FORMAT_Y16:
- case HAL_PIXEL_FORMAT_RAW16:
- case HAL_PIXEL_FORMAT_RAW10:
- case HAL_PIXEL_FORMAT_RAW12:
- case HAL_PIXEL_FORMAT_RAW_OPAQUE:
- case HAL_PIXEL_FORMAT_BLOB:
- case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
- case HAL_PIXEL_FORMAT_YCbCr_420_888:
- case HAL_PIXEL_FORMAT_YCbCr_422_SP:
- case HAL_PIXEL_FORMAT_YCrCb_420_SP:
- case HAL_PIXEL_FORMAT_YCbCr_422_I:
- return true;
- default:
- return false;
- }
-}
-
-binder::Status CameraDeviceClient::createSurfaceFromGbp(
- OutputStreamInfo& streamInfo, bool isStreamInfoValid,
- sp<Surface>& surface, const sp<IGraphicBufferProducer>& gbp,
- const String8 &cameraId, const CameraMetadata &physicalCameraMetadata) {
-
- // bufferProducer must be non-null
- if (gbp == nullptr) {
- String8 msg = String8::format("Camera %s: Surface is NULL", cameraId.string());
- ALOGW("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
- }
- // HACK b/10949105
- // Query consumer usage bits to set async operation mode for
- // GLConsumer using controlledByApp parameter.
- bool useAsync = false;
- uint64_t consumerUsage = 0;
- status_t err;
- if ((err = gbp->getConsumerUsage(&consumerUsage)) != OK) {
- String8 msg = String8::format("Camera %s: Failed to query Surface consumer usage: %s (%d)",
- cameraId.string(), strerror(-err), err);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string());
- }
- if (consumerUsage & GraphicBuffer::USAGE_HW_TEXTURE) {
- ALOGW("%s: Camera %s with consumer usage flag: %" PRIu64 ": Forcing asynchronous mode for stream",
- __FUNCTION__, cameraId.string(), consumerUsage);
- useAsync = true;
- }
-
- uint64_t disallowedFlags = GraphicBuffer::USAGE_HW_VIDEO_ENCODER |
- GRALLOC_USAGE_RENDERSCRIPT;
- uint64_t allowedFlags = GraphicBuffer::USAGE_SW_READ_MASK |
- GraphicBuffer::USAGE_HW_TEXTURE |
- GraphicBuffer::USAGE_HW_COMPOSER;
- bool flexibleConsumer = (consumerUsage & disallowedFlags) == 0 &&
- (consumerUsage & allowedFlags) != 0;
-
- surface = new Surface(gbp, useAsync);
- ANativeWindow *anw = surface.get();
-
- int width, height, format;
- android_dataspace dataSpace;
- if ((err = anw->query(anw, NATIVE_WINDOW_WIDTH, &width)) != OK) {
- String8 msg = String8::format("Camera %s: Failed to query Surface width: %s (%d)",
- cameraId.string(), strerror(-err), err);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string());
- }
- if ((err = anw->query(anw, NATIVE_WINDOW_HEIGHT, &height)) != OK) {
- String8 msg = String8::format("Camera %s: Failed to query Surface height: %s (%d)",
- cameraId.string(), strerror(-err), err);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string());
- }
- if ((err = anw->query(anw, NATIVE_WINDOW_FORMAT, &format)) != OK) {
- String8 msg = String8::format("Camera %s: Failed to query Surface format: %s (%d)",
- cameraId.string(), strerror(-err), err);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string());
- }
- if ((err = anw->query(anw, NATIVE_WINDOW_DEFAULT_DATASPACE,
- reinterpret_cast<int*>(&dataSpace))) != OK) {
- String8 msg = String8::format("Camera %s: Failed to query Surface dataspace: %s (%d)",
- cameraId.string(), strerror(-err), err);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string());
- }
-
- // FIXME: remove this override since the default format should be
- // IMPLEMENTATION_DEFINED. b/9487482 & b/35317944
- if ((format >= HAL_PIXEL_FORMAT_RGBA_8888 && format <= HAL_PIXEL_FORMAT_BGRA_8888) &&
- ((consumerUsage & GRALLOC_USAGE_HW_MASK) &&
- ((consumerUsage & GRALLOC_USAGE_SW_READ_MASK) == 0))) {
- ALOGW("%s: Camera %s: Overriding format %#x to IMPLEMENTATION_DEFINED",
- __FUNCTION__, cameraId.string(), format);
- format = HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
- }
- // Round dimensions to the nearest dimensions available for this format
- if (flexibleConsumer && isPublicFormat(format) &&
- !CameraDeviceClient::roundBufferDimensionNearest(width, height,
- format, dataSpace, physicalCameraMetadata, /*out*/&width, /*out*/&height)) {
- String8 msg = String8::format("Camera %s: No supported stream configurations with "
- "format %#x defined, failed to create output stream",
- cameraId.string(), format);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
- }
-
- if (!isStreamInfoValid) {
- streamInfo.width = width;
- streamInfo.height = height;
- streamInfo.format = format;
- streamInfo.dataSpace = dataSpace;
- streamInfo.consumerUsage = consumerUsage;
- return binder::Status::ok();
- }
- if (width != streamInfo.width) {
- String8 msg = String8::format("Camera %s:Surface width doesn't match: %d vs %d",
- cameraId.string(), width, streamInfo.width);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
- }
- if (height != streamInfo.height) {
- String8 msg = String8::format("Camera %s:Surface height doesn't match: %d vs %d",
- cameraId.string(), height, streamInfo.height);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
- }
- if (format != streamInfo.format) {
- String8 msg = String8::format("Camera %s:Surface format doesn't match: %d vs %d",
- cameraId.string(), format, streamInfo.format);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
- }
- if (format != HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) {
- if (dataSpace != streamInfo.dataSpace) {
- String8 msg = String8::format("Camera %s:Surface dataSpace doesn't match: %d vs %d",
- cameraId.string(), dataSpace, streamInfo.dataSpace);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
- }
- //At the native side, there isn't a way to check whether 2 surfaces come from the same
- //surface class type. Use usage flag to approximate the comparison.
- if (consumerUsage != streamInfo.consumerUsage) {
- String8 msg = String8::format(
- "Camera %s:Surface usage flag doesn't match %" PRIu64 " vs %" PRIu64 "",
- cameraId.string(), consumerUsage, streamInfo.consumerUsage);
- ALOGE("%s: %s", __FUNCTION__, msg.string());
- return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
- }
- }
- return binder::Status::ok();
-}
-
-bool CameraDeviceClient::roundBufferDimensionNearest(int32_t width, int32_t height,
- int32_t format, android_dataspace dataSpace, const CameraMetadata& info,
- /*out*/int32_t* outWidth, /*out*/int32_t* outHeight) {
-
- camera_metadata_ro_entry streamConfigs =
- (dataSpace == HAL_DATASPACE_DEPTH) ?
- info.find(ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS) :
- (dataSpace == static_cast<android_dataspace>(HAL_DATASPACE_HEIF)) ?
- info.find(ANDROID_HEIC_AVAILABLE_HEIC_STREAM_CONFIGURATIONS) :
- info.find(ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS);
-
- int32_t bestWidth = -1;
- int32_t bestHeight = -1;
-
- // Iterate through listed stream configurations and find the one with the smallest euclidean
- // distance from the given dimensions for the given format.
- for (size_t i = 0; i < streamConfigs.count; i += 4) {
- int32_t fmt = streamConfigs.data.i32[i];
- int32_t w = streamConfigs.data.i32[i + 1];
- int32_t h = streamConfigs.data.i32[i + 2];
-
- // Ignore input/output type for now
- if (fmt == format) {
- if (w == width && h == height) {
- bestWidth = width;
- bestHeight = height;
- break;
- } else if (w <= ROUNDING_WIDTH_CAP && (bestWidth == -1 ||
- CameraDeviceClient::euclidDistSquare(w, h, width, height) <
- CameraDeviceClient::euclidDistSquare(bestWidth, bestHeight, width, height))) {
- bestWidth = w;
- bestHeight = h;
- }
- }
- }
-
- if (bestWidth == -1) {
- // Return false if no configurations for this format were listed
- return false;
- }
-
- // Set the outputs to the closet width/height
- if (outWidth != NULL) {
- *outWidth = bestWidth;
- }
- if (outHeight != NULL) {
- *outHeight = bestHeight;
- }
-
- // Return true if at least one configuration for this format was listed
- return true;
-}
-
-int64_t CameraDeviceClient::euclidDistSquare(int32_t x0, int32_t y0, int32_t x1, int32_t y1) {
- int64_t d0 = x0 - x1;
- int64_t d1 = y0 - y1;
- return d0 * d0 + d1 * d1;
-}
-
// Create a request object from a template.
binder::Status CameraDeviceClient::createDefaultRequest(int templateId,
/*out*/
@@ -1896,8 +1441,9 @@
}
sp<Surface> surface;
- res = createSurfaceFromGbp(mStreamInfoMap[streamId], true /*isStreamInfoValid*/,
- surface, bufferProducer, mCameraIdStr, mDevice->infoPhysical(physicalId));
+ res = SessionConfigurationUtils::createSurfaceFromGbp(mStreamInfoMap[streamId],
+ true /*isStreamInfoValid*/, surface, bufferProducer, mCameraIdStr,
+ mDevice->infoPhysical(physicalId));
if (!res.isOk())
return res;
diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.h b/services/camera/libcameraservice/api2/CameraDeviceClient.h
index 5cd16ee..2807aee 100644
--- a/services/camera/libcameraservice/api2/CameraDeviceClient.h
+++ b/services/camera/libcameraservice/api2/CameraDeviceClient.h
@@ -204,16 +204,6 @@
virtual void notifyRequestQueueEmpty();
virtual void notifyRepeatingRequestError(long lastFrameNumber);
- // utility function to convert AIDL SessionConfiguration to HIDL
- // streamConfiguration. Also checks for validity of SessionConfiguration and
- // returns a non-ok binder::Status if the passed in session configuration
- // isn't valid.
- static binder::Status
- convertToHALStreamCombination(const SessionConfiguration& sessionConfiguration,
- const String8 &cameraId, const CameraMetadata &deviceInfo,
- metadataGetter getMetadata, const std::vector<std::string> &physicalCameraIds,
- hardware::camera::device::V3_4::StreamConfiguration &streamConfiguration,
- bool *earlyExit);
/**
* Interface used by independent components of CameraDeviceClient.
*/
@@ -266,18 +256,8 @@
/** Utility members */
binder::Status checkPidStatus(const char* checkLocation);
- static binder::Status checkOperatingMode(int operatingMode, const CameraMetadata &staticInfo,
- const String8 &cameraId);
- static binder::Status checkSurfaceType(size_t numBufferProducers, bool deferredConsumer,
- int surfaceType);
- static void mapStreamInfo(const OutputStreamInfo &streamInfo,
- camera3_stream_rotation_t rotation, String8 physicalId,
- hardware::camera::device::V3_4::Stream *stream /*out*/);
bool enforceRequestPermissions(CameraMetadata& metadata);
- // Find the square of the euclidean distance between two points
- static int64_t euclidDistSquare(int32_t x0, int32_t y0, int32_t x1, int32_t y1);
-
// Create an output stream with surface deferred for future.
binder::Status createDeferredSurfaceStreamLocked(
const hardware::camera2::params::OutputConfiguration &outputConfiguration,
@@ -288,33 +268,11 @@
// cases.
binder::Status setStreamTransformLocked(int streamId);
- // Find the closest dimensions for a given format in available stream configurations with
- // a width <= ROUNDING_WIDTH_CAP
- static const int32_t ROUNDING_WIDTH_CAP = 1920;
- static bool roundBufferDimensionNearest(int32_t width, int32_t height, int32_t format,
- android_dataspace dataSpace, const CameraMetadata& info,
- /*out*/int32_t* outWidth, /*out*/int32_t* outHeight);
-
- //check if format is not custom format
- static bool isPublicFormat(int32_t format);
-
- // Create a Surface from an IGraphicBufferProducer. Returns error if
- // IGraphicBufferProducer's property doesn't match with streamInfo
- static binder::Status createSurfaceFromGbp(OutputStreamInfo& streamInfo, bool isStreamInfoValid,
- sp<Surface>& surface, const sp<IGraphicBufferProducer>& gbp, const String8 &cameraId,
- const CameraMetadata &physicalCameraMetadata);
-
-
// Utility method to insert the surface into SurfaceMap
binder::Status insertGbpLocked(const sp<IGraphicBufferProducer>& gbp,
/*out*/SurfaceMap* surfaceMap, /*out*/Vector<int32_t>* streamIds,
/*out*/int32_t* currentStreamId);
- // Check that the physicalCameraId passed in is spported by the camera
- // device.
- static binder::Status checkPhysicalCameraId(const std::vector<std::string> &physicalCameraIds,
- const String8 &physicalCameraId, const String8 &logicalCameraId);
-
// IGraphicsBufferProducer binder -> Stream ID + Surface ID for output streams
KeyedVector<sp<IBinder>, StreamSurfaceId> mStreamMap;
@@ -346,7 +304,6 @@
KeyedVector<sp<IBinder>, sp<CompositeStream>> mCompositeStreamMap;
- static const int32_t MAX_SURFACES_PER_STREAM = 4;
sp<CameraProviderManager> mProviderManager;
};
diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp
index 723d6ec..d27f11f 100644
--- a/services/camera/libcameraservice/device3/Camera3Device.cpp
+++ b/services/camera/libcameraservice/device3/Camera3Device.cpp
@@ -2313,6 +2313,15 @@
newRequest->mRotateAndCropAuto = false;
}
+ auto zoomRatioEntry =
+ newRequest->mSettingsList.begin()->metadata.find(ANDROID_CONTROL_ZOOM_RATIO);
+ if (zoomRatioEntry.count > 0 &&
+ zoomRatioEntry.data.f[0] == 1.0f) {
+ newRequest->mZoomRatioIs1x = true;
+ } else {
+ newRequest->mZoomRatioIs1x = false;
+ }
+
return newRequest;
}
@@ -4433,13 +4442,17 @@
parent->mDistortionMappers.end()) {
continue;
}
- res = parent->mDistortionMappers[it->cameraId].correctCaptureRequest(
- &(it->metadata));
- if (res != OK) {
- SET_ERR("RequestThread: Unable to correct capture requests "
- "for lens distortion for request %d: %s (%d)",
- halRequest->frame_number, strerror(-res), res);
- return INVALID_OPERATION;
+
+ if (!captureRequest->mDistortionCorrectionUpdated) {
+ res = parent->mDistortionMappers[it->cameraId].correctCaptureRequest(
+ &(it->metadata));
+ if (res != OK) {
+ SET_ERR("RequestThread: Unable to correct capture requests "
+ "for lens distortion for request %d: %s (%d)",
+ halRequest->frame_number, strerror(-res), res);
+ return INVALID_OPERATION;
+ }
+ captureRequest->mDistortionCorrectionUpdated = true;
}
}
@@ -4450,21 +4463,24 @@
continue;
}
- camera_metadata_entry_t e = it->metadata.find(ANDROID_CONTROL_ZOOM_RATIO);
- if (e.count > 0 && e.data.f[0] != 1.0f) {
+ if (!captureRequest->mZoomRatioIs1x) {
cameraIdsWithZoom.insert(it->cameraId);
}
- res = parent->mZoomRatioMappers[it->cameraId].updateCaptureRequest(
- &(it->metadata));
- if (res != OK) {
- SET_ERR("RequestThread: Unable to correct capture requests "
- "for zoom ratio for request %d: %s (%d)",
- halRequest->frame_number, strerror(-res), res);
- return INVALID_OPERATION;
+ if (!captureRequest->mZoomRatioUpdated) {
+ res = parent->mZoomRatioMappers[it->cameraId].updateCaptureRequest(
+ &(it->metadata));
+ if (res != OK) {
+ SET_ERR("RequestThread: Unable to correct capture requests "
+ "for zoom ratio for request %d: %s (%d)",
+ halRequest->frame_number, strerror(-res), res);
+ return INVALID_OPERATION;
+ }
+ captureRequest->mZoomRatioUpdated = true;
}
}
- if (captureRequest->mRotateAndCropAuto) {
+ if (captureRequest->mRotateAndCropAuto &&
+ !captureRequest->mRotationAndCropUpdated) {
for (it = captureRequest->mSettingsList.begin();
it != captureRequest->mSettingsList.end(); it++) {
auto mapper = parent->mRotateAndCropMappers.find(it->cameraId);
@@ -4478,6 +4494,7 @@
}
}
}
+ captureRequest->mRotationAndCropUpdated = true;
}
}
}
diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h
index 2aad09a..c579071 100644
--- a/services/camera/libcameraservice/device3/Camera3Device.h
+++ b/services/camera/libcameraservice/device3/Camera3Device.h
@@ -517,6 +517,19 @@
// overriding of ROTATE_AND_CROP value and adjustment of coordinates
// in several other controls in both the request and the result
bool mRotateAndCropAuto;
+ // Whether this capture request has its zoom ratio set to 1.0x before
+ // the framework overrides it for camera HAL consumption.
+ bool mZoomRatioIs1x;
+
+
+ // Whether this capture request's distortion correction update has
+ // been done.
+ bool mDistortionCorrectionUpdated = false;
+ // Whether this capture request's rotation and crop update has been
+ // done.
+ bool mRotationAndCropUpdated = false;
+ // Whether this capture request's zoom ratio update has been done.
+ bool mZoomRatioUpdated = false;
};
typedef List<sp<CaptureRequest> > RequestList;
diff --git a/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp b/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp
index 888671c..ba68a63 100644
--- a/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp
+++ b/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp
@@ -14,20 +14,493 @@
* limitations under the License.
*/
#include "SessionConfigurationUtils.h"
-#include "../api2/CameraDeviceClient.h"
+#include "../api2/DepthCompositeStream.h"
+#include "../api2/HeicCompositeStream.h"
+#include "common/CameraDeviceBase.h"
+#include "../CameraService.h"
+#include "device3/Camera3Device.h"
+#include "device3/Camera3OutputStream.h"
+
+// Convenience methods for constructing binder::Status objects for error returns
+
+#define STATUS_ERROR(errorCode, errorString) \
+ binder::Status::fromServiceSpecificError(errorCode, \
+ String8::format("%s:%d: %s", __FUNCTION__, __LINE__, errorString))
+
+#define STATUS_ERROR_FMT(errorCode, errorString, ...) \
+ binder::Status::fromServiceSpecificError(errorCode, \
+ String8::format("%s:%d: " errorString, __FUNCTION__, __LINE__, \
+ __VA_ARGS__))
+
+using android::camera3::OutputStreamInfo;
+using android::camera3::OutputStreamInfo;
+using android::hardware::camera2::ICameraDeviceUser;
namespace android {
+int64_t SessionConfigurationUtils::euclidDistSquare(int32_t x0, int32_t y0, int32_t x1, int32_t y1) {
+ int64_t d0 = x0 - x1;
+ int64_t d1 = y0 - y1;
+ return d0 * d0 + d1 * d1;
+}
+
+bool SessionConfigurationUtils::roundBufferDimensionNearest(int32_t width, int32_t height,
+ int32_t format, android_dataspace dataSpace, const CameraMetadata& info,
+ /*out*/int32_t* outWidth, /*out*/int32_t* outHeight) {
+
+ camera_metadata_ro_entry streamConfigs =
+ (dataSpace == HAL_DATASPACE_DEPTH) ?
+ info.find(ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS) :
+ (dataSpace == static_cast<android_dataspace>(HAL_DATASPACE_HEIF)) ?
+ info.find(ANDROID_HEIC_AVAILABLE_HEIC_STREAM_CONFIGURATIONS) :
+ info.find(ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS);
+
+ int32_t bestWidth = -1;
+ int32_t bestHeight = -1;
+
+ // Iterate through listed stream configurations and find the one with the smallest euclidean
+ // distance from the given dimensions for the given format.
+ for (size_t i = 0; i < streamConfigs.count; i += 4) {
+ int32_t fmt = streamConfigs.data.i32[i];
+ int32_t w = streamConfigs.data.i32[i + 1];
+ int32_t h = streamConfigs.data.i32[i + 2];
+
+ // Ignore input/output type for now
+ if (fmt == format) {
+ if (w == width && h == height) {
+ bestWidth = width;
+ bestHeight = height;
+ break;
+ } else if (w <= ROUNDING_WIDTH_CAP && (bestWidth == -1 ||
+ SessionConfigurationUtils::euclidDistSquare(w, h, width, height) <
+ SessionConfigurationUtils::euclidDistSquare(bestWidth, bestHeight, width,
+ height))) {
+ bestWidth = w;
+ bestHeight = h;
+ }
+ }
+ }
+
+ if (bestWidth == -1) {
+ // Return false if no configurations for this format were listed
+ return false;
+ }
+
+ // Set the outputs to the closet width/height
+ if (outWidth != NULL) {
+ *outWidth = bestWidth;
+ }
+ if (outHeight != NULL) {
+ *outHeight = bestHeight;
+ }
+
+ // Return true if at least one configuration for this format was listed
+ return true;
+}
+
+bool SessionConfigurationUtils::isPublicFormat(int32_t format)
+{
+ switch(format) {
+ case HAL_PIXEL_FORMAT_RGBA_8888:
+ case HAL_PIXEL_FORMAT_RGBX_8888:
+ case HAL_PIXEL_FORMAT_RGB_888:
+ case HAL_PIXEL_FORMAT_RGB_565:
+ case HAL_PIXEL_FORMAT_BGRA_8888:
+ case HAL_PIXEL_FORMAT_YV12:
+ case HAL_PIXEL_FORMAT_Y8:
+ case HAL_PIXEL_FORMAT_Y16:
+ case HAL_PIXEL_FORMAT_RAW16:
+ case HAL_PIXEL_FORMAT_RAW10:
+ case HAL_PIXEL_FORMAT_RAW12:
+ case HAL_PIXEL_FORMAT_RAW_OPAQUE:
+ case HAL_PIXEL_FORMAT_BLOB:
+ case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
+ case HAL_PIXEL_FORMAT_YCbCr_420_888:
+ case HAL_PIXEL_FORMAT_YCbCr_422_SP:
+ case HAL_PIXEL_FORMAT_YCrCb_420_SP:
+ case HAL_PIXEL_FORMAT_YCbCr_422_I:
+ return true;
+ default:
+ return false;
+ }
+}
+
+binder::Status SessionConfigurationUtils::createSurfaceFromGbp(
+ OutputStreamInfo& streamInfo, bool isStreamInfoValid,
+ sp<Surface>& surface, const sp<IGraphicBufferProducer>& gbp,
+ const String8 &cameraId, const CameraMetadata &physicalCameraMetadata) {
+
+ // bufferProducer must be non-null
+ if (gbp == nullptr) {
+ String8 msg = String8::format("Camera %s: Surface is NULL", cameraId.string());
+ ALOGW("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
+ }
+ // HACK b/10949105
+ // Query consumer usage bits to set async operation mode for
+ // GLConsumer using controlledByApp parameter.
+ bool useAsync = false;
+ uint64_t consumerUsage = 0;
+ status_t err;
+ if ((err = gbp->getConsumerUsage(&consumerUsage)) != OK) {
+ String8 msg = String8::format("Camera %s: Failed to query Surface consumer usage: %s (%d)",
+ cameraId.string(), strerror(-err), err);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string());
+ }
+ if (consumerUsage & GraphicBuffer::USAGE_HW_TEXTURE) {
+ ALOGW("%s: Camera %s with consumer usage flag: %" PRIu64 ": Forcing asynchronous mode for"
+ "stream", __FUNCTION__, cameraId.string(), consumerUsage);
+ useAsync = true;
+ }
+
+ uint64_t disallowedFlags = GraphicBuffer::USAGE_HW_VIDEO_ENCODER |
+ GRALLOC_USAGE_RENDERSCRIPT;
+ uint64_t allowedFlags = GraphicBuffer::USAGE_SW_READ_MASK |
+ GraphicBuffer::USAGE_HW_TEXTURE |
+ GraphicBuffer::USAGE_HW_COMPOSER;
+ bool flexibleConsumer = (consumerUsage & disallowedFlags) == 0 &&
+ (consumerUsage & allowedFlags) != 0;
+
+ surface = new Surface(gbp, useAsync);
+ ANativeWindow *anw = surface.get();
+
+ int width, height, format;
+ android_dataspace dataSpace;
+ if ((err = anw->query(anw, NATIVE_WINDOW_WIDTH, &width)) != OK) {
+ String8 msg = String8::format("Camera %s: Failed to query Surface width: %s (%d)",
+ cameraId.string(), strerror(-err), err);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string());
+ }
+ if ((err = anw->query(anw, NATIVE_WINDOW_HEIGHT, &height)) != OK) {
+ String8 msg = String8::format("Camera %s: Failed to query Surface height: %s (%d)",
+ cameraId.string(), strerror(-err), err);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string());
+ }
+ if ((err = anw->query(anw, NATIVE_WINDOW_FORMAT, &format)) != OK) {
+ String8 msg = String8::format("Camera %s: Failed to query Surface format: %s (%d)",
+ cameraId.string(), strerror(-err), err);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string());
+ }
+ if ((err = anw->query(anw, NATIVE_WINDOW_DEFAULT_DATASPACE,
+ reinterpret_cast<int*>(&dataSpace))) != OK) {
+ String8 msg = String8::format("Camera %s: Failed to query Surface dataspace: %s (%d)",
+ cameraId.string(), strerror(-err), err);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_INVALID_OPERATION, msg.string());
+ }
+
+ // FIXME: remove this override since the default format should be
+ // IMPLEMENTATION_DEFINED. b/9487482 & b/35317944
+ if ((format >= HAL_PIXEL_FORMAT_RGBA_8888 && format <= HAL_PIXEL_FORMAT_BGRA_8888) &&
+ ((consumerUsage & GRALLOC_USAGE_HW_MASK) &&
+ ((consumerUsage & GRALLOC_USAGE_SW_READ_MASK) == 0))) {
+ ALOGW("%s: Camera %s: Overriding format %#x to IMPLEMENTATION_DEFINED",
+ __FUNCTION__, cameraId.string(), format);
+ format = HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
+ }
+ // Round dimensions to the nearest dimensions available for this format
+ if (flexibleConsumer && isPublicFormat(format) &&
+ !SessionConfigurationUtils::roundBufferDimensionNearest(width, height,
+ format, dataSpace, physicalCameraMetadata, /*out*/&width, /*out*/&height)) {
+ String8 msg = String8::format("Camera %s: No supported stream configurations with "
+ "format %#x defined, failed to create output stream",
+ cameraId.string(), format);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
+ }
+
+ if (!isStreamInfoValid) {
+ streamInfo.width = width;
+ streamInfo.height = height;
+ streamInfo.format = format;
+ streamInfo.dataSpace = dataSpace;
+ streamInfo.consumerUsage = consumerUsage;
+ return binder::Status::ok();
+ }
+ if (width != streamInfo.width) {
+ String8 msg = String8::format("Camera %s:Surface width doesn't match: %d vs %d",
+ cameraId.string(), width, streamInfo.width);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
+ }
+ if (height != streamInfo.height) {
+ String8 msg = String8::format("Camera %s:Surface height doesn't match: %d vs %d",
+ cameraId.string(), height, streamInfo.height);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
+ }
+ if (format != streamInfo.format) {
+ String8 msg = String8::format("Camera %s:Surface format doesn't match: %d vs %d",
+ cameraId.string(), format, streamInfo.format);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
+ }
+ if (format != HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) {
+ if (dataSpace != streamInfo.dataSpace) {
+ String8 msg = String8::format("Camera %s:Surface dataSpace doesn't match: %d vs %d",
+ cameraId.string(), dataSpace, streamInfo.dataSpace);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
+ }
+ //At the native side, there isn't a way to check whether 2 surfaces come from the same
+ //surface class type. Use usage flag to approximate the comparison.
+ if (consumerUsage != streamInfo.consumerUsage) {
+ String8 msg = String8::format(
+ "Camera %s:Surface usage flag doesn't match %" PRIu64 " vs %" PRIu64 "",
+ cameraId.string(), consumerUsage, streamInfo.consumerUsage);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
+ }
+ }
+ return binder::Status::ok();
+}
+
+
+void SessionConfigurationUtils::mapStreamInfo(const OutputStreamInfo &streamInfo,
+ camera3_stream_rotation_t rotation, String8 physicalId,
+ hardware::camera::device::V3_4::Stream *stream /*out*/) {
+ if (stream == nullptr) {
+ return;
+ }
+
+ stream->v3_2.streamType = hardware::camera::device::V3_2::StreamType::OUTPUT;
+ stream->v3_2.width = streamInfo.width;
+ stream->v3_2.height = streamInfo.height;
+ stream->v3_2.format = Camera3Device::mapToPixelFormat(streamInfo.format);
+ auto u = streamInfo.consumerUsage;
+ camera3::Camera3OutputStream::applyZSLUsageQuirk(streamInfo.format, &u);
+ stream->v3_2.usage = Camera3Device::mapToConsumerUsage(u);
+ stream->v3_2.dataSpace = Camera3Device::mapToHidlDataspace(streamInfo.dataSpace);
+ stream->v3_2.rotation = Camera3Device::mapToStreamRotation(rotation);
+ stream->v3_2.id = -1; // Invalid stream id
+ stream->physicalCameraId = std::string(physicalId.string());
+ stream->bufferSize = 0;
+}
+
+binder::Status SessionConfigurationUtils::checkPhysicalCameraId(
+ const std::vector<std::string> &physicalCameraIds, const String8 &physicalCameraId,
+ const String8 &logicalCameraId) {
+ if (physicalCameraId.size() == 0) {
+ return binder::Status::ok();
+ }
+ if (std::find(physicalCameraIds.begin(), physicalCameraIds.end(),
+ physicalCameraId.string()) == physicalCameraIds.end()) {
+ String8 msg = String8::format("Camera %s: Camera doesn't support physicalCameraId %s.",
+ logicalCameraId.string(), physicalCameraId.string());
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
+ }
+ return binder::Status::ok();
+}
+
+binder::Status SessionConfigurationUtils::checkSurfaceType(size_t numBufferProducers,
+ bool deferredConsumer, int surfaceType) {
+ if (numBufferProducers > MAX_SURFACES_PER_STREAM) {
+ ALOGE("%s: GraphicBufferProducer count %zu for stream exceeds limit of %d",
+ __FUNCTION__, numBufferProducers, MAX_SURFACES_PER_STREAM);
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, "Surface count is too high");
+ } else if ((numBufferProducers == 0) && (!deferredConsumer)) {
+ ALOGE("%s: Number of consumers cannot be smaller than 1", __FUNCTION__);
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, "No valid consumers.");
+ }
+
+ bool validSurfaceType = ((surfaceType == OutputConfiguration::SURFACE_TYPE_SURFACE_VIEW) ||
+ (surfaceType == OutputConfiguration::SURFACE_TYPE_SURFACE_TEXTURE));
+
+ if (deferredConsumer && !validSurfaceType) {
+ ALOGE("%s: Target surface has invalid surfaceType = %d.", __FUNCTION__, surfaceType);
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, "Target Surface is invalid");
+ }
+
+ return binder::Status::ok();
+}
+
+binder::Status SessionConfigurationUtils::checkOperatingMode(int operatingMode,
+ const CameraMetadata &staticInfo, const String8 &cameraId) {
+ if (operatingMode < 0) {
+ String8 msg = String8::format(
+ "Camera %s: Invalid operating mode %d requested", cameraId.string(), operatingMode);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT,
+ msg.string());
+ }
+
+ bool isConstrainedHighSpeed = (operatingMode == ICameraDeviceUser::CONSTRAINED_HIGH_SPEED_MODE);
+ if (isConstrainedHighSpeed) {
+ camera_metadata_ro_entry_t entry = staticInfo.find(ANDROID_REQUEST_AVAILABLE_CAPABILITIES);
+ bool isConstrainedHighSpeedSupported = false;
+ for(size_t i = 0; i < entry.count; ++i) {
+ uint8_t capability = entry.data.u8[i];
+ if (capability == ANDROID_REQUEST_AVAILABLE_CAPABILITIES_CONSTRAINED_HIGH_SPEED_VIDEO) {
+ isConstrainedHighSpeedSupported = true;
+ break;
+ }
+ }
+ if (!isConstrainedHighSpeedSupported) {
+ String8 msg = String8::format(
+ "Camera %s: Try to create a constrained high speed configuration on a device"
+ " that doesn't support it.", cameraId.string());
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT,
+ msg.string());
+ }
+ }
+
+ return binder::Status::ok();
+}
+
binder::Status
SessionConfigurationUtils::convertToHALStreamCombination(
const SessionConfiguration& sessionConfiguration,
const String8 &logicalCameraId, const CameraMetadata &deviceInfo,
metadataGetter getMetadata, const std::vector<std::string> &physicalCameraIds,
hardware::camera::device::V3_4::StreamConfiguration &streamConfiguration, bool *earlyExit) {
- // TODO: http://b/148329298 Move the other dependencies from
- // CameraDeviceClient into SessionConfigurationUtils.
- return CameraDeviceClient::convertToHALStreamCombination(sessionConfiguration, logicalCameraId,
- deviceInfo, getMetadata, physicalCameraIds, streamConfiguration, earlyExit);
+
+ auto operatingMode = sessionConfiguration.getOperatingMode();
+ binder::Status res = checkOperatingMode(operatingMode, deviceInfo, logicalCameraId);
+ if (!res.isOk()) {
+ return res;
+ }
+
+ if (earlyExit == nullptr) {
+ String8 msg("earlyExit nullptr");
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
+ }
+ *earlyExit = false;
+ auto ret = Camera3Device::mapToStreamConfigurationMode(
+ static_cast<camera3_stream_configuration_mode_t> (operatingMode),
+ /*out*/ &streamConfiguration.operationMode);
+ if (ret != OK) {
+ String8 msg = String8::format(
+ "Camera %s: Failed mapping operating mode %d requested: %s (%d)",
+ logicalCameraId.string(), operatingMode, strerror(-ret), ret);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT,
+ msg.string());
+ }
+
+ bool isInputValid = (sessionConfiguration.getInputWidth() > 0) &&
+ (sessionConfiguration.getInputHeight() > 0) &&
+ (sessionConfiguration.getInputFormat() > 0);
+ auto outputConfigs = sessionConfiguration.getOutputConfigurations();
+ size_t streamCount = outputConfigs.size();
+ streamCount = isInputValid ? streamCount + 1 : streamCount;
+ streamConfiguration.streams.resize(streamCount);
+ size_t streamIdx = 0;
+ if (isInputValid) {
+ streamConfiguration.streams[streamIdx++] = {{/*streamId*/0,
+ hardware::camera::device::V3_2::StreamType::INPUT,
+ static_cast<uint32_t> (sessionConfiguration.getInputWidth()),
+ static_cast<uint32_t> (sessionConfiguration.getInputHeight()),
+ Camera3Device::mapToPixelFormat(sessionConfiguration.getInputFormat()),
+ /*usage*/ 0, HAL_DATASPACE_UNKNOWN,
+ hardware::camera::device::V3_2::StreamRotation::ROTATION_0},
+ /*physicalId*/ nullptr, /*bufferSize*/0};
+ }
+
+ for (const auto &it : outputConfigs) {
+ const std::vector<sp<IGraphicBufferProducer>>& bufferProducers =
+ it.getGraphicBufferProducers();
+ bool deferredConsumer = it.isDeferred();
+ String8 physicalCameraId = String8(it.getPhysicalCameraId());
+ size_t numBufferProducers = bufferProducers.size();
+ bool isStreamInfoValid = false;
+ OutputStreamInfo streamInfo;
+
+ res = checkSurfaceType(numBufferProducers, deferredConsumer, it.getSurfaceType());
+ if (!res.isOk()) {
+ return res;
+ }
+ res = checkPhysicalCameraId(physicalCameraIds, physicalCameraId,
+ logicalCameraId);
+ if (!res.isOk()) {
+ return res;
+ }
+
+ if (deferredConsumer) {
+ streamInfo.width = it.getWidth();
+ streamInfo.height = it.getHeight();
+ streamInfo.format = HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
+ streamInfo.dataSpace = android_dataspace_t::HAL_DATASPACE_UNKNOWN;
+ auto surfaceType = it.getSurfaceType();
+ streamInfo.consumerUsage = GraphicBuffer::USAGE_HW_TEXTURE;
+ if (surfaceType == OutputConfiguration::SURFACE_TYPE_SURFACE_VIEW) {
+ streamInfo.consumerUsage |= GraphicBuffer::USAGE_HW_COMPOSER;
+ }
+ mapStreamInfo(streamInfo, CAMERA3_STREAM_ROTATION_0, physicalCameraId,
+ &streamConfiguration.streams[streamIdx++]);
+ isStreamInfoValid = true;
+
+ if (numBufferProducers == 0) {
+ continue;
+ }
+ }
+
+ for (auto& bufferProducer : bufferProducers) {
+ sp<Surface> surface;
+ const CameraMetadata &physicalDeviceInfo = getMetadata(physicalCameraId);
+ res = createSurfaceFromGbp(streamInfo, isStreamInfoValid, surface, bufferProducer,
+ logicalCameraId,
+ physicalCameraId.size() > 0 ? physicalDeviceInfo : deviceInfo );
+
+ if (!res.isOk())
+ return res;
+
+ if (!isStreamInfoValid) {
+ bool isDepthCompositeStream =
+ camera3::DepthCompositeStream::isDepthCompositeStream(surface);
+ bool isHeicCompositeStream =
+ camera3::HeicCompositeStream::isHeicCompositeStream(surface);
+ if (isDepthCompositeStream || isHeicCompositeStream) {
+ // We need to take in to account that composite streams can have
+ // additional internal camera streams.
+ std::vector<OutputStreamInfo> compositeStreams;
+ if (isDepthCompositeStream) {
+ ret = camera3::DepthCompositeStream::getCompositeStreamInfo(streamInfo,
+ deviceInfo, &compositeStreams);
+ } else {
+ ret = camera3::HeicCompositeStream::getCompositeStreamInfo(streamInfo,
+ deviceInfo, &compositeStreams);
+ }
+ if (ret != OK) {
+ String8 msg = String8::format(
+ "Camera %s: Failed adding composite streams: %s (%d)",
+ logicalCameraId.string(), strerror(-ret), ret);
+ ALOGE("%s: %s", __FUNCTION__, msg.string());
+ return STATUS_ERROR(CameraService::ERROR_ILLEGAL_ARGUMENT, msg.string());
+ }
+
+ if (compositeStreams.size() == 0) {
+ // No internal streams means composite stream not
+ // supported.
+ *earlyExit = true;
+ return binder::Status::ok();
+ } else if (compositeStreams.size() > 1) {
+ streamCount += compositeStreams.size() - 1;
+ streamConfiguration.streams.resize(streamCount);
+ }
+
+ for (const auto& compositeStream : compositeStreams) {
+ mapStreamInfo(compositeStream,
+ static_cast<camera3_stream_rotation_t> (it.getRotation()),
+ physicalCameraId, &streamConfiguration.streams[streamIdx++]);
+ }
+ } else {
+ mapStreamInfo(streamInfo,
+ static_cast<camera3_stream_rotation_t> (it.getRotation()),
+ physicalCameraId, &streamConfiguration.streams[streamIdx++]);
+ }
+ isStreamInfoValid = true;
+ }
+ }
+ }
+ return binder::Status::ok();
+
}
}// namespace android
diff --git a/services/camera/libcameraservice/utils/SessionConfigurationUtils.h b/services/camera/libcameraservice/utils/SessionConfigurationUtils.h
index cfb9f17..6ce2cd7 100644
--- a/services/camera/libcameraservice/utils/SessionConfigurationUtils.h
+++ b/services/camera/libcameraservice/utils/SessionConfigurationUtils.h
@@ -23,6 +23,9 @@
#include <camera/camera2/SubmitInfo.h>
#include <android/hardware/camera/device/3.4/ICameraDeviceSession.h>
+#include <hardware/camera3.h>
+#include <device3/Camera3StreamInterface.h>
+
#include <stdint.h>
namespace android {
@@ -31,6 +34,41 @@
class SessionConfigurationUtils {
public:
+
+ static int64_t euclidDistSquare(int32_t x0, int32_t y0, int32_t x1, int32_t y1);
+
+ // Find the closest dimensions for a given format in available stream configurations with
+ // a width <= ROUNDING_WIDTH_CAP
+ static bool roundBufferDimensionNearest(int32_t width, int32_t height, int32_t format,
+ android_dataspace dataSpace, const CameraMetadata& info,
+ /*out*/int32_t* outWidth, /*out*/int32_t* outHeight);
+
+ //check if format is not custom format
+ static bool isPublicFormat(int32_t format);
+
+ // Create a Surface from an IGraphicBufferProducer. Returns error if
+ // IGraphicBufferProducer's property doesn't match with streamInfo
+ static binder::Status createSurfaceFromGbp(
+ camera3::OutputStreamInfo& streamInfo, bool isStreamInfoValid,
+ sp<Surface>& surface, const sp<IGraphicBufferProducer>& gbp,
+ const String8 &cameraId, const CameraMetadata &physicalCameraMetadata);
+
+ static void mapStreamInfo(const camera3::OutputStreamInfo &streamInfo,
+ camera3_stream_rotation_t rotation, String8 physicalId,
+ hardware::camera::device::V3_4::Stream *stream /*out*/);
+
+ // Check that the physicalCameraId passed in is spported by the camera
+ // device.
+ static binder::Status checkPhysicalCameraId(
+ const std::vector<std::string> &physicalCameraIds, const String8 &physicalCameraId,
+ const String8 &logicalCameraId);
+
+ static binder::Status checkSurfaceType(size_t numBufferProducers,
+ bool deferredConsumer, int surfaceType);
+
+ static binder::Status checkOperatingMode(int operatingMode,
+ const CameraMetadata &staticInfo, const String8 &cameraId);
+
// utility function to convert AIDL SessionConfiguration to HIDL
// streamConfiguration. Also checks for validity of SessionConfiguration and
// returns a non-ok binder::Status if the passed in session configuration
@@ -41,6 +79,10 @@
metadataGetter getMetadata, const std::vector<std::string> &physicalCameraIds,
hardware::camera::device::V3_4::StreamConfiguration &streamConfiguration,
bool *earlyExit);
+
+ static const int32_t MAX_SURFACES_PER_STREAM = 4;
+
+ static const int32_t ROUNDING_WIDTH_CAP = 1920;
};
} // android
diff --git a/services/oboeservice/AAudioServiceEndpoint.h b/services/oboeservice/AAudioServiceEndpoint.h
index a171cb0..04b906a 100644
--- a/services/oboeservice/AAudioServiceEndpoint.h
+++ b/services/oboeservice/AAudioServiceEndpoint.h
@@ -47,7 +47,11 @@
virtual aaudio_result_t open(const aaudio::AAudioStreamRequest &request) = 0;
- virtual aaudio_result_t close() = 0;
+ /*
+ * Perform any cleanup necessary before deleting the stream.
+ * This might include releasing and closing internal streams.
+ */
+ virtual void close() = 0;
aaudio_result_t registerStream(android::sp<AAudioServiceStreamBase> stream);
diff --git a/services/oboeservice/AAudioServiceEndpointCapture.cpp b/services/oboeservice/AAudioServiceEndpointCapture.cpp
index de36d50..206a264 100644
--- a/services/oboeservice/AAudioServiceEndpointCapture.cpp
+++ b/services/oboeservice/AAudioServiceEndpointCapture.cpp
@@ -36,8 +36,8 @@
using namespace aaudio; // TODO just import names needed
AAudioServiceEndpointCapture::AAudioServiceEndpointCapture(AAudioService &audioService)
- : mStreamInternalCapture(audioService, true) {
- mStreamInternal = &mStreamInternalCapture;
+ : AAudioServiceEndpointShared(
+ (AudioStreamInternal *)(new AudioStreamInternalCapture(audioService, true))) {
}
aaudio_result_t AAudioServiceEndpointCapture::open(const aaudio::AAudioStreamRequest &request) {
diff --git a/services/oboeservice/AAudioServiceEndpointCapture.h b/services/oboeservice/AAudioServiceEndpointCapture.h
index ae5a189..2ca43cf 100644
--- a/services/oboeservice/AAudioServiceEndpointCapture.h
+++ b/services/oboeservice/AAudioServiceEndpointCapture.h
@@ -37,7 +37,6 @@
void *callbackLoop() override;
private:
- AudioStreamInternalCapture mStreamInternalCapture;
std::unique_ptr<uint8_t[]> mDistributionBuffer;
};
diff --git a/services/oboeservice/AAudioServiceEndpointMMAP.cpp b/services/oboeservice/AAudioServiceEndpointMMAP.cpp
index 0843e0b..04c6453 100644
--- a/services/oboeservice/AAudioServiceEndpointMMAP.cpp
+++ b/services/oboeservice/AAudioServiceEndpointMMAP.cpp
@@ -226,7 +226,7 @@
return result;
}
-aaudio_result_t AAudioServiceEndpointMMAP::close() {
+void AAudioServiceEndpointMMAP::close() {
if (mMmapStream != nullptr) {
// Needs to be explicitly cleared or CTS will fail but it is not clear why.
mMmapStream.clear();
@@ -235,8 +235,6 @@
// FIXME Make closing synchronous.
AudioClock::sleepForNanos(100 * AAUDIO_NANOS_PER_MILLISECOND);
}
-
- return AAUDIO_OK;
}
aaudio_result_t AAudioServiceEndpointMMAP::startStream(sp<AAudioServiceStreamBase> stream,
diff --git a/services/oboeservice/AAudioServiceEndpointMMAP.h b/services/oboeservice/AAudioServiceEndpointMMAP.h
index 3d10861..b6003b6 100644
--- a/services/oboeservice/AAudioServiceEndpointMMAP.h
+++ b/services/oboeservice/AAudioServiceEndpointMMAP.h
@@ -50,7 +50,7 @@
aaudio_result_t open(const aaudio::AAudioStreamRequest &request) override;
- aaudio_result_t close() override;
+ void close() override;
aaudio_result_t startStream(android::sp<AAudioServiceStreamBase> stream,
audio_port_handle_t *clientHandle) override;
diff --git a/services/oboeservice/AAudioServiceEndpointPlay.cpp b/services/oboeservice/AAudioServiceEndpointPlay.cpp
index 1603e41..730d161 100644
--- a/services/oboeservice/AAudioServiceEndpointPlay.cpp
+++ b/services/oboeservice/AAudioServiceEndpointPlay.cpp
@@ -42,8 +42,8 @@
#define BURSTS_PER_BUFFER_DEFAULT 2
AAudioServiceEndpointPlay::AAudioServiceEndpointPlay(AAudioService &audioService)
- : mStreamInternalPlay(audioService, true) {
- mStreamInternal = &mStreamInternalPlay;
+ : AAudioServiceEndpointShared(
+ (AudioStreamInternal *)(new AudioStreamInternalPlay(audioService, true))) {
}
aaudio_result_t AAudioServiceEndpointPlay::open(const aaudio::AAudioStreamRequest &request) {
diff --git a/services/oboeservice/AAudioServiceEndpointPlay.h b/services/oboeservice/AAudioServiceEndpointPlay.h
index 981e430..160a1de 100644
--- a/services/oboeservice/AAudioServiceEndpointPlay.h
+++ b/services/oboeservice/AAudioServiceEndpointPlay.h
@@ -45,7 +45,6 @@
void *callbackLoop() override;
private:
- AudioStreamInternalPlay mStreamInternalPlay; // for playing output of mixer
bool mLatencyTuningEnabled = false; // TODO implement tuning
AAudioMixer mMixer; //
};
diff --git a/services/oboeservice/AAudioServiceEndpointShared.cpp b/services/oboeservice/AAudioServiceEndpointShared.cpp
index dc21886..f5de59f 100644
--- a/services/oboeservice/AAudioServiceEndpointShared.cpp
+++ b/services/oboeservice/AAudioServiceEndpointShared.cpp
@@ -40,6 +40,9 @@
// This is the maximum size in frames. The effective size can be tuned smaller at runtime.
#define DEFAULT_BUFFER_CAPACITY (48 * 8)
+AAudioServiceEndpointShared::AAudioServiceEndpointShared(AudioStreamInternal *streamInternal)
+ : mStreamInternal(streamInternal) {}
+
std::string AAudioServiceEndpointShared::dump() const {
std::stringstream result;
@@ -84,8 +87,8 @@
return result;
}
-aaudio_result_t AAudioServiceEndpointShared::close() {
- return getStreamInternal()->releaseCloseFinal();
+void AAudioServiceEndpointShared::close() {
+ getStreamInternal()->releaseCloseFinal();
}
// Glue between C and C++ callbacks.
diff --git a/services/oboeservice/AAudioServiceEndpointShared.h b/services/oboeservice/AAudioServiceEndpointShared.h
index bfc1744..020b926 100644
--- a/services/oboeservice/AAudioServiceEndpointShared.h
+++ b/services/oboeservice/AAudioServiceEndpointShared.h
@@ -35,12 +35,13 @@
class AAudioServiceEndpointShared : public AAudioServiceEndpoint {
public:
+ explicit AAudioServiceEndpointShared(AudioStreamInternal *streamInternal);
std::string dump() const override;
aaudio_result_t open(const aaudio::AAudioStreamRequest &request) override;
- aaudio_result_t close() override;
+ void close() override;
aaudio_result_t startStream(android::sp<AAudioServiceStreamBase> stream,
audio_port_handle_t *clientHandle) override;
@@ -57,15 +58,15 @@
protected:
AudioStreamInternal *getStreamInternal() const {
- return mStreamInternal;
+ return mStreamInternal.get();
};
aaudio_result_t startSharingThread_l();
aaudio_result_t stopSharingThread();
- // pointer to object statically allocated in subclasses
- AudioStreamInternal *mStreamInternal = nullptr;
+ // An MMAP stream that is shared by multiple clients.
+ android::sp<AudioStreamInternal> mStreamInternal;
std::atomic<bool> mCallbackEnabled{false};