Merge "Camera: Invalidate buffer cache for partial buffer request failure" into sc-qpr1-dev am: 1c16bc6ac9
Original change: https://googleplex-android-review.googlesource.com/c/platform/frameworks/av/+/15779277
Change-Id: Ic5bba64d6598698a2f044c03b14fe871f60c310b
diff --git a/media/libaaudio/include/aaudio/AAudio.h b/media/libaaudio/include/aaudio/AAudio.h
index 4b08295..f97fe4d 100644
--- a/media/libaaudio/include/aaudio/AAudio.h
+++ b/media/libaaudio/include/aaudio/AAudio.h
@@ -565,6 +565,145 @@
};
typedef int32_t aaudio_session_id_t;
+/**
+ * Defines the audio channel mask.
+ * Channel masks are used to describe the samples and their
+ * arrangement in the audio frame. They are also used in the endpoint
+ * (e.g. a USB audio interface, a DAC connected to headphones) to
+ * specify allowable configurations of a particular device.
+ *
+ * Added in API level 32.
+ */
+enum {
+ /**
+ * Invalid channel mask
+ */
+ AAUDIO_CHANNEL_INVALID = -1,
+
+ /**
+ * Output audio channel mask
+ */
+ AAUDIO_CHANNEL_FRONT_LEFT = 1 << 0,
+ AAUDIO_CHANNEL_FRONT_RIGHT = 1 << 1,
+ AAUDIO_CHANNEL_FRONT_CENTER = 1 << 2,
+ AAUDIO_CHANNEL_LOW_FREQUENCY = 1 << 3,
+ AAUDIO_CHANNEL_BACK_LEFT = 1 << 4,
+ AAUDIO_CHANNEL_BACK_RIGHT = 1 << 5,
+ AAUDIO_CHANNEL_FRONT_LEFT_OF_CENTER = 1 << 6,
+ AAUDIO_CHANNEL_FRONT_RIGHT_OF_CENTER = 1 << 7,
+ AAUDIO_CHANNEL_BACK_CENTER = 1 << 8,
+ AAUDIO_CHANNEL_SIDE_LEFT = 1 << 9,
+ AAUDIO_CHANNEL_SIDE_RIGHT = 1 << 10,
+ AAUDIO_CHANNEL_TOP_CENTER = 1 << 11,
+ AAUDIO_CHANNEL_TOP_FRONT_LEFT = 1 << 12,
+ AAUDIO_CHANNEL_TOP_FRONT_CENTER = 1 << 13,
+ AAUDIO_CHANNEL_TOP_FRONT_RIGHT = 1 << 14,
+ AAUDIO_CHANNEL_TOP_BACK_LEFT = 1 << 15,
+ AAUDIO_CHANNEL_TOP_BACK_CENTER = 1 << 16,
+ AAUDIO_CHANNEL_TOP_BACK_RIGHT = 1 << 17,
+ AAUDIO_CHANNEL_TOP_SIDE_LEFT = 1 << 18,
+ AAUDIO_CHANNEL_TOP_SIDE_RIGHT = 1 << 19,
+ AAUDIO_CHANNEL_BOTTOM_FRONT_LEFT = 1 << 20,
+ AAUDIO_CHANNEL_BOTTOM_FRONT_CENTER = 1 << 21,
+ AAUDIO_CHANNEL_BOTTOM_FRONT_RIGHT = 1 << 22,
+ AAUDIO_CHANNEL_LOW_FREQUENCY_2 = 1 << 23,
+ AAUDIO_CHANNEL_FRONT_WIDE_LEFT = 1 << 24,
+ AAUDIO_CHANNEL_FRONT_WIDE_RIGHT = 1 << 25,
+
+ AAUDIO_CHANNEL_MONO = AAUDIO_CHANNEL_FRONT_LEFT,
+ AAUDIO_CHANNEL_STEREO = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT,
+ AAUDIO_CHANNEL_2POINT1 = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_LOW_FREQUENCY,
+ AAUDIO_CHANNEL_TRI = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_FRONT_CENTER,
+ AAUDIO_CHANNEL_TRI_BACK = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_BACK_CENTER,
+ AAUDIO_CHANNEL_3POINT1 = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_FRONT_CENTER |
+ AAUDIO_CHANNEL_LOW_FREQUENCY,
+ AAUDIO_CHANNEL_2POINT0POINT2 = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_TOP_SIDE_LEFT |
+ AAUDIO_CHANNEL_TOP_SIDE_RIGHT,
+ AAUDIO_CHANNEL_2POINT1POINT2 = AAUDIO_CHANNEL_2POINT0POINT2 |
+ AAUDIO_CHANNEL_LOW_FREQUENCY,
+ AAUDIO_CHANNEL_3POINT0POINT2 = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_FRONT_CENTER |
+ AAUDIO_CHANNEL_TOP_SIDE_LEFT |
+ AAUDIO_CHANNEL_TOP_SIDE_RIGHT,
+ AAUDIO_CHANNEL_3POINT1POINT2 = AAUDIO_CHANNEL_3POINT0POINT2 |
+ AAUDIO_CHANNEL_LOW_FREQUENCY,
+ AAUDIO_CHANNEL_QUAD = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_BACK_LEFT |
+ AAUDIO_CHANNEL_BACK_RIGHT,
+ AAUDIO_CHANNEL_QUAD_SIDE = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_SIDE_LEFT |
+ AAUDIO_CHANNEL_SIDE_RIGHT,
+ AAUDIO_CHANNEL_SURROUND = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_FRONT_CENTER |
+ AAUDIO_CHANNEL_BACK_CENTER,
+ AAUDIO_CHANNEL_PENTA = AAUDIO_CHANNEL_QUAD |
+ AAUDIO_CHANNEL_FRONT_CENTER,
+ // aka 5POINT1_BACK
+ AAUDIO_CHANNEL_5POINT1 = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_FRONT_CENTER |
+ AAUDIO_CHANNEL_LOW_FREQUENCY |
+ AAUDIO_CHANNEL_BACK_LEFT |
+ AAUDIO_CHANNEL_BACK_RIGHT,
+ AAUDIO_CHANNEL_5POINT1_SIDE = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_FRONT_CENTER |
+ AAUDIO_CHANNEL_LOW_FREQUENCY |
+ AAUDIO_CHANNEL_SIDE_LEFT |
+ AAUDIO_CHANNEL_SIDE_RIGHT,
+ AAUDIO_CHANNEL_6POINT1 = AAUDIO_CHANNEL_FRONT_LEFT |
+ AAUDIO_CHANNEL_FRONT_RIGHT |
+ AAUDIO_CHANNEL_FRONT_CENTER |
+ AAUDIO_CHANNEL_LOW_FREQUENCY |
+ AAUDIO_CHANNEL_BACK_LEFT |
+ AAUDIO_CHANNEL_BACK_RIGHT |
+ AAUDIO_CHANNEL_BACK_CENTER,
+ AAUDIO_CHANNEL_7POINT1 = AAUDIO_CHANNEL_5POINT1 |
+ AAUDIO_CHANNEL_SIDE_LEFT |
+ AAUDIO_CHANNEL_SIDE_RIGHT,
+ AAUDIO_CHANNEL_5POINT1POINT2 = AAUDIO_CHANNEL_5POINT1 |
+ AAUDIO_CHANNEL_TOP_SIDE_LEFT |
+ AAUDIO_CHANNEL_TOP_SIDE_RIGHT,
+ AAUDIO_CHANNEL_5POINT1POINT4 = AAUDIO_CHANNEL_5POINT1 |
+ AAUDIO_CHANNEL_TOP_FRONT_LEFT |
+ AAUDIO_CHANNEL_TOP_FRONT_RIGHT |
+ AAUDIO_CHANNEL_TOP_BACK_LEFT |
+ AAUDIO_CHANNEL_TOP_BACK_RIGHT,
+ AAUDIO_CHANNEL_7POINT1POINT2 = AAUDIO_CHANNEL_7POINT1 |
+ AAUDIO_CHANNEL_TOP_SIDE_LEFT |
+ AAUDIO_CHANNEL_TOP_SIDE_RIGHT,
+ AAUDIO_CHANNEL_7POINT1POINT4 = AAUDIO_CHANNEL_7POINT1 |
+ AAUDIO_CHANNEL_TOP_FRONT_LEFT |
+ AAUDIO_CHANNEL_TOP_FRONT_RIGHT |
+ AAUDIO_CHANNEL_TOP_BACK_LEFT |
+ AAUDIO_CHANNEL_TOP_BACK_RIGHT,
+ AAUDIO_CHANNEL_9POINT1POINT4 = AAUDIO_CHANNEL_7POINT1POINT4 |
+ AAUDIO_CHANNEL_FRONT_WIDE_LEFT |
+ AAUDIO_CHANNEL_FRONT_WIDE_RIGHT,
+ AAUDIO_CHANNEL_9POINT1POINT6 = AAUDIO_CHANNEL_9POINT1POINT4 |
+ AAUDIO_CHANNEL_TOP_SIDE_LEFT |
+ AAUDIO_CHANNEL_TOP_SIDE_RIGHT,
+
+ AAUDIO_CHANNEL_FRONT_BACK = AAUDIO_CHANNEL_FRONT_CENTER |
+ AAUDIO_CHANNEL_BACK_CENTER,
+};
+typedef uint32_t aaudio_channel_mask_t;
+
typedef struct AAudioStreamStruct AAudioStream;
typedef struct AAudioStreamBuilderStruct AAudioStreamBuilder;
@@ -699,6 +838,11 @@
* If an exact value is specified then an opened stream will use that value.
* If a stream cannot be opened with the specified value then the open will fail.
*
+ * As the channel count provided here may be different from the corresponding channel count
+ * of channel mask used in {@link AAudioStreamBuilder_setChannelMask}, the last called function
+ * will be respected if both this function and {@link AAudioStreamBuilder_setChannelMask} are
+ * called.
+ *
* Available since API level 26.
*
* @param builder reference provided by AAudio_createStreamBuilder()
@@ -714,6 +858,8 @@
*
* @param builder reference provided by AAudio_createStreamBuilder()
* @param samplesPerFrame Number of samples in a frame.
+ *
+ * @deprecated use {@link AAudioStreamBuilder_setChannelCount}
*/
AAUDIO_API void AAudioStreamBuilder_setSamplesPerFrame(AAudioStreamBuilder* builder,
int32_t samplesPerFrame) __INTRODUCED_IN(26);
@@ -1136,6 +1282,32 @@
AAUDIO_API aaudio_result_t AAudioStreamBuilder_delete(AAudioStreamBuilder* builder)
__INTRODUCED_IN(26);
+/**
+ * Set audio channel mask for the stream.
+ *
+ * The default, if you do not call this function, is {@link #AAUDIO_UNSPECIFIED}.
+ * If both channel mask and count are not set, then stereo will then be chosen when the
+ * stream is opened.
+ * After opening a stream with an unspecified value, the application must query for the
+ * actual value, which may vary by device.
+ *
+ * If an exact value is specified then an opened stream will use that value.
+ * If a stream cannot be opened with the specified value then the open will fail.
+ *
+ * As the corresponding channel count of provided channel mask here may be different
+ * from the channel count used in {@link AAudioStreamBuilder_setChannelCount} or
+ * {@link AAudioStreamBuilder_setSamplesPerFrame}, the last called function will be
+ * respected if this function and {@link AAudioStreamBuilder_setChannelCount} or
+ * {@link AAudioStreamBuilder_setSamplesPerFrame} are called.
+ *
+ * Available since API level 32.
+ *
+ * @param builder reference provided by AAudio_createStreamBuilder()
+ * @param channelMask Audio channel mask desired.
+ */
+AAUDIO_API void AAudioStreamBuilder_setChannelMask(AAudioStreamBuilder* builder,
+ aaudio_channel_mask_t channelMask) __INTRODUCED_IN(32);
+
// ============================================================
// Stream Control
// ============================================================
@@ -1652,6 +1824,18 @@
AAUDIO_API bool AAudioStream_isPrivacySensitive(AAudioStream* stream)
__INTRODUCED_IN(30);
+/**
+ * Return the channel mask for the stream. This will be the mask set using
+ * {@link #AAudioStreamBuilder_setChannelMask}, or {@link #AAUDIO_UNSPECIFIED} otherwise.
+ *
+ * Available since API level 32.
+ *
+ * @param stream reference provided by AAudioStreamBuilder_openStream()
+ * @return actual channel mask
+ */
+AAUDIO_API aaudio_channel_mask_t AAudioStream_getChannelMask(AAudioStream* stream)
+ __INTRODUCED_IN(32);
+
#ifdef __cplusplus
}
#endif
diff --git a/media/libaaudio/src/binding/AAudioStreamConfiguration.cpp b/media/libaaudio/src/binding/AAudioStreamConfiguration.cpp
index 2d501ef..dc19bb3 100644
--- a/media/libaaudio/src/binding/AAudioStreamConfiguration.cpp
+++ b/media/libaaudio/src/binding/AAudioStreamConfiguration.cpp
@@ -30,7 +30,7 @@
using android::media::audio::common::AudioFormat;
AAudioStreamConfiguration::AAudioStreamConfiguration(const StreamParameters& parcelable) {
- setSamplesPerFrame(parcelable.samplesPerFrame);
+ setChannelMask(parcelable.channelMask);
setSampleRate(parcelable.sampleRate);
setDeviceId(parcelable.deviceId);
static_assert(sizeof(aaudio_sharing_mode_t) == sizeof(parcelable.sharingMode));
@@ -63,7 +63,7 @@
StreamParameters AAudioStreamConfiguration::parcelable() const {
StreamParameters result;
- result.samplesPerFrame = getSamplesPerFrame();
+ result.channelMask = getChannelMask();
result.sampleRate = getSampleRate();
result.deviceId = getDeviceId();
static_assert(sizeof(aaudio_sharing_mode_t) == sizeof(result.sharingMode));
diff --git a/media/libaaudio/src/binding/aidl/aaudio/StreamParameters.aidl b/media/libaaudio/src/binding/aidl/aaudio/StreamParameters.aidl
index b7c4f70..1a971b0 100644
--- a/media/libaaudio/src/binding/aidl/aaudio/StreamParameters.aidl
+++ b/media/libaaudio/src/binding/aidl/aaudio/StreamParameters.aidl
@@ -19,7 +19,7 @@
import android.media.audio.common.AudioFormat;
parcelable StreamParameters {
- int samplesPerFrame; // = AAUDIO_UNSPECIFIED;
+ int channelMask; // = AAUDIO_UNSPECIFIED;
int sampleRate; // = AAUDIO_UNSPECIFIED;
int deviceId; // = AAUDIO_UNSPECIFIED;
int /* aaudio_sharing_mode_t */ sharingMode; // = AAUDIO_SHARING_MODE_SHARED;
diff --git a/media/libaaudio/src/client/AudioStreamInternal.cpp b/media/libaaudio/src/client/AudioStreamInternal.cpp
index 6d2d464..3063a7b 100644
--- a/media/libaaudio/src/client/AudioStreamInternal.cpp
+++ b/media/libaaudio/src/client/AudioStreamInternal.cpp
@@ -123,9 +123,9 @@
request.getConfiguration().setDeviceId(getDeviceId());
request.getConfiguration().setSampleRate(getSampleRate());
- request.getConfiguration().setSamplesPerFrame(getSamplesPerFrame());
request.getConfiguration().setDirection(getDirection());
request.getConfiguration().setSharingMode(getSharingMode());
+ request.getConfiguration().setChannelMask(getChannelMask());
request.getConfiguration().setUsage(getUsage());
request.getConfiguration().setContentType(getContentType());
@@ -138,7 +138,8 @@
mServiceStreamHandle = mServiceInterface.openStream(request, configurationOutput);
if (mServiceStreamHandle < 0
- && request.getConfiguration().getSamplesPerFrame() == 1 // mono?
+ && (request.getConfiguration().getSamplesPerFrame() == 1
+ || request.getConfiguration().getChannelMask() == AAUDIO_CHANNEL_MONO)
&& getDirection() == AAUDIO_DIRECTION_OUTPUT
&& !isInService()) {
// if that failed then try switching from mono to stereo if OUTPUT.
@@ -146,7 +147,7 @@
// that writes to a stereo MMAP stream.
ALOGD("%s() - openStream() returned %d, try switching from MONO to STEREO",
__func__, mServiceStreamHandle);
- request.getConfiguration().setSamplesPerFrame(2); // stereo
+ request.getConfiguration().setChannelMask(AAUDIO_CHANNEL_STEREO);
mServiceStreamHandle = mServiceInterface.openStream(request, configurationOutput);
}
if (mServiceStreamHandle < 0) {
@@ -174,9 +175,10 @@
goto error;
}
// Save results of the open.
- if (getSamplesPerFrame() == AAUDIO_UNSPECIFIED) {
- setSamplesPerFrame(configurationOutput.getSamplesPerFrame());
+ if (getChannelMask() == AAUDIO_UNSPECIFIED) {
+ setChannelMask(configurationOutput.getChannelMask());
}
+
mDeviceChannelCount = configurationOutput.getSamplesPerFrame();
setSampleRate(configurationOutput.getSampleRate());
diff --git a/media/libaaudio/src/core/AAudioAudio.cpp b/media/libaaudio/src/core/AAudioAudio.cpp
index d103aca..02e7f5f 100644
--- a/media/libaaudio/src/core/AAudioAudio.cpp
+++ b/media/libaaudio/src/core/AAudioAudio.cpp
@@ -128,7 +128,8 @@
int32_t samplesPerFrame)
{
AudioStreamBuilder *streamBuilder = convertAAudioBuilderToStreamBuilder(builder);
- streamBuilder->setSamplesPerFrame(samplesPerFrame);
+ const aaudio_channel_mask_t channelMask = AAudioConvert_channelCountToMask(samplesPerFrame);
+ streamBuilder->setChannelMask(channelMask);
}
AAUDIO_API void AAudioStreamBuilder_setDirection(AAudioStreamBuilder* builder,
@@ -223,6 +224,13 @@
streamBuilder->setFramesPerDataCallback(frames);
}
+AAUDIO_API void AAudioStreamBuilder_setChannelMask(AAudioStreamBuilder* builder,
+ aaudio_channel_mask_t channelMask)
+{
+ AudioStreamBuilder *streamBuilder = convertAAudioBuilderToStreamBuilder(builder);
+ streamBuilder->setChannelMask(channelMask);
+}
+
AAUDIO_API aaudio_result_t AAudioStreamBuilder_openStream(AAudioStreamBuilder* builder,
AAudioStream** streamPtr)
{
@@ -562,3 +570,11 @@
AudioStream *audioStream = convertAAudioStreamToAudioStream(stream);
return audioStream->isPrivacySensitive();
}
+
+AAUDIO_API aaudio_channel_mask_t AAudioStream_getChannelMask(AAudioStream* stream)
+{
+ AudioStream *audioStream = convertAAudioStreamToAudioStream(stream);
+ const aaudio_channel_mask_t channelMask = audioStream->getChannelMask();
+ // Do not return channel index masks as they are not public.
+ return AAudio_isChannelIndexMask(channelMask) ? AAUDIO_UNSPECIFIED : channelMask;
+}
diff --git a/media/libaaudio/src/core/AAudioStreamParameters.cpp b/media/libaaudio/src/core/AAudioStreamParameters.cpp
index acfac24..59d94eb 100644
--- a/media/libaaudio/src/core/AAudioStreamParameters.cpp
+++ b/media/libaaudio/src/core/AAudioStreamParameters.cpp
@@ -49,6 +49,7 @@
mIsPrivacySensitive = other.mIsPrivacySensitive;
mOpPackageName = other.mOpPackageName;
mAttributionTag = other.mAttributionTag;
+ mChannelMask = other.mChannelMask;
}
static aaudio_result_t isFormatValid(audio_format_t format) {
@@ -187,7 +188,94 @@
// break;
}
- return AAUDIO_OK;
+ return validateChannelMask();
+}
+
+bool AAudioStreamParameters::validateChannelMask() const {
+ if (mChannelMask == AAUDIO_UNSPECIFIED) {
+ return AAUDIO_OK;
+ }
+
+ if (mChannelMask & AAUDIO_CHANNEL_BIT_INDEX) {
+ switch (mChannelMask) {
+ case AAUDIO_CHANNEL_INDEX_MASK_1:
+ case AAUDIO_CHANNEL_INDEX_MASK_2:
+ case AAUDIO_CHANNEL_INDEX_MASK_3:
+ case AAUDIO_CHANNEL_INDEX_MASK_4:
+ case AAUDIO_CHANNEL_INDEX_MASK_5:
+ case AAUDIO_CHANNEL_INDEX_MASK_6:
+ case AAUDIO_CHANNEL_INDEX_MASK_7:
+ case AAUDIO_CHANNEL_INDEX_MASK_8:
+ case AAUDIO_CHANNEL_INDEX_MASK_9:
+ case AAUDIO_CHANNEL_INDEX_MASK_10:
+ case AAUDIO_CHANNEL_INDEX_MASK_11:
+ case AAUDIO_CHANNEL_INDEX_MASK_12:
+ case AAUDIO_CHANNEL_INDEX_MASK_13:
+ case AAUDIO_CHANNEL_INDEX_MASK_14:
+ case AAUDIO_CHANNEL_INDEX_MASK_15:
+ case AAUDIO_CHANNEL_INDEX_MASK_16:
+ case AAUDIO_CHANNEL_INDEX_MASK_17:
+ case AAUDIO_CHANNEL_INDEX_MASK_18:
+ case AAUDIO_CHANNEL_INDEX_MASK_19:
+ case AAUDIO_CHANNEL_INDEX_MASK_20:
+ case AAUDIO_CHANNEL_INDEX_MASK_21:
+ case AAUDIO_CHANNEL_INDEX_MASK_22:
+ case AAUDIO_CHANNEL_INDEX_MASK_23:
+ case AAUDIO_CHANNEL_INDEX_MASK_24:
+ return AAUDIO_OK;
+ default:
+ ALOGD("Invalid channel index mask %#x", mChannelMask);
+ return AAUDIO_ERROR_ILLEGAL_ARGUMENT;
+ }
+ }
+
+ if (getDirection() == AAUDIO_DIRECTION_INPUT) {
+ switch (mChannelMask) {
+ case AAUDIO_CHANNEL_MONO:
+ case AAUDIO_CHANNEL_STEREO:
+ case AAUDIO_CHANNEL_FRONT_BACK:
+ case AAUDIO_CHANNEL_2POINT0POINT2:
+ case AAUDIO_CHANNEL_2POINT1POINT2:
+ case AAUDIO_CHANNEL_3POINT0POINT2:
+ case AAUDIO_CHANNEL_3POINT1POINT2:
+ case AAUDIO_CHANNEL_5POINT1:
+ return AAUDIO_OK;
+ default:
+ ALOGD("Invalid channel mask %#x, IN", mChannelMask);
+ return AAUDIO_ERROR_ILLEGAL_ARGUMENT;
+ }
+ } else {
+ switch (mChannelMask) {
+ case AAUDIO_CHANNEL_MONO:
+ case AAUDIO_CHANNEL_STEREO:
+ case AAUDIO_CHANNEL_2POINT1:
+ case AAUDIO_CHANNEL_TRI:
+ case AAUDIO_CHANNEL_TRI_BACK:
+ case AAUDIO_CHANNEL_3POINT1:
+ case AAUDIO_CHANNEL_2POINT0POINT2:
+ case AAUDIO_CHANNEL_2POINT1POINT2:
+ case AAUDIO_CHANNEL_3POINT0POINT2:
+ case AAUDIO_CHANNEL_3POINT1POINT2:
+ case AAUDIO_CHANNEL_QUAD:
+ case AAUDIO_CHANNEL_QUAD_SIDE:
+ case AAUDIO_CHANNEL_SURROUND:
+ case AAUDIO_CHANNEL_PENTA:
+ case AAUDIO_CHANNEL_5POINT1:
+ case AAUDIO_CHANNEL_5POINT1_SIDE:
+ case AAUDIO_CHANNEL_5POINT1POINT2:
+ case AAUDIO_CHANNEL_5POINT1POINT4:
+ case AAUDIO_CHANNEL_6POINT1:
+ case AAUDIO_CHANNEL_7POINT1:
+ case AAUDIO_CHANNEL_7POINT1POINT2:
+ case AAUDIO_CHANNEL_7POINT1POINT4:
+ case AAUDIO_CHANNEL_9POINT1POINT4:
+ case AAUDIO_CHANNEL_9POINT1POINT6:
+ return AAUDIO_OK;
+ default:
+ ALOGD("Invalid channel mask %#x. OUT", mChannelMask);
+ return AAUDIO_ERROR_ILLEGAL_ARGUMENT;
+ }
+ }
}
void AAudioStreamParameters::dump() const {
@@ -195,6 +283,7 @@
ALOGD("mSessionId = %6d", mSessionId);
ALOGD("mSampleRate = %6d", mSampleRate);
ALOGD("mSamplesPerFrame = %6d", mSamplesPerFrame);
+ ALOGD("mChannelMask = %#x", mChannelMask);
ALOGD("mSharingMode = %6d", (int)mSharingMode);
ALOGD("mAudioFormat = %6d", (int)mAudioFormat);
ALOGD("mDirection = %6d", mDirection);
diff --git a/media/libaaudio/src/core/AAudioStreamParameters.h b/media/libaaudio/src/core/AAudioStreamParameters.h
index 5737052..a5c8043 100644
--- a/media/libaaudio/src/core/AAudioStreamParameters.h
+++ b/media/libaaudio/src/core/AAudioStreamParameters.h
@@ -49,13 +49,6 @@
return mSamplesPerFrame;
}
- /**
- * This is also known as channelCount.
- */
- void setSamplesPerFrame(int32_t samplesPerFrame) {
- mSamplesPerFrame = samplesPerFrame;
- }
-
audio_format_t getFormat() const {
return mAudioFormat;
}
@@ -153,6 +146,15 @@
mAttributionTag = attributionTag;
}
+ aaudio_channel_mask_t getChannelMask() const {
+ return mChannelMask;
+ }
+
+ void setChannelMask(aaudio_channel_mask_t channelMask) {
+ mChannelMask = channelMask;
+ mSamplesPerFrame = AAudioConvert_channelMaskToCount(channelMask);
+ }
+
/**
* @return bytes per frame of getFormat()
*/
@@ -171,6 +173,8 @@
void dump() const;
private:
+ bool validateChannelMask() const;
+
int32_t mSamplesPerFrame = AAUDIO_UNSPECIFIED;
int32_t mSampleRate = AAUDIO_UNSPECIFIED;
int32_t mDeviceId = AAUDIO_UNSPECIFIED;
@@ -186,6 +190,7 @@
bool mIsPrivacySensitive = false;
std::optional<std::string> mOpPackageName = {};
std::optional<std::string> mAttributionTag = {};
+ aaudio_channel_mask_t mChannelMask = AAUDIO_UNSPECIFIED;
};
} /* namespace aaudio */
diff --git a/media/libaaudio/src/core/AudioStream.cpp b/media/libaaudio/src/core/AudioStream.cpp
index 09d9535..ffc3b9d 100644
--- a/media/libaaudio/src/core/AudioStream.cpp
+++ b/media/libaaudio/src/core/AudioStream.cpp
@@ -76,6 +76,7 @@
// Copy parameters from the Builder because the Builder may be deleted after this call.
// TODO AudioStream should be a subclass of AudioStreamParameters
mSamplesPerFrame = builder.getSamplesPerFrame();
+ mChannelMask = builder.getChannelMask();
mSampleRate = builder.getSampleRate();
mDeviceId = builder.getDeviceId();
mFormat = builder.getFormat();
diff --git a/media/libaaudio/src/core/AudioStream.h b/media/libaaudio/src/core/AudioStream.h
index 9835c8c..47693f8 100644
--- a/media/libaaudio/src/core/AudioStream.h
+++ b/media/libaaudio/src/core/AudioStream.h
@@ -270,7 +270,8 @@
}
/**
- * This is only valid after setSamplesPerFrame() and setFormat() have been called.
+ * This is only valid after setChannelMask() and setFormat()
+ * have been called.
*/
int32_t getBytesPerFrame() const {
return mSamplesPerFrame * getBytesPerSample();
@@ -284,7 +285,7 @@
}
/**
- * This is only valid after setSamplesPerFrame() and setDeviceFormat() have been called.
+ * This is only valid after setChannelMask() and setDeviceFormat() have been called.
*/
int32_t getBytesPerDeviceFrame() const {
return getSamplesPerFrame() * audio_bytes_per_sample(getDeviceFormat());
@@ -318,6 +319,15 @@
return mFramesPerDataCallback;
}
+ aaudio_channel_mask_t getChannelMask() const {
+ return mChannelMask;
+ }
+
+ void setChannelMask(aaudio_channel_mask_t channelMask) {
+ mChannelMask = channelMask;
+ mSamplesPerFrame = AAudioConvert_channelMaskToCount(channelMask);
+ }
+
/**
* @return true if data callback has been specified
*/
@@ -495,11 +505,6 @@
}
// This should not be called after the open() call.
- void setSamplesPerFrame(int32_t samplesPerFrame) {
- mSamplesPerFrame = samplesPerFrame;
- }
-
- // This should not be called after the open() call.
void setFramesPerBurst(int32_t framesPerBurst) {
mFramesPerBurst = framesPerBurst;
}
@@ -633,6 +638,7 @@
// These do not change after open().
int32_t mSamplesPerFrame = AAUDIO_UNSPECIFIED;
+ aaudio_channel_mask_t mChannelMask = AAUDIO_UNSPECIFIED;
int32_t mSampleRate = AAUDIO_UNSPECIFIED;
int32_t mDeviceId = AAUDIO_UNSPECIFIED;
aaudio_sharing_mode_t mSharingMode = AAUDIO_SHARING_MODE_SHARED;
diff --git a/media/libaaudio/src/core/AudioStreamBuilder.cpp b/media/libaaudio/src/core/AudioStreamBuilder.cpp
index e015592..5e1e007 100644
--- a/media/libaaudio/src/core/AudioStreamBuilder.cpp
+++ b/media/libaaudio/src/core/AudioStreamBuilder.cpp
@@ -268,8 +268,8 @@
void AudioStreamBuilder::logParameters() const {
// This is very helpful for debugging in the future. Please leave it in.
- ALOGI("rate = %6d, channels = %d, format = %d, sharing = %s, dir = %s",
- getSampleRate(), getSamplesPerFrame(), getFormat(),
+ ALOGI("rate = %6d, channels = %d, channelMask = %#x, format = %d, sharing = %s, dir = %s",
+ getSampleRate(), getSamplesPerFrame(), getChannelMask(), getFormat(),
AAudio_convertSharingModeToShortText(getSharingMode()),
AAudio_convertDirectionToText(getDirection()));
ALOGI("device = %6d, sessionId = %d, perfMode = %d, callback: %s with frames = %d",
diff --git a/media/libaaudio/src/legacy/AudioStreamRecord.cpp b/media/libaaudio/src/legacy/AudioStreamRecord.cpp
index dc66742..fe8fb19 100644
--- a/media/libaaudio/src/legacy/AudioStreamRecord.cpp
+++ b/media/libaaudio/src/legacy/AudioStreamRecord.cpp
@@ -65,11 +65,8 @@
const audio_session_t sessionId = AAudioConvert_aaudioToAndroidSessionId(requestedSessionId);
// TODO Support UNSPECIFIED in AudioRecord. For now, use stereo if unspecified.
- int32_t samplesPerFrame = (getSamplesPerFrame() == AAUDIO_UNSPECIFIED)
- ? 2 : getSamplesPerFrame();
- audio_channel_mask_t channelMask = samplesPerFrame <= 2 ?
- audio_channel_in_mask_from_count(samplesPerFrame) :
- audio_channel_mask_for_index_assignment_from_count(samplesPerFrame);
+ audio_channel_mask_t channelMask =
+ AAudio_getChannelMaskForOpen(getChannelMask(), getSamplesPerFrame(), true /*isInput*/);
size_t frameCount = (builder.getBufferCapacity() == AAUDIO_UNSPECIFIED) ? 0
: builder.getBufferCapacity();
@@ -115,7 +112,7 @@
constexpr int32_t kMostLikelySampleRateForFast = 48000;
if (getFormat() == AUDIO_FORMAT_PCM_FLOAT
&& perfMode == AAUDIO_PERFORMANCE_MODE_LOW_LATENCY
- && (samplesPerFrame <= 2) // FAST only for mono and stereo
+ && (audio_channel_count_from_in_mask(channelMask) <= 2) // FAST only for mono and stereo
&& (getSampleRate() == kMostLikelySampleRateForFast
|| getSampleRate() == AAUDIO_UNSPECIFIED)) {
setDeviceFormat(AUDIO_FORMAT_PCM_16_BIT);
@@ -228,7 +225,9 @@
.set(AMEDIAMETRICS_PROP_ENCODINGCLIENT, toString(requestedFormat).c_str()).record();
// Get the actual values from the AudioRecord.
- setSamplesPerFrame(mAudioRecord->channelCount());
+ setChannelMask(AAudioConvert_androidToAAudioChannelMask(
+ mAudioRecord->channelMask(), true /*isInput*/,
+ AAudio_isChannelIndexMask(getChannelMask())));
setSampleRate(mAudioRecord->getSampleRate());
setBufferCapacity(getBufferCapacityFromDevice());
setFramesPerBurst(getFramesPerBurstFromDevice());
diff --git a/media/libaaudio/src/legacy/AudioStreamTrack.cpp b/media/libaaudio/src/legacy/AudioStreamTrack.cpp
index 1d412c0..2291ad7 100644
--- a/media/libaaudio/src/legacy/AudioStreamTrack.cpp
+++ b/media/libaaudio/src/legacy/AudioStreamTrack.cpp
@@ -66,13 +66,8 @@
const aaudio_session_id_t requestedSessionId = builder.getSessionId();
const audio_session_t sessionId = AAudioConvert_aaudioToAndroidSessionId(requestedSessionId);
- // Try to create an AudioTrack
- // Use stereo if unspecified.
- int32_t samplesPerFrame = (getSamplesPerFrame() == AAUDIO_UNSPECIFIED)
- ? 2 : getSamplesPerFrame();
- audio_channel_mask_t channelMask = samplesPerFrame <= 2 ?
- audio_channel_out_mask_from_count(samplesPerFrame) :
- audio_channel_mask_for_index_assignment_from_count(samplesPerFrame);
+ audio_channel_mask_t channelMask =
+ AAudio_getChannelMaskForOpen(getChannelMask(), getSamplesPerFrame(), false /*isInput*/);
audio_output_flags_t flags;
aaudio_performance_mode_t perfMode = getPerformanceMode();
@@ -199,7 +194,9 @@
doSetVolume();
// Get the actual values from the AudioTrack.
- setSamplesPerFrame(mAudioTrack->channelCount());
+ setChannelMask(AAudioConvert_androidToAAudioChannelMask(
+ mAudioTrack->channelMask(), false /*isInput*/,
+ AAudio_isChannelIndexMask(getChannelMask())));
setFormat(mAudioTrack->format());
setDeviceFormat(mAudioTrack->format());
setSampleRate(mAudioTrack->getSampleRate());
diff --git a/media/libaaudio/src/libaaudio.map.txt b/media/libaaudio/src/libaaudio.map.txt
index 1dd44d1..8fa9e38 100644
--- a/media/libaaudio/src/libaaudio.map.txt
+++ b/media/libaaudio/src/libaaudio.map.txt
@@ -25,6 +25,7 @@
AAudioStreamBuilder_setPrivacySensitive; # introduced=30
AAudioStreamBuilder_setPackageName; # introduced=31
AAudioStreamBuilder_setAttributionTag; # introduced=31
+ AAudioStreamBuilder_setChannelMask; # introduced=32
AAudioStreamBuilder_openStream;
AAudioStreamBuilder_delete;
AAudioStream_close;
@@ -61,6 +62,7 @@
AAudioStream_isMMapUsed;
AAudioStream_isPrivacySensitive; # introduced=30
AAudioStream_release; # introduced=30
+ AAudioStream_getChannelMask; # introduced=32
local:
*;
};
diff --git a/media/libaaudio/src/utility/AAudioUtilities.cpp b/media/libaaudio/src/utility/AAudioUtilities.cpp
index d795725..d829934 100644
--- a/media/libaaudio/src/utility/AAudioUtilities.cpp
+++ b/media/libaaudio/src/utility/AAudioUtilities.cpp
@@ -256,6 +256,248 @@
return privacySensitive ? AUDIO_FLAG_CAPTURE_PRIVATE : AUDIO_FLAG_NONE;
}
+audio_channel_mask_t AAudioConvert_aaudioToAndroidChannelLayoutMask(
+ aaudio_channel_mask_t channelMask, bool isInput) {
+ if (isInput) {
+ switch (channelMask) {
+ case AAUDIO_CHANNEL_MONO:
+ return AUDIO_CHANNEL_IN_MONO;
+ case AAUDIO_CHANNEL_STEREO:
+ return AUDIO_CHANNEL_IN_STEREO;
+ case AAUDIO_CHANNEL_FRONT_BACK:
+ return AUDIO_CHANNEL_IN_FRONT_BACK;
+ case AAUDIO_CHANNEL_2POINT0POINT2:
+ return AUDIO_CHANNEL_IN_2POINT0POINT2;
+ case AAUDIO_CHANNEL_2POINT1POINT2:
+ return AUDIO_CHANNEL_IN_2POINT1POINT2;
+ case AAUDIO_CHANNEL_3POINT0POINT2:
+ return AUDIO_CHANNEL_IN_3POINT0POINT2;
+ case AAUDIO_CHANNEL_3POINT1POINT2:
+ return AUDIO_CHANNEL_IN_3POINT1POINT2;
+ case AAUDIO_CHANNEL_5POINT1:
+ return AUDIO_CHANNEL_IN_5POINT1;
+ default:
+ ALOGE("%s() %#x unrecognized", __func__, channelMask);
+ return AUDIO_CHANNEL_INVALID;
+ }
+ } else {
+ switch (channelMask) {
+ case AAUDIO_CHANNEL_MONO:
+ return AUDIO_CHANNEL_OUT_MONO;
+ case AAUDIO_CHANNEL_STEREO:
+ return AUDIO_CHANNEL_OUT_STEREO;
+ case AAUDIO_CHANNEL_2POINT1:
+ return AUDIO_CHANNEL_OUT_2POINT1;
+ case AAUDIO_CHANNEL_TRI:
+ return AUDIO_CHANNEL_OUT_TRI;
+ case AAUDIO_CHANNEL_TRI_BACK:
+ return AUDIO_CHANNEL_OUT_TRI_BACK;
+ case AAUDIO_CHANNEL_3POINT1:
+ return AUDIO_CHANNEL_OUT_3POINT1;
+ case AAUDIO_CHANNEL_2POINT0POINT2:
+ return AUDIO_CHANNEL_OUT_2POINT0POINT2;
+ case AAUDIO_CHANNEL_2POINT1POINT2:
+ return AUDIO_CHANNEL_OUT_2POINT1POINT2;
+ case AAUDIO_CHANNEL_3POINT0POINT2:
+ return AUDIO_CHANNEL_OUT_3POINT0POINT2;
+ case AAUDIO_CHANNEL_3POINT1POINT2:
+ return AUDIO_CHANNEL_OUT_3POINT1POINT2;
+ case AAUDIO_CHANNEL_QUAD:
+ return AUDIO_CHANNEL_OUT_QUAD;
+ case AAUDIO_CHANNEL_QUAD_SIDE:
+ return AUDIO_CHANNEL_OUT_QUAD_SIDE;
+ case AAUDIO_CHANNEL_SURROUND:
+ return AUDIO_CHANNEL_OUT_SURROUND;
+ case AAUDIO_CHANNEL_PENTA:
+ return AUDIO_CHANNEL_OUT_PENTA;
+ case AAUDIO_CHANNEL_5POINT1:
+ return AUDIO_CHANNEL_OUT_5POINT1;
+ case AAUDIO_CHANNEL_5POINT1_SIDE:
+ return AUDIO_CHANNEL_OUT_5POINT1_SIDE;
+ case AAUDIO_CHANNEL_5POINT1POINT2:
+ return AUDIO_CHANNEL_OUT_5POINT1POINT2;
+ case AAUDIO_CHANNEL_5POINT1POINT4:
+ return AUDIO_CHANNEL_OUT_5POINT1POINT4;
+ case AAUDIO_CHANNEL_6POINT1:
+ return AUDIO_CHANNEL_OUT_6POINT1;
+ case AAUDIO_CHANNEL_7POINT1:
+ return AUDIO_CHANNEL_OUT_7POINT1;
+ case AAUDIO_CHANNEL_7POINT1POINT2:
+ return AUDIO_CHANNEL_OUT_7POINT1POINT2;
+ case AAUDIO_CHANNEL_7POINT1POINT4:
+ return AUDIO_CHANNEL_OUT_7POINT1POINT4;
+ // TODO: add 9point1point4 and 9point1point6 when they are added in audio-hal-enums.h
+ // case AAUDIO_CHANNEL_9POINT1POINT4:
+ // return AUDIO_CHANNEL_OUT_9POINT1POINT4;
+ // case AAUDIO_CHANNEL_9POINT1POINT6:
+ // return AUDIO_CHANNEL_OUT_9POINT1POINT6;
+ default:
+ ALOGE("%s() %#x unrecognized", __func__, channelMask);
+ return AUDIO_CHANNEL_INVALID;
+ }
+ }
+}
+
+aaudio_channel_mask_t AAudioConvert_androidToAAudioChannelLayoutMask(
+ audio_channel_mask_t channelMask, bool isInput) {
+ if (isInput) {
+ switch (channelMask) {
+ case AUDIO_CHANNEL_IN_MONO:
+ return AAUDIO_CHANNEL_MONO;
+ case AUDIO_CHANNEL_IN_STEREO:
+ return AAUDIO_CHANNEL_STEREO;
+ case AUDIO_CHANNEL_IN_FRONT_BACK:
+ return AAUDIO_CHANNEL_FRONT_BACK;
+ case AUDIO_CHANNEL_IN_2POINT0POINT2:
+ return AAUDIO_CHANNEL_2POINT0POINT2;
+ case AUDIO_CHANNEL_IN_2POINT1POINT2:
+ return AAUDIO_CHANNEL_2POINT1POINT2;
+ case AUDIO_CHANNEL_IN_3POINT0POINT2:
+ return AAUDIO_CHANNEL_3POINT0POINT2;
+ case AUDIO_CHANNEL_IN_3POINT1POINT2:
+ return AAUDIO_CHANNEL_3POINT1POINT2;
+ case AUDIO_CHANNEL_IN_5POINT1:
+ return AAUDIO_CHANNEL_5POINT1;
+ default:
+ ALOGE("%s() %#x unrecognized", __func__, channelMask);
+ return AAUDIO_CHANNEL_INVALID;
+ }
+ } else {
+ switch (channelMask) {
+ case AUDIO_CHANNEL_OUT_MONO:
+ return AAUDIO_CHANNEL_MONO;
+ case AUDIO_CHANNEL_OUT_STEREO:
+ return AAUDIO_CHANNEL_STEREO;
+ case AUDIO_CHANNEL_OUT_2POINT1:
+ return AAUDIO_CHANNEL_2POINT1;
+ case AUDIO_CHANNEL_OUT_TRI:
+ return AAUDIO_CHANNEL_TRI;
+ case AUDIO_CHANNEL_OUT_TRI_BACK:
+ return AAUDIO_CHANNEL_TRI_BACK;
+ case AUDIO_CHANNEL_OUT_3POINT1:
+ return AAUDIO_CHANNEL_3POINT1;
+ case AUDIO_CHANNEL_OUT_2POINT0POINT2:
+ return AAUDIO_CHANNEL_2POINT0POINT2;
+ case AUDIO_CHANNEL_OUT_2POINT1POINT2:
+ return AAUDIO_CHANNEL_2POINT1POINT2;
+ case AUDIO_CHANNEL_OUT_3POINT0POINT2:
+ return AAUDIO_CHANNEL_3POINT0POINT2;
+ case AUDIO_CHANNEL_OUT_3POINT1POINT2:
+ return AAUDIO_CHANNEL_3POINT1POINT2;
+ case AUDIO_CHANNEL_OUT_QUAD:
+ return AAUDIO_CHANNEL_QUAD;
+ case AUDIO_CHANNEL_OUT_QUAD_SIDE:
+ return AAUDIO_CHANNEL_QUAD_SIDE;
+ case AUDIO_CHANNEL_OUT_SURROUND:
+ return AAUDIO_CHANNEL_SURROUND;
+ case AUDIO_CHANNEL_OUT_PENTA:
+ return AAUDIO_CHANNEL_PENTA;
+ case AUDIO_CHANNEL_OUT_5POINT1:
+ return AAUDIO_CHANNEL_5POINT1;
+ case AUDIO_CHANNEL_OUT_5POINT1_SIDE:
+ return AAUDIO_CHANNEL_5POINT1_SIDE;
+ case AUDIO_CHANNEL_OUT_5POINT1POINT2:
+ return AAUDIO_CHANNEL_5POINT1POINT2;
+ case AUDIO_CHANNEL_OUT_5POINT1POINT4:
+ return AAUDIO_CHANNEL_5POINT1POINT4;
+ case AUDIO_CHANNEL_OUT_6POINT1:
+ return AAUDIO_CHANNEL_6POINT1;
+ case AUDIO_CHANNEL_OUT_7POINT1:
+ return AAUDIO_CHANNEL_7POINT1;
+ case AUDIO_CHANNEL_OUT_7POINT1POINT2:
+ return AAUDIO_CHANNEL_7POINT1POINT2;
+ case AUDIO_CHANNEL_OUT_7POINT1POINT4:
+ return AAUDIO_CHANNEL_7POINT1POINT4;
+ // TODO: add 9point1point4 and 9point1point6 when they are added in audio-hal-enums.h
+ // case AUDIO_CHANNEL_OUT_9POINT1POINT4:
+ // return AAUDIO_CHANNEL_9POINT1POINT4;
+ // case AUDIO_CHANNEL_OUT_9POINT1POINT6:
+ // return AAUDIO_CHANNEL_9POINT1POINT6;
+ default:
+ ALOGE("%s() %#x unrecognized", __func__, channelMask);
+ return AAUDIO_CHANNEL_INVALID;
+ }
+ }
+}
+
+int32_t AAudioConvert_channelMaskToCount(aaudio_channel_mask_t channelMask) {
+ return __builtin_popcount(channelMask & ~AAUDIO_CHANNEL_BIT_INDEX);
+}
+
+aaudio_channel_mask_t AAudioConvert_channelCountToMask(int32_t channelCount) {
+ if (channelCount < 0 || channelCount > AUDIO_CHANNEL_COUNT_MAX) {
+ return AAUDIO_CHANNEL_INVALID;
+ }
+
+ if (channelCount == 0) {
+ return AAUDIO_UNSPECIFIED;
+ }
+
+ // Return index mask if the channel count is greater than 2.
+ return AAUDIO_CHANNEL_BIT_INDEX | ((1 << channelCount) - 1);
+}
+
+aaudio_channel_mask_t AAudioConvert_androidToAAudioChannelIndexMask(
+ audio_channel_mask_t channelMask) {
+ if (audio_channel_mask_get_representation(channelMask) != AUDIO_CHANNEL_REPRESENTATION_INDEX) {
+ ALOGE("%s() %#x not an index mask", __func__, channelMask);
+ return AAUDIO_CHANNEL_INVALID;
+ }
+ return (channelMask & ~AUDIO_CHANNEL_INDEX_HDR) | AAUDIO_CHANNEL_BIT_INDEX;
+}
+
+audio_channel_mask_t AAudioConvert_aaudioToAndroidChannelIndexMask(
+ aaudio_channel_mask_t channelMask) {
+ if (!AAudio_isChannelIndexMask(channelMask)) {
+ ALOGE("%s() %#x not an index mask", __func__, channelMask);
+ return AUDIO_CHANNEL_INVALID;
+ }
+ return audio_channel_mask_for_index_assignment_from_count(
+ AAudioConvert_channelMaskToCount(channelMask));
+}
+
+aaudio_channel_mask_t AAudioConvert_androidToAAudioChannelMask(
+ audio_channel_mask_t channelMask, bool isInput, bool indexMaskRequired) {
+ if (audio_channel_mask_get_representation(channelMask) == AUDIO_CHANNEL_REPRESENTATION_INDEX) {
+ return AAudioConvert_androidToAAudioChannelIndexMask(channelMask);
+ }
+ if (indexMaskRequired) {
+ // Require index mask, `channelMask` here is a position mask.
+ const int channelCount = isInput ? audio_channel_count_from_in_mask(channelMask)
+ : audio_channel_count_from_out_mask(channelMask);
+ return AAudioConvert_channelCountToMask(channelCount);
+ }
+ return AAudioConvert_androidToAAudioChannelLayoutMask(channelMask, isInput);
+}
+
+audio_channel_mask_t AAudioConvert_aaudioToAndroidChannelMask(
+ aaudio_channel_mask_t channelMask, bool isInput) {
+ return AAudio_isChannelIndexMask(channelMask)
+ ? AAudioConvert_aaudioToAndroidChannelIndexMask(channelMask)
+ : AAudioConvert_aaudioToAndroidChannelLayoutMask(channelMask, isInput);
+}
+
+bool AAudio_isChannelIndexMask(aaudio_channel_mask_t channelMask) {
+ return (channelMask & AAUDIO_CHANNEL_BIT_INDEX) == AAUDIO_CHANNEL_BIT_INDEX;
+}
+
+audio_channel_mask_t AAudio_getChannelMaskForOpen(
+ aaudio_channel_mask_t channelMask, int32_t samplesPerFrame, bool isInput) {
+ if (channelMask != AAUDIO_UNSPECIFIED) {
+ if (AAudio_isChannelIndexMask(channelMask) && samplesPerFrame <= 2) {
+ // When it is index mask and the count is less than 3, use position mask
+ // instead of index mask for opening a stream. This may need to be revisited
+ // when making channel index mask public.
+ return isInput ? audio_channel_in_mask_from_count(samplesPerFrame)
+ : audio_channel_out_mask_from_count(samplesPerFrame);
+ }
+ return AAudioConvert_aaudioToAndroidChannelMask(channelMask, isInput);
+ }
+
+ // Return stereo when unspecified.
+ return isInput ? AUDIO_CHANNEL_IN_STEREO : AUDIO_CHANNEL_OUT_STEREO;
+}
+
int32_t AAudioConvert_framesToBytes(int32_t numFrames,
int32_t bytesPerFrame,
int32_t *sizeInBytes) {
diff --git a/media/libaaudio/src/utility/AAudioUtilities.h b/media/libaaudio/src/utility/AAudioUtilities.h
index 82eb77d..5eda30c 100644
--- a/media/libaaudio/src/utility/AAudioUtilities.h
+++ b/media/libaaudio/src/utility/AAudioUtilities.h
@@ -96,6 +96,33 @@
audio_flags_mask_t AAudioConvert_privacySensitiveToAudioFlagsMask(
bool privacySensitive);
+audio_channel_mask_t AAudioConvert_aaudioToAndroidChannelLayoutMask(
+ aaudio_channel_mask_t channelMask, bool isInput);
+
+aaudio_channel_mask_t AAudioConvert_androidToAAudioChannelLayoutMask(
+ audio_channel_mask_t channelMask, bool isInput);
+
+aaudio_channel_mask_t AAudioConvert_androidToAAudioChannelIndexMask(
+ audio_channel_mask_t channelMask);
+
+audio_channel_mask_t AAudioConvert_aaudioToAndroidChannelIndexMask(
+ aaudio_channel_mask_t channelMask);
+
+aaudio_channel_mask_t AAudioConvert_androidToAAudioChannelMask(
+ audio_channel_mask_t channelMask, bool isInput, bool indexMaskRequired);
+
+audio_channel_mask_t AAudioConvert_aaudioToAndroidChannelMask(
+ aaudio_channel_mask_t channelMask, bool isInput);
+
+bool AAudio_isChannelIndexMask(aaudio_channel_mask_t channelMask);
+
+int32_t AAudioConvert_channelMaskToCount(aaudio_channel_mask_t channelMask);
+
+aaudio_channel_mask_t AAudioConvert_channelCountToMask(int32_t channelCount);
+
+audio_channel_mask_t AAudio_getChannelMaskForOpen(
+ aaudio_channel_mask_t channelMask, int32_t samplesPerFrame, bool isInput);
+
// Note that this code may be replaced by Settings or by some other system configuration tool.
/**
@@ -318,4 +345,36 @@
std::atomic<int> mRequested{0};
std::atomic<int> mAcknowledged{0};
};
+
+enum {
+ /**
+ * Audio channel index mask, only used internally.
+ */
+ AAUDIO_CHANNEL_BIT_INDEX = 0x80000000,
+ AAUDIO_CHANNEL_INDEX_MASK_1 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 1) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_2 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 2) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_3 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 3) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_4 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 4) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_5 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 5) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_6 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 6) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_7 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 7) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_8 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 8) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_9 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 9) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_10 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 10) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_11 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 11) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_12 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 12) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_13 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 13) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_14 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 14) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_15 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 15) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_16 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 16) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_17 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 17) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_18 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 18) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_19 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 19) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_20 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 20) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_21 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 21) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_22 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 22) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_23 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 23) - 1,
+ AAUDIO_CHANNEL_INDEX_MASK_24 = AAUDIO_CHANNEL_BIT_INDEX | (1 << 24) - 1,
+};
+
#endif //UTILITY_AAUDIO_UTILITIES_H
diff --git a/media/libaudioclient/AidlConversion.cpp b/media/libaudioclient/AidlConversion.cpp
index 321e7f9..acab774 100644
--- a/media/libaudioclient/AidlConversion.cpp
+++ b/media/libaudioclient/AidlConversion.cpp
@@ -1361,6 +1361,10 @@
return AUDIO_FLAG_NO_SYSTEM_CAPTURE;
case media::AudioFlag::CAPTURE_PRIVATE:
return AUDIO_FLAG_CAPTURE_PRIVATE;
+ case media::AudioFlag::CONTENT_SPATIALIZED:
+ return AUDIO_FLAG_CONTENT_SPATIALIZED;
+ case media::AudioFlag::NEVER_SPATIALIZE:
+ return AUDIO_FLAG_NEVER_SPATIALIZE;
}
return unexpected(BAD_VALUE);
}
@@ -1398,6 +1402,10 @@
return media::AudioFlag::NO_SYSTEM_CAPTURE;
case AUDIO_FLAG_CAPTURE_PRIVATE:
return media::AudioFlag::CAPTURE_PRIVATE;
+ case AUDIO_FLAG_CONTENT_SPATIALIZED:
+ return media::AudioFlag::CONTENT_SPATIALIZED;
+ case AUDIO_FLAG_NEVER_SPATIALIZE:
+ return media::AudioFlag::NEVER_SPATIALIZE;
}
return unexpected(BAD_VALUE);
}
diff --git a/media/libaudioclient/Android.bp b/media/libaudioclient/Android.bp
index 9c307ff..a1c1234 100644
--- a/media/libaudioclient/Android.bp
+++ b/media/libaudioclient/Android.bp
@@ -25,11 +25,13 @@
static_libs: [
"audioflinger-aidl-cpp",
"audiopolicy-aidl-cpp",
+ "spatializer-aidl-cpp",
"av-types-aidl-cpp",
],
export_static_lib_headers: [
"audioflinger-aidl-cpp",
"audiopolicy-aidl-cpp",
+ "spatializer-aidl-cpp",
"av-types-aidl-cpp",
],
target: {
@@ -112,6 +114,7 @@
"audioclient-types-aidl-cpp",
"audioflinger-aidl-cpp",
"audiopolicy-aidl-cpp",
+ "spatializer-aidl-cpp",
"audiopolicy-types-aidl-cpp",
"av-types-aidl-cpp",
"capture_state_listener-aidl-cpp",
@@ -137,6 +140,7 @@
export_shared_lib_headers: [
"audioflinger-aidl-cpp",
"audiopolicy-aidl-cpp",
+ "spatializer-aidl-cpp",
"framework-permission-aidl-cpp",
"libbinder",
],
@@ -389,6 +393,8 @@
"aidl/android/media/AudioVolumeGroup.aidl",
"aidl/android/media/DeviceRole.aidl",
"aidl/android/media/SoundTriggerSession.aidl",
+ "aidl/android/media/SpatializationLevel.aidl",
+ "aidl/android/media/SpatializerHeadTrackingMode.aidl",
],
imports: [
"audio_common-aidl",
@@ -459,6 +465,7 @@
srcs: [
"aidl/android/media/GetInputForAttrResponse.aidl",
"aidl/android/media/GetOutputForAttrResponse.aidl",
+ "aidl/android/media/GetSpatializerResponse.aidl",
"aidl/android/media/Int.aidl",
"aidl/android/media/RecordClientInfo.aidl",
"aidl/android/media/IAudioPolicyService.aidl",
@@ -470,6 +477,33 @@
"audiopolicy-types-aidl",
"capture_state_listener-aidl",
"framework-permission-aidl",
+ "spatializer-aidl",
+ ],
+
+ double_loadable: true,
+ backend: {
+ cpp: {
+ min_sdk_version: "29",
+ apex_available: [
+ "//apex_available:platform",
+ "com.android.media",
+ ],
+ },
+ },
+}
+
+aidl_interface {
+ name: "spatializer-aidl",
+ unstable: true,
+ local_include_dir: "aidl",
+ host_supported: true,
+ vendor_available: true,
+ srcs: [
+ "aidl/android/media/INativeSpatializerCallback.aidl",
+ "aidl/android/media/ISpatializer.aidl",
+ ],
+ imports: [
+ "audiopolicy-types-aidl",
],
double_loadable: true,
diff --git a/media/libaudioclient/AudioEffect.cpp b/media/libaudioclient/AudioEffect.cpp
index 6ad5483..9091599 100644
--- a/media/libaudioclient/AudioEffect.cpp
+++ b/media/libaudioclient/AudioEffect.cpp
@@ -70,7 +70,8 @@
audio_session_t sessionId,
audio_io_handle_t io,
const AudioDeviceTypeAddr& device,
- bool probe)
+ bool probe,
+ bool notifyFramesProcessed)
{
sp<media::IEffect> iEffect;
sp<IMemory> cblk;
@@ -124,6 +125,7 @@
request.device = VALUE_OR_RETURN_STATUS(legacy2aidl_AudioDeviceTypeAddress(device));
request.attributionSource = mClientAttributionSource;
request.probe = probe;
+ request.notifyFramesProcessed = notifyFramesProcessed;
media::CreateEffectResponse response;
@@ -194,7 +196,8 @@
audio_session_t sessionId,
audio_io_handle_t io,
const AudioDeviceTypeAddr& device,
- bool probe)
+ bool probe,
+ bool notifyFramesProcessed)
{
effect_uuid_t type;
effect_uuid_t *pType = nullptr;
@@ -211,7 +214,8 @@
pUuid = &uuid;
}
- return set(pType, pUuid, priority, cbf, user, sessionId, io, device, probe);
+ return set(pType, pUuid, priority, cbf, user, sessionId, io,
+ device, probe, notifyFramesProcessed);
}
@@ -522,6 +526,13 @@
}
}
+void AudioEffect::framesProcessed(int32_t frames)
+{
+ if (mCbf != NULL) {
+ mCbf(EVENT_FRAMES_PROCESSED, mUserData, &frames);
+ }
+}
+
// -------------------------------------------------------------------------
status_t AudioEffect::queryNumberEffects(uint32_t *numEffects)
diff --git a/media/libaudioclient/AudioSystem.cpp b/media/libaudioclient/AudioSystem.cpp
index f1eeaa3..c7967e5 100644
--- a/media/libaudioclient/AudioSystem.cpp
+++ b/media/libaudioclient/AudioSystem.cpp
@@ -480,6 +480,12 @@
return af->systemReady();
}
+status_t AudioSystem::audioPolicyReady() {
+ const sp<IAudioFlinger>& af = AudioSystem::get_audio_flinger();
+ if (af == 0) return NO_INIT;
+ return af->audioPolicyReady();
+}
+
status_t AudioSystem::getFrameCountHAL(audio_io_handle_t ioHandle,
size_t* frameCount) {
const sp<IAudioFlinger>& af = AudioSystem::get_audio_flinger();
@@ -2234,6 +2240,47 @@
return OK;
}
+status_t AudioSystem::getSpatializer(const sp<media::INativeSpatializerCallback>& callback,
+ sp<media::ISpatializer>* spatializer) {
+ const sp<IAudioPolicyService>& aps = AudioSystem::get_audio_policy_service();
+ if (spatializer == nullptr) {
+ return BAD_VALUE;
+ }
+ if (aps == 0) {
+ return PERMISSION_DENIED;
+ }
+ media::GetSpatializerResponse response;
+ RETURN_STATUS_IF_ERROR(statusTFromBinderStatus(
+ aps->getSpatializer(callback, &response)));
+
+ *spatializer = response.spatializer;
+ return OK;
+}
+
+status_t AudioSystem::canBeSpatialized(const audio_attributes_t *attr,
+ const audio_config_t *config,
+ const AudioDeviceTypeAddrVector &devices,
+ bool *canBeSpatialized) {
+ const sp<IAudioPolicyService>& aps = AudioSystem::get_audio_policy_service();
+ if (aps == 0) {
+ return PERMISSION_DENIED;
+ }
+ audio_attributes_t attributes = attr != nullptr ? *attr : AUDIO_ATTRIBUTES_INITIALIZER;
+ audio_config_t configuration = config != nullptr ? *config : AUDIO_CONFIG_INITIALIZER;
+
+ std::optional<media::AudioAttributesInternal> attrAidl = VALUE_OR_RETURN_STATUS(
+ legacy2aidl_audio_attributes_t_AudioAttributesInternal(attributes));
+ std::optional<media::AudioConfig> configAidl = VALUE_OR_RETURN_STATUS(
+ legacy2aidl_audio_config_t_AudioConfig(configuration));
+ std::vector<media::AudioDevice> devicesAidl = VALUE_OR_RETURN_STATUS(
+ convertContainer<std::vector<media::AudioDevice>>(devices,
+ legacy2aidl_AudioDeviceTypeAddress));
+ RETURN_STATUS_IF_ERROR(statusTFromBinderStatus(
+ aps->canBeSpatialized(attrAidl, configAidl, devicesAidl, canBeSpatialized)));
+ return OK;
+}
+
+
class CaptureStateListenerImpl : public media::BnCaptureStateListener,
public IBinder::DeathRecipient {
public:
diff --git a/media/libaudioclient/IAudioFlinger.cpp b/media/libaudioclient/IAudioFlinger.cpp
index cae81f0..2af1c50 100644
--- a/media/libaudioclient/IAudioFlinger.cpp
+++ b/media/libaudioclient/IAudioFlinger.cpp
@@ -715,6 +715,10 @@
return statusTFromBinderStatus(mDelegate->systemReady());
}
+status_t AudioFlingerClientAdapter::audioPolicyReady() {
+ return statusTFromBinderStatus(mDelegate->audioPolicyReady());
+}
+
size_t AudioFlingerClientAdapter::frameCountHAL(audio_io_handle_t ioHandle) const {
auto result = [&]() -> ConversionResult<size_t> {
int32_t ioHandleAidl = VALUE_OR_RETURN(legacy2aidl_audio_io_handle_t_int32_t(ioHandle));
@@ -1189,6 +1193,11 @@
return Status::fromStatusT(mDelegate->systemReady());
}
+Status AudioFlingerServerAdapter::audioPolicyReady() {
+ mDelegate->audioPolicyReady();
+ return Status::ok();
+}
+
Status AudioFlingerServerAdapter::frameCountHAL(int32_t ioHandle, int64_t* _aidl_return) {
audio_io_handle_t ioHandleLegacy = VALUE_OR_RETURN_BINDER(
aidl2legacy_int32_t_audio_io_handle_t(ioHandle));
diff --git a/media/libaudioclient/aidl/android/media/AudioFlag.aidl b/media/libaudioclient/aidl/android/media/AudioFlag.aidl
index 58b493b..91361fb 100644
--- a/media/libaudioclient/aidl/android/media/AudioFlag.aidl
+++ b/media/libaudioclient/aidl/android/media/AudioFlag.aidl
@@ -34,4 +34,6 @@
MUTE_HAPTIC = 11,
NO_SYSTEM_CAPTURE = 12,
CAPTURE_PRIVATE = 13,
+ CONTENT_SPATIALIZED = 14,
+ NEVER_SPATIALIZE = 15,
}
diff --git a/media/libaudioclient/aidl/android/media/AudioVibratorInfo.aidl b/media/libaudioclient/aidl/android/media/AudioVibratorInfo.aidl
index f88fc3c..8538d8a 100644
--- a/media/libaudioclient/aidl/android/media/AudioVibratorInfo.aidl
+++ b/media/libaudioclient/aidl/android/media/AudioVibratorInfo.aidl
@@ -24,4 +24,5 @@
int id;
float resonantFrequency;
float qFactor;
+ float maxAmplitude;
}
diff --git a/media/libaudioclient/aidl/android/media/CreateEffectRequest.aidl b/media/libaudioclient/aidl/android/media/CreateEffectRequest.aidl
index 2d274f4..35a56eb 100644
--- a/media/libaudioclient/aidl/android/media/CreateEffectRequest.aidl
+++ b/media/libaudioclient/aidl/android/media/CreateEffectRequest.aidl
@@ -37,4 +37,6 @@
AudioDevice device;
AttributionSourceState attributionSource;
boolean probe;
+ /** true if a callback must be sent each time audio frames are processed */
+ boolean notifyFramesProcessed;
}
diff --git a/media/libaudioclient/aidl/android/media/GetSpatializerResponse.aidl b/media/libaudioclient/aidl/android/media/GetSpatializerResponse.aidl
new file mode 100644
index 0000000..25115ac
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/GetSpatializerResponse.aidl
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+import android.media.ISpatializer;
+
+/**
+ * Used as a return value for IAudioPolicyService.getSpatializer() method
+ * {@hide}
+ */
+ parcelable GetSpatializerResponse {
+ /* The ISpatializer interface if successful, null if not */
+ @nullable ISpatializer spatializer;
+}
diff --git a/media/libaudioclient/aidl/android/media/IAudioFlingerService.aidl b/media/libaudioclient/aidl/android/media/IAudioFlingerService.aidl
index d2cae6d..7ffcc33 100644
--- a/media/libaudioclient/aidl/android/media/IAudioFlingerService.aidl
+++ b/media/libaudioclient/aidl/android/media/IAudioFlingerService.aidl
@@ -197,6 +197,9 @@
/* Indicate JAVA services are ready (scheduling, power management ...) */
oneway void systemReady();
+ /* Indicate audio policy service is ready */
+ oneway void audioPolicyReady();
+
// Returns the number of frames per audio HAL buffer.
long frameCountHAL(int /* audio_io_handle_t */ ioHandle);
diff --git a/media/libaudioclient/aidl/android/media/IAudioPolicyService.aidl b/media/libaudioclient/aidl/android/media/IAudioPolicyService.aidl
index 4c3955a..5f0a1de 100644
--- a/media/libaudioclient/aidl/android/media/IAudioPolicyService.aidl
+++ b/media/libaudioclient/aidl/android/media/IAudioPolicyService.aidl
@@ -47,8 +47,10 @@
import android.media.EffectDescriptor;
import android.media.GetInputForAttrResponse;
import android.media.GetOutputForAttrResponse;
+import android.media.GetSpatializerResponse;
import android.media.IAudioPolicyServiceClient;
import android.media.ICaptureStateListener;
+import android.media.INativeSpatializerCallback;
import android.media.Int;
import android.media.SoundTriggerSession;
@@ -348,4 +350,29 @@
DeviceRole role);
boolean registerSoundTriggerCaptureStateListener(ICaptureStateListener listener);
+
+ /** If a spatializer stage effect is present on the platform, this will return an
+ * ISpatializer interface (see GetSpatializerResponse,aidl) to control this
+ * feature.
+ * If no spatializer stage is present, a null interface is returned.
+ * The INativeSpatializerCallback passed must not be null.
+ * Only one ISpatializer interface can exist at a given time. The native audio policy
+ * service will reject the request if an interface was already acquired and previous owner
+ * did not die or call ISpatializer.release().
+ */
+ GetSpatializerResponse getSpatializer(INativeSpatializerCallback callback);
+
+ /** Queries if some kind of spatialization will be performed if the audio playback context
+ * described by the provided arguments is present.
+ * The context is made of:
+ * - The audio attributes describing the playback use case.
+ * - The audio configuration describing the audio format, channels, sampling rate...
+ * - The devices describing the sink audio device selected for playback.
+ * All arguments are optional and only the specified arguments are used to match against
+ * supported criteria. For instance, supplying no argument will tell if spatialization is
+ * supported or not in general.
+ */
+ boolean canBeSpatialized(in @nullable AudioAttributesInternal attr,
+ in @nullable AudioConfig config,
+ in AudioDevice[] devices);
}
diff --git a/media/libaudioclient/aidl/android/media/IEffectClient.aidl b/media/libaudioclient/aidl/android/media/IEffectClient.aidl
index 3b6bcf1..37b442d 100644
--- a/media/libaudioclient/aidl/android/media/IEffectClient.aidl
+++ b/media/libaudioclient/aidl/android/media/IEffectClient.aidl
@@ -43,4 +43,10 @@
* TODO(ytai): replace opaque byte arrays with strongly typed parameters.
*/
oneway void commandExecuted(int cmdCode, in byte[] cmdData, in byte[] replyData);
+
+ /**
+ * Called whenever audio frames have been processed by the effect engine.
+ * @param frames number of frames processed.
+ */
+ oneway void framesProcessed(int frames);
}
diff --git a/media/libaudioclient/aidl/android/media/INativeSpatializerCallback.aidl b/media/libaudioclient/aidl/android/media/INativeSpatializerCallback.aidl
new file mode 100644
index 0000000..f34df05
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/INativeSpatializerCallback.aidl
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+import android.media.SpatializationLevel;
+
+/**
+ * The INativeSpatializerCallback interface is a callback associated to the
+ * ISpatializer interface. The callback is used by the spatializer stage
+ * implementation in native audio server to communicate stage changes to the
+ * client controlling the spatializer with the ISpatializer interface.
+ * {@hide}
+ */
+interface INativeSpatializerCallback {
+ /** Called when the spatialization level applied by the vitualizer stage changes
+ * (e.g. when the spatializer is enabled or disabled)
+ */
+ oneway void onLevelChanged(SpatializationLevel level);
+}
diff --git a/media/libaudioclient/aidl/android/media/ISpatializer.aidl b/media/libaudioclient/aidl/android/media/ISpatializer.aidl
new file mode 100644
index 0000000..3c2eda6
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/ISpatializer.aidl
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+import android.media.SpatializationLevel;
+import android.media.SpatializerHeadTrackingMode;
+
+/**
+ * The ISpatializer interface is used to control the native audio service implementation
+ * of the spatializer stage with headtracking when present on a platform.
+ * It is intended for exclusive use by the java AudioService running in system_server.
+ * It provides APIs to discover the feature availability and options as well as control and report
+ * the active state and modes of the spatializer and head tracking effect.
+ * {@hide}
+ */
+interface ISpatializer {
+ /** Releases a ISpatializer interface previously acquired. */
+ void release();
+
+ /** Reports the list of supported spatialization levels (see SpatializationLevel.aidl).
+ * The list should never be empty if an ISpatializer interface was successfully
+ * retrieved with IAudioPolicyService.getSpatializer().
+ */
+ SpatializationLevel[] getSupportedLevels();
+
+ /** Selects the desired spatialization level (see SpatializationLevel.aidl). Selecting a level
+ * different from SpatializationLevel.NONE with create the specialized multichannel output
+ * mixer, create and enable the spatializer effect and let the audio policy attach eligible
+ * AudioTrack to this output stream.
+ */
+ void setLevel(SpatializationLevel level);
+
+ /** Gets the selected spatialization level (see SpatializationLevel.aidl) */
+ SpatializationLevel getLevel();
+
+ /** Reports the list of supported head tracking modes (see SpatializerHeadTrackingMode.aidl).
+ * The list can be empty if the spatializer implementation does not support head tracking or if
+ * no head tracking device is connected.
+ */
+ SpatializerHeadTrackingMode[] getSupportedHeadTrackingModes();
+
+ /** Selects the desired head tracking mode (see SpatializerHeadTrackingMode.aidl) */
+ void setDesiredHeadTrackingMode(SpatializerHeadTrackingMode mode);
+
+ /** Gets the actual head tracking mode. Can be different from the desired mode if conditions to
+ * enable the desired mode are not met (e.g if the head tracking device was removed)
+ */
+ SpatializerHeadTrackingMode getActualHeadTrackingMode();
+
+ /** Reset the head tracking algorithm to consider current head pose as neutral */
+ void recenterHeadTracker();
+
+ /** Set the screen to stage transform to use by the head tracking algorithm */
+ void setGlobalTransform(in float[] screenToStage);
+}
diff --git a/media/libaudioclient/aidl/android/media/OpenOutputRequest.aidl b/media/libaudioclient/aidl/android/media/OpenOutputRequest.aidl
index 06b12e9..1541948 100644
--- a/media/libaudioclient/aidl/android/media/OpenOutputRequest.aidl
+++ b/media/libaudioclient/aidl/android/media/OpenOutputRequest.aidl
@@ -17,6 +17,7 @@
package android.media;
import android.media.AudioConfig;
+import android.media.AudioConfigBase;
import android.media.AudioPort;
/**
@@ -25,7 +26,8 @@
parcelable OpenOutputRequest {
/** Interpreted as audio_module_handle_t. */
int module;
- AudioConfig config;
+ AudioConfig halConfig;
+ AudioConfigBase mixerConfig;
/** Type must be DEVICE. */
AudioPort device;
/** Bitmask, indexed by AudioOutputFlag. */
diff --git a/media/libaudioclient/aidl/android/media/SpatializationLevel.aidl b/media/libaudioclient/aidl/android/media/SpatializationLevel.aidl
new file mode 100644
index 0000000..cef42bb
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/SpatializationLevel.aidl
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+/**
+ * The spatialization level or mode supported by the spatializer stage effect implementation.
+ * Used by methods of the ISpatializer interface.
+ * {@hide}
+ */
+@Backing(type="byte")
+enum SpatializationLevel {
+ /** Spatialization is disabled. */
+ NONE = 0,
+ /** The spatializer accepts audio with positional multichannel masks (e.g 5.1). */
+ SPATIALIZER_MULTICHANNEL = 1,
+ /** The spatializer accepts audio made of a channel bed of positional multichannels (e.g 5.1)
+ * and audio objects positioned independently via meta data.
+ */
+ SPATIALIZER_MCHAN_BED_PLUS_OBJECTS = 2,
+}
diff --git a/media/libaudioclient/aidl/android/media/SpatializerHeadTrackingMode.aidl b/media/libaudioclient/aidl/android/media/SpatializerHeadTrackingMode.aidl
new file mode 100644
index 0000000..58e0f61
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/SpatializerHeadTrackingMode.aidl
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+
+/**
+ * The head tracking mode supported by the spatializer effect implementation.
+ * Used by methods of the ISpatializer interface.
+ * {@hide}
+ */
+@Backing(type="byte")
+enum SpatializerHeadTrackingMode {
+ /** Head tracking is active in a mode not listed below (forward compatibility) */
+ OTHER = 0,
+ /** Head tracking is disabled */
+ DISABLED = 1,
+ /** Head tracking is performed relative to the real work environment */
+ RELATIVE_WORLD = 2,
+ /** Head tracking is performed relative to the device's screen */
+ RELATIVE_SCREEN = 3,
+}
diff --git a/media/libaudioclient/fuzzer/audioflinger_fuzzer.cpp b/media/libaudioclient/fuzzer/audioflinger_fuzzer.cpp
index d03c6fa..bd9e158 100644
--- a/media/libaudioclient/fuzzer/audioflinger_fuzzer.cpp
+++ b/media/libaudioclient/fuzzer/audioflinger_fuzzer.cpp
@@ -383,6 +383,9 @@
const std::vector<uint8_t> &replyData __unused) override {
return binder::Status::ok();
}
+ binder::Status framesProcessed(int32_t frames __unused) override {
+ return binder::Status::ok();
+ }
};
status_t AudioFlingerFuzzer::invokeAudioEffect() {
@@ -424,6 +427,7 @@
request.attributionSource.packageName = opPackageName;
request.attributionSource.pid = VALUE_OR_RETURN_STATUS(legacy2aidl_pid_t_int32_t(getpid()));
request.probe = false;
+ request.notifyFramesProcessed = false;
media::CreateEffectResponse response{};
status_t status = af->createEffect(request, &response);
@@ -648,11 +652,15 @@
sp<DeviceDescriptorBase> device = new DeviceDescriptorBase(getValue(&mFdp, kDevices));
audio_output_flags_t flags = getValue(&mFdp, kOutputFlags);
+ audio_config_base_t mixerConfig = AUDIO_CONFIG_BASE_INITIALIZER;
+
media::OpenOutputRequest request{};
media::OpenOutputResponse response{};
request.module = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_module_handle_t_int32_t(module));
- request.config = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_config_t_AudioConfig(config));
+ request.halConfig = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_config_t_AudioConfig(config));
+ request.mixerConfig =
+ VALUE_OR_RETURN_STATUS(legacy2aidl_audio_config_base_t_AudioConfigBase(mixerConfig));
request.device = VALUE_OR_RETURN_STATUS(legacy2aidl_DeviceDescriptorBase(device));
request.flags = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_output_flags_t_int32_t_mask(flags));
diff --git a/media/libaudioclient/include/media/AudioCommonTypes.h b/media/libaudioclient/include/media/AudioCommonTypes.h
index 5dfe5fc..5f0c590 100644
--- a/media/libaudioclient/include/media/AudioCommonTypes.h
+++ b/media/libaudioclient/include/media/AudioCommonTypes.h
@@ -41,6 +41,42 @@
return !(lhs==rhs);
}
+constexpr bool operator==(const audio_offload_info_t &lhs, const audio_offload_info_t &rhs)
+{
+ return lhs.version == rhs.version && lhs.size == rhs.size &&
+ lhs.sample_rate == rhs.sample_rate && lhs.channel_mask == rhs.channel_mask &&
+ lhs.format == rhs.format && lhs.stream_type == rhs.stream_type &&
+ lhs.bit_rate == rhs.bit_rate && lhs.duration_us == rhs.duration_us &&
+ lhs.has_video == rhs.has_video && lhs.is_streaming == rhs.is_streaming &&
+ lhs.bit_width == rhs.bit_width && lhs.offload_buffer_size == rhs.offload_buffer_size &&
+ lhs.usage == rhs.usage && lhs.encapsulation_mode == rhs.encapsulation_mode &&
+ lhs.content_id == rhs.content_id && lhs.sync_id == rhs.sync_id;
+}
+constexpr bool operator!=(const audio_offload_info_t &lhs, const audio_offload_info_t &rhs)
+{
+ return !(lhs==rhs);
+}
+
+constexpr bool operator==(const audio_config_t &lhs, const audio_config_t &rhs)
+{
+ return lhs.sample_rate == rhs.sample_rate && lhs.channel_mask == rhs.channel_mask &&
+ lhs.format == rhs.format && lhs.offload_info == rhs.offload_info;
+}
+constexpr bool operator!=(const audio_config_t &lhs, const audio_config_t &rhs)
+{
+ return !(lhs==rhs);
+}
+
+constexpr bool operator==(const audio_config_base_t &lhs, const audio_config_base_t &rhs)
+{
+ return lhs.sample_rate == rhs.sample_rate && lhs.channel_mask == rhs.channel_mask &&
+ lhs.format == rhs.format;
+}
+constexpr bool operator!=(const audio_config_base_t &lhs, const audio_config_base_t &rhs)
+{
+ return !(lhs==rhs);
+}
+
enum volume_group_t : uint32_t;
static const volume_group_t VOLUME_GROUP_NONE = static_cast<volume_group_t>(-1);
diff --git a/media/libaudioclient/include/media/AudioEffect.h b/media/libaudioclient/include/media/AudioEffect.h
index 3c19ec1..dd4d2da 100644
--- a/media/libaudioclient/include/media/AudioEffect.h
+++ b/media/libaudioclient/include/media/AudioEffect.h
@@ -283,7 +283,8 @@
EVENT_CONTROL_STATUS_CHANGED = 0,
EVENT_ENABLE_STATUS_CHANGED = 1,
EVENT_PARAMETER_CHANGED = 2,
- EVENT_ERROR = 3
+ EVENT_ERROR = 3,
+ EVENT_FRAMES_PROCESSED = 4,
};
/* Callback function notifying client application of a change in effect engine state or
@@ -389,7 +390,8 @@
audio_session_t sessionId = AUDIO_SESSION_OUTPUT_MIX,
audio_io_handle_t io = AUDIO_IO_HANDLE_NONE,
const AudioDeviceTypeAddr& device = {},
- bool probe = false);
+ bool probe = false,
+ bool notifyFramesProcessed = false);
/*
* Same as above but with type and uuid specified by character strings.
*/
@@ -401,7 +403,8 @@
audio_session_t sessionId = AUDIO_SESSION_OUTPUT_MIX,
audio_io_handle_t io = AUDIO_IO_HANDLE_NONE,
const AudioDeviceTypeAddr& device = {},
- bool probe = false);
+ bool probe = false,
+ bool notifyFramesProcessed = false);
/* Result of constructing the AudioEffect. This must be checked
* before using any AudioEffect API.
@@ -552,6 +555,7 @@
virtual void commandExecuted(int32_t cmdCode,
const std::vector<uint8_t>& cmdData,
const std::vector<uint8_t>& replyData);
+ virtual void framesProcessed(int32_t frames);
private:
@@ -587,6 +591,14 @@
}
return binder::Status::ok();
}
+ binder::Status framesProcessed(int32_t frames) override {
+ sp<AudioEffect> effect = mEffect.promote();
+ if (effect != 0) {
+ effect->framesProcessed(frames);
+ }
+ return binder::Status::ok();
+ }
+
// IBinder::DeathRecipient
virtual void binderDied(const wp<IBinder>& /*who*/) {
diff --git a/media/libaudioclient/include/media/AudioRecord.h b/media/libaudioclient/include/media/AudioRecord.h
index 326919a..f17ee3a 100644
--- a/media/libaudioclient/include/media/AudioRecord.h
+++ b/media/libaudioclient/include/media/AudioRecord.h
@@ -264,6 +264,7 @@
size_t frameCount() const { return mFrameCount; }
size_t frameSize() const { return mFrameSize; }
audio_source_t inputSource() const { return mAttributes.source; }
+ audio_channel_mask_t channelMask() const { return mChannelMask; }
/*
* Return the period of the notification callback in frames.
diff --git a/media/libaudioclient/include/media/AudioSystem.h b/media/libaudioclient/include/media/AudioSystem.h
index 8ba23ad..869bd6e 100644
--- a/media/libaudioclient/include/media/AudioSystem.h
+++ b/media/libaudioclient/include/media/AudioSystem.h
@@ -22,6 +22,8 @@
#include <android/media/AudioVibratorInfo.h>
#include <android/media/BnAudioFlingerClient.h>
#include <android/media/BnAudioPolicyServiceClient.h>
+#include <android/media/INativeSpatializerCallback.h>
+#include <android/media/ISpatializer.h>
#include <android/content/AttributionSourceState.h>
#include <media/AidlConversionUtil.h>
#include <media/AudioDeviceTypeAddr.h>
@@ -225,6 +227,9 @@
// Indicate JAVA services are ready (scheduling, power management ...)
static status_t systemReady();
+ // Indicate audio policy service is ready
+ static status_t audioPolicyReady();
+
// Returns the number of frames per audio HAL buffer.
// Corresponds to audio_stream->get_buffer_size()/audio_stream_in_frame_size() for input.
// See also getFrameCount().
@@ -485,6 +490,49 @@
static status_t getDeviceForStrategy(product_strategy_t strategy,
AudioDeviceTypeAddr &device);
+
+ /**
+ * If a spatializer stage effect is present on the platform, this will return an
+ * ISpatializer interface to control this feature.
+ * If no spatializer stage is present, a null interface is returned.
+ * The INativeSpatializerCallback passed must not be null.
+ * Only one ISpatializer interface can exist at a given time. The native audio policy
+ * service will reject the request if an interface was already acquired and previous owner
+ * did not die or call ISpatializer.release().
+ * @param callback in: the callback to receive state updates if the ISpatializer
+ * interface is acquired.
+ * @param spatializer out: the ISpatializer interface made available to control the
+ * platform spatializer
+ * @return NO_ERROR in case of success, DEAD_OBJECT, NO_INIT, PERMISSION_DENIED, BAD_VALUE
+ * in case of error.
+ */
+ static status_t getSpatializer(const sp<media::INativeSpatializerCallback>& callback,
+ sp<media::ISpatializer>* spatializer);
+
+ /**
+ * Queries if some kind of spatialization will be performed if the audio playback context
+ * described by the provided arguments is present.
+ * The context is made of:
+ * - The audio attributes describing the playback use case.
+ * - The audio configuration describing the audio format, channels, sampling rate ...
+ * - The devices describing the sink audio device selected for playback.
+ * All arguments are optional and only the specified arguments are used to match against
+ * supported criteria. For instance, supplying no argument will tell if spatialization is
+ * supported or not in general.
+ * @param attr audio attributes describing the playback use case
+ * @param config audio configuration describing the audio format, channels, sampling rate...
+ * @param devices the sink audio device selected for playback
+ * @param canBeSpatialized out: true if spatialization is enabled for this context,
+ * false otherwise
+ * @return NO_ERROR in case of success, DEAD_OBJECT, NO_INIT, BAD_VALUE
+ * in case of error.
+ */
+ static status_t canBeSpatialized(const audio_attributes_t *attr,
+ const audio_config_t *config,
+ const AudioDeviceTypeAddrVector &devices,
+ bool *canBeSpatialized);
+
+
// A listener for capture state changes.
class CaptureStateListener : public RefBase {
public:
@@ -497,11 +545,11 @@
virtual ~CaptureStateListener() = default;
};
- // Regiseters a listener for sound trigger capture state changes.
+ // Registers a listener for sound trigger capture state changes.
// There may only be one such listener registered at any point.
- // The listener onStateChanged() method will be invoked sychronously from
+ // The listener onStateChanged() method will be invoked synchronously from
// this call with the initial value.
- // The listener onServiceDied() method will be invoked sychronously from
+ // The listener onServiceDied() method will be invoked synchronously from
// this call if initial attempt to register failed.
// If the audio policy service cannot be reached, this method will return
// PERMISSION_DENIED and will not invoke the callback, otherwise, it will
diff --git a/media/libaudioclient/include/media/AudioTrack.h b/media/libaudioclient/include/media/AudioTrack.h
index cb00990..5fea637 100644
--- a/media/libaudioclient/include/media/AudioTrack.h
+++ b/media/libaudioclient/include/media/AudioTrack.h
@@ -401,6 +401,7 @@
uint32_t channelCount() const { return mChannelCount; }
size_t frameCount() const { return mFrameCount; }
+ audio_channel_mask_t channelMask() const { return mChannelMask; }
/*
* Return the period of the notification callback in frames.
diff --git a/media/libaudioclient/include/media/IAudioFlinger.h b/media/libaudioclient/include/media/IAudioFlinger.h
index 0e059f7..9e5019e 100644
--- a/media/libaudioclient/include/media/IAudioFlinger.h
+++ b/media/libaudioclient/include/media/IAudioFlinger.h
@@ -329,6 +329,9 @@
/* Indicate JAVA services are ready (scheduling, power management ...) */
virtual status_t systemReady() = 0;
+ // Indicate audio policy service is ready
+ virtual status_t audioPolicyReady() = 0;
+
// Returns the number of frames per audio HAL buffer.
virtual size_t frameCountHAL(audio_io_handle_t ioHandle) const = 0;
@@ -432,6 +435,8 @@
status_t setAudioPortConfig(const struct audio_port_config* config) override;
audio_hw_sync_t getAudioHwSyncForSession(audio_session_t sessionId) override;
status_t systemReady() override;
+ status_t audioPolicyReady() override;
+
size_t frameCountHAL(audio_io_handle_t ioHandle) const override;
status_t getMicrophones(std::vector<media::MicrophoneInfo>* microphones) override;
status_t setAudioHalPids(const std::vector<pid_t>& pids) override;
@@ -514,6 +519,7 @@
SET_AUDIO_PORT_CONFIG = media::BnAudioFlingerService::TRANSACTION_setAudioPortConfig,
GET_AUDIO_HW_SYNC_FOR_SESSION = media::BnAudioFlingerService::TRANSACTION_getAudioHwSyncForSession,
SYSTEM_READY = media::BnAudioFlingerService::TRANSACTION_systemReady,
+ AUDIO_POLICY_READY = media::BnAudioFlingerService::TRANSACTION_audioPolicyReady,
FRAME_COUNT_HAL = media::BnAudioFlingerService::TRANSACTION_frameCountHAL,
GET_MICROPHONES = media::BnAudioFlingerService::TRANSACTION_getMicrophones,
SET_MASTER_BALANCE = media::BnAudioFlingerService::TRANSACTION_setMasterBalance,
@@ -624,6 +630,7 @@
Status setAudioPortConfig(const media::AudioPortConfig& config) override;
Status getAudioHwSyncForSession(int32_t sessionId, int32_t* _aidl_return) override;
Status systemReady() override;
+ Status audioPolicyReady() override;
Status frameCountHAL(int32_t ioHandle, int64_t* _aidl_return) override;
Status getMicrophones(std::vector<media::MicrophoneInfoData>* _aidl_return) override;
Status setAudioHalPids(const std::vector<int32_t>& pids) override;
diff --git a/media/libaudiohal/impl/ConversionHelperHidl.cpp b/media/libaudiohal/impl/ConversionHelperHidl.cpp
index 32eaa31..0503698 100644
--- a/media/libaudiohal/impl/ConversionHelperHidl.cpp
+++ b/media/libaudiohal/impl/ConversionHelperHidl.cpp
@@ -105,6 +105,15 @@
}
// static
+void ConversionHelperHidl::argsFromHal(
+ const Vector<String16>& args, hidl_vec<hidl_string> *hidlArgs) {
+ hidlArgs->resize(args.size());
+ for (size_t i = 0; i < args.size(); ++i) {
+ (*hidlArgs)[i] = String8(args[i]).c_str();
+ }
+}
+
+// static
status_t ConversionHelperHidl::analyzeResult(const Result& result) {
switch (result) {
case Result::OK: return OK;
diff --git a/media/libaudiohal/impl/ConversionHelperHidl.h b/media/libaudiohal/impl/ConversionHelperHidl.h
index 59122c7..2909013 100644
--- a/media/libaudiohal/impl/ConversionHelperHidl.h
+++ b/media/libaudiohal/impl/ConversionHelperHidl.h
@@ -21,6 +21,8 @@
#include <hidl/HidlSupport.h>
#include <system/audio.h>
#include <utils/String8.h>
+#include <utils/String16.h>
+#include <utils/Vector.h>
using ::android::hardware::audio::CPP_VERSION::ParameterValue;
using CoreResult = ::android::hardware::audio::CPP_VERSION::Result;
@@ -37,6 +39,7 @@
static status_t keysFromHal(const String8& keys, hidl_vec<hidl_string> *hidlKeys);
static status_t parametersFromHal(const String8& kvPairs, hidl_vec<ParameterValue> *hidlParams);
static void parametersToHal(const hidl_vec<ParameterValue>& parameters, String8 *values);
+ static void argsFromHal(const Vector<String16>& args, hidl_vec<hidl_string> *hidlArgs);
ConversionHelperHidl(const char* className);
diff --git a/media/libaudiohal/impl/DeviceHalHidl.cpp b/media/libaudiohal/impl/DeviceHalHidl.cpp
index 02d66ae..aa94eea 100644
--- a/media/libaudiohal/impl/DeviceHalHidl.cpp
+++ b/media/libaudiohal/impl/DeviceHalHidl.cpp
@@ -457,11 +457,13 @@
}
#endif
-status_t DeviceHalHidl::dump(int fd) {
+status_t DeviceHalHidl::dump(int fd, const Vector<String16>& args) {
if (mDevice == 0) return NO_INIT;
native_handle_t* hidlHandle = native_handle_create(1, 0);
hidlHandle->data[0] = fd;
- Return<void> ret = mDevice->debug(hidlHandle, {} /* options */);
+ hidl_vec<hidl_string> hidlArgs;
+ argsFromHal(args, &hidlArgs);
+ Return<void> ret = mDevice->debug(hidlHandle, hidlArgs);
native_handle_delete(hidlHandle);
// TODO(b/111997867, b/177271958) Workaround - remove when fixed.
diff --git a/media/libaudiohal/impl/DeviceHalHidl.h b/media/libaudiohal/impl/DeviceHalHidl.h
index 2c847cf..2694ab3 100644
--- a/media/libaudiohal/impl/DeviceHalHidl.h
+++ b/media/libaudiohal/impl/DeviceHalHidl.h
@@ -119,7 +119,7 @@
status_t addDeviceEffect(audio_port_handle_t device, sp<EffectHalInterface> effect) override;
status_t removeDeviceEffect(audio_port_handle_t device, sp<EffectHalInterface> effect) override;
- virtual status_t dump(int fd);
+ status_t dump(int fd, const Vector<String16>& args) override;
private:
friend class DevicesFactoryHalHidl;
diff --git a/media/libaudiohal/impl/DeviceHalLocal.cpp b/media/libaudiohal/impl/DeviceHalLocal.cpp
index af7dc1a..e0304af 100644
--- a/media/libaudiohal/impl/DeviceHalLocal.cpp
+++ b/media/libaudiohal/impl/DeviceHalLocal.cpp
@@ -233,7 +233,7 @@
return INVALID_OPERATION;
}
-status_t DeviceHalLocal::dump(int fd) {
+status_t DeviceHalLocal::dump(int fd, const Vector<String16>& /* args */) {
return mDev->dump(mDev, fd);
}
diff --git a/media/libaudiohal/impl/DeviceHalLocal.h b/media/libaudiohal/impl/DeviceHalLocal.h
index 46b510b..2fde936 100644
--- a/media/libaudiohal/impl/DeviceHalLocal.h
+++ b/media/libaudiohal/impl/DeviceHalLocal.h
@@ -112,7 +112,7 @@
status_t addDeviceEffect(audio_port_handle_t device, sp<EffectHalInterface> effect) override;
status_t removeDeviceEffect(audio_port_handle_t device, sp<EffectHalInterface> effect) override;
- virtual status_t dump(int fd);
+ status_t dump(int fd, const Vector<String16>& args) override;
void closeOutputStream(struct audio_stream_out *stream_out);
void closeInputStream(struct audio_stream_in *stream_in);
diff --git a/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp b/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp
index f042b92..ffe0d72 100644
--- a/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp
+++ b/media/libaudiohal/impl/EffectsFactoryHalHidl.cpp
@@ -73,7 +73,9 @@
uint32_t index, effect_descriptor_t *pDescriptor) {
// TODO: We need somehow to track the changes on the server side
// or figure out how to convert everybody to query all the descriptors at once.
- // TODO: check for nullptr
+ if (pDescriptor == nullptr) {
+ return BAD_VALUE;
+ }
if (mLastDescriptors.size() == 0) {
status_t queryResult = queryAllDescriptors();
if (queryResult != OK) return queryResult;
@@ -85,7 +87,9 @@
status_t EffectsFactoryHalHidl::getDescriptor(
const effect_uuid_t *pEffectUuid, effect_descriptor_t *pDescriptor) {
- // TODO: check for nullptr
+ if (pDescriptor == nullptr || pEffectUuid == nullptr) {
+ return BAD_VALUE;
+ }
if (mEffectsFactory == 0) return NO_INIT;
Uuid hidlUuid;
UuidUtils::uuidFromHal(*pEffectUuid, &hidlUuid);
@@ -105,6 +109,33 @@
return processReturn(__FUNCTION__, ret);
}
+status_t EffectsFactoryHalHidl::getDescriptors(const effect_uuid_t *pEffectType,
+ std::vector<effect_descriptor_t> *descriptors) {
+ if (pEffectType == nullptr || descriptors == nullptr) {
+ return BAD_VALUE;
+ }
+
+ uint32_t numEffects = 0;
+ status_t status = queryNumberEffects(&numEffects);
+ if (status != NO_ERROR) {
+ ALOGW("%s error %d from FactoryHal queryNumberEffects", __func__, status);
+ return status;
+ }
+
+ for (uint32_t i = 0; i < numEffects; i++) {
+ effect_descriptor_t descriptor;
+ status = getDescriptor(i, &descriptor);
+ if (status != NO_ERROR) {
+ ALOGW("%s error %d from FactoryHal getDescriptor", __func__, status);
+ continue;
+ }
+ if (memcmp(&descriptor.type, pEffectType, sizeof(effect_uuid_t)) == 0) {
+ descriptors->push_back(descriptor);
+ }
+ }
+ return descriptors->empty() ? NAME_NOT_FOUND : NO_ERROR;
+}
+
status_t EffectsFactoryHalHidl::createEffect(
const effect_uuid_t *pEffectUuid, int32_t sessionId, int32_t ioId,
int32_t deviceId __unused, sp<EffectHalInterface> *effect) {
diff --git a/media/libaudiohal/impl/EffectsFactoryHalHidl.h b/media/libaudiohal/impl/EffectsFactoryHalHidl.h
index 5fa85e7..ff26d9f 100644
--- a/media/libaudiohal/impl/EffectsFactoryHalHidl.h
+++ b/media/libaudiohal/impl/EffectsFactoryHalHidl.h
@@ -45,6 +45,9 @@
virtual status_t getDescriptor(const effect_uuid_t *pEffectUuid,
effect_descriptor_t *pDescriptor);
+ virtual status_t getDescriptors(const effect_uuid_t *pEffectType,
+ std::vector<effect_descriptor_t> *descriptors);
+
// Creates an effect engine of the specified type.
// To release the effect engine, it is necessary to release references
// to the returned effect object.
diff --git a/media/libaudiohal/impl/StreamHalHidl.cpp b/media/libaudiohal/impl/StreamHalHidl.cpp
index 129b1c1..e63aded 100644
--- a/media/libaudiohal/impl/StreamHalHidl.cpp
+++ b/media/libaudiohal/impl/StreamHalHidl.cpp
@@ -152,11 +152,13 @@
return processReturn("standby", mStream->standby());
}
-status_t StreamHalHidl::dump(int fd) {
+status_t StreamHalHidl::dump(int fd, const Vector<String16>& args) {
if (!mStream) return NO_INIT;
native_handle_t* hidlHandle = native_handle_create(1, 0);
hidlHandle->data[0] = fd;
- Return<void> ret = mStream->debug(hidlHandle, {} /* options */);
+ hidl_vec<hidl_string> hidlArgs;
+ argsFromHal(args, &hidlArgs);
+ Return<void> ret = mStream->debug(hidlHandle, hidlArgs);
native_handle_delete(hidlHandle);
// TODO(b/111997867, b/177271958) Workaround - remove when fixed.
diff --git a/media/libaudiohal/impl/StreamHalHidl.h b/media/libaudiohal/impl/StreamHalHidl.h
index 970903b..6f5dd04 100644
--- a/media/libaudiohal/impl/StreamHalHidl.h
+++ b/media/libaudiohal/impl/StreamHalHidl.h
@@ -71,7 +71,7 @@
// Put the audio hardware input/output into standby mode.
virtual status_t standby();
- virtual status_t dump(int fd);
+ virtual status_t dump(int fd, const Vector<String16>& args) override;
// Start a stream operating in mmap mode.
virtual status_t start();
diff --git a/media/libaudiohal/impl/StreamHalLocal.cpp b/media/libaudiohal/impl/StreamHalLocal.cpp
index 34bd5df..11fac61 100644
--- a/media/libaudiohal/impl/StreamHalLocal.cpp
+++ b/media/libaudiohal/impl/StreamHalLocal.cpp
@@ -87,7 +87,8 @@
return mStream->standby(mStream);
}
-status_t StreamHalLocal::dump(int fd) {
+status_t StreamHalLocal::dump(int fd, const Vector<String16>& args) {
+ (void) args;
status_t status = mStream->dump(mStream, fd);
mStreamPowerLog.dump(fd);
return status;
diff --git a/media/libaudiohal/impl/StreamHalLocal.h b/media/libaudiohal/impl/StreamHalLocal.h
index b260495..493c521 100644
--- a/media/libaudiohal/impl/StreamHalLocal.h
+++ b/media/libaudiohal/impl/StreamHalLocal.h
@@ -50,7 +50,7 @@
// Put the audio hardware input/output into standby mode.
virtual status_t standby();
- virtual status_t dump(int fd);
+ virtual status_t dump(int fd, const Vector<String16>& args) override;
// Start a stream operating in mmap mode.
virtual status_t start() = 0;
diff --git a/media/libaudiohal/include/media/audiohal/DeviceHalInterface.h b/media/libaudiohal/include/media/audiohal/DeviceHalInterface.h
index 29ef011..69cbcec 100644
--- a/media/libaudiohal/include/media/audiohal/DeviceHalInterface.h
+++ b/media/libaudiohal/include/media/audiohal/DeviceHalInterface.h
@@ -120,7 +120,7 @@
virtual status_t removeDeviceEffect(
audio_port_handle_t device, sp<EffectHalInterface> effect) = 0;
- virtual status_t dump(int fd) = 0;
+ virtual status_t dump(int fd, const Vector<String16>& args) = 0;
protected:
// Subclasses can not be constructed directly by clients.
diff --git a/media/libaudiohal/include/media/audiohal/EffectsFactoryHalInterface.h b/media/libaudiohal/include/media/audiohal/EffectsFactoryHalInterface.h
index 9fb56ae..3e505bd 100644
--- a/media/libaudiohal/include/media/audiohal/EffectsFactoryHalInterface.h
+++ b/media/libaudiohal/include/media/audiohal/EffectsFactoryHalInterface.h
@@ -37,6 +37,9 @@
virtual status_t getDescriptor(const effect_uuid_t *pEffectUuid,
effect_descriptor_t *pDescriptor) = 0;
+ virtual status_t getDescriptors(const effect_uuid_t *pEffectType,
+ std::vector<effect_descriptor_t> *descriptors) = 0;
+
// Creates an effect engine of the specified type.
// To release the effect engine, it is necessary to release references
// to the returned effect object.
diff --git a/media/libaudiohal/include/media/audiohal/StreamHalInterface.h b/media/libaudiohal/include/media/audiohal/StreamHalInterface.h
index 2be12fb..2b5b2db 100644
--- a/media/libaudiohal/include/media/audiohal/StreamHalInterface.h
+++ b/media/libaudiohal/include/media/audiohal/StreamHalInterface.h
@@ -25,6 +25,7 @@
#include <utils/Errors.h>
#include <utils/RefBase.h>
#include <utils/String8.h>
+#include <utils/Vector.h>
namespace android {
@@ -69,7 +70,7 @@
// Put the audio hardware input/output into standby mode.
virtual status_t standby() = 0;
- virtual status_t dump(int fd) = 0;
+ virtual status_t dump(int fd, const Vector<String16>& args = {}) = 0;
// Start a stream operating in mmap mode.
virtual status_t start() = 0;
diff --git a/media/libaudioprocessing/AudioMixer.cpp b/media/libaudioprocessing/AudioMixer.cpp
index d85e2e9..e68c002 100644
--- a/media/libaudioprocessing/AudioMixer.cpp
+++ b/media/libaudioprocessing/AudioMixer.cpp
@@ -434,6 +434,12 @@
track->mHapticIntensity = hapticIntensity;
}
} break;
+ case HAPTIC_MAX_AMPLITUDE: {
+ const float hapticMaxAmplitude = *reinterpret_cast<float*>(value);
+ if (track->mHapticMaxAmplitude != hapticMaxAmplitude) {
+ track->mHapticMaxAmplitude = hapticMaxAmplitude;
+ }
+ } break;
default:
LOG_ALWAYS_FATAL("setParameter track: bad param %d", param);
}
@@ -553,6 +559,7 @@
// haptic
t->mHapticPlaybackEnabled = false;
t->mHapticIntensity = os::HapticScale::NONE;
+ t->mHapticMaxAmplitude = NAN;
t->mMixerHapticChannelMask = AUDIO_CHANNEL_NONE;
t->mMixerHapticChannelCount = 0;
t->mAdjustInChannelCount = t->channelCount + t->mHapticChannelCount;
@@ -602,7 +609,8 @@
switch (t->mMixerFormat) {
// Mixer format should be AUDIO_FORMAT_PCM_FLOAT.
case AUDIO_FORMAT_PCM_FLOAT: {
- os::scaleHapticData((float*) buffer, sampleCount, t->mHapticIntensity);
+ os::scaleHapticData((float*) buffer, sampleCount, t->mHapticIntensity,
+ t->mHapticMaxAmplitude);
} break;
default:
LOG_ALWAYS_FATAL("bad mMixerFormat: %#x", t->mMixerFormat);
diff --git a/media/libaudioprocessing/AudioMixerOps.h b/media/libaudioprocessing/AudioMixerOps.h
index cd47dc6..2988c67 100644
--- a/media/libaudioprocessing/AudioMixerOps.h
+++ b/media/libaudioprocessing/AudioMixerOps.h
@@ -17,6 +17,8 @@
#ifndef ANDROID_AUDIO_MIXER_OPS_H
#define ANDROID_AUDIO_MIXER_OPS_H
+#include <audio_utils/channels.h>
+#include <audio_utils/primitives.h>
#include <system/audio.h>
namespace android {
@@ -229,15 +231,26 @@
* complexity of working on interleaved streams is now getting
* too high, and likely limits compiler optimization.
*/
-template <int MIXTYPE, int NCHAN,
+
+// compile-time function.
+constexpr inline bool usesCenterChannel(audio_channel_mask_t mask) {
+ using namespace audio_utils::channels;
+ for (size_t i = 0; i < std::size(kSideFromChannelIdx); ++i) {
+ if ((mask & (1 << i)) != 0 && kSideFromChannelIdx[i] == AUDIO_GEOMETRY_SIDE_CENTER) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/*
+ * Applies stereo volume to the audio data based on proper left right channel affinity
+ * (templated channel MASK parameter).
+ */
+template <int MIXTYPE, audio_channel_mask_t MASK,
typename TO, typename TI, typename TV,
typename F>
-void stereoVolumeHelper(TO*& out, const TI*& in, const TV *vol, F f) {
- static_assert(NCHAN > 0 && NCHAN <= FCC_LIMIT);
- static_assert(MIXTYPE == MIXTYPE_MULTI_STEREOVOL
- || MIXTYPE == MIXTYPE_MULTI_SAVEONLY_STEREOVOL
- || MIXTYPE == MIXTYPE_STEREOEXPAND
- || MIXTYPE == MIXTYPE_MONOEXPAND);
+void stereoVolumeHelperWithChannelMask(TO*& out, const TI*& in, const TV *vol, F f) {
auto proc = [](auto& a, const auto& b) {
if constexpr (MIXTYPE == MIXTYPE_MULTI_STEREOVOL
|| MIXTYPE == MIXTYPE_STEREOEXPAND
@@ -250,59 +263,109 @@
auto inp = [&in]() -> const TI& {
if constexpr (MIXTYPE == MIXTYPE_STEREOEXPAND
|| MIXTYPE == MIXTYPE_MONOEXPAND) {
- return *in;
+ return *in; // note STEREOEXPAND assumes replicated L/R channels (see doc below).
} else {
return *in++;
}
};
- // HALs should only expose the canonical channel masks.
- proc(*out++, f(inp(), vol[0])); // front left
- if constexpr (NCHAN == 1) return;
- proc(*out++, f(inp(), vol[1])); // front right
- if constexpr (NCHAN == 2) return;
- if constexpr (NCHAN == 4) {
- proc(*out++, f(inp(), vol[0])); // back left
- proc(*out++, f(inp(), vol[1])); // back right
- return;
- }
-
- // TODO: Precompute center volume if not ramping.
std::decay_t<TV> center;
- if constexpr (std::is_floating_point_v<TV>) {
- center = (vol[0] + vol[1]) * 0.5; // do not use divide
- } else {
- center = (vol[0] >> 1) + (vol[1] >> 1); // rounds to 0.
- }
- proc(*out++, f(inp(), center)); // center (or 2.1 LFE)
- if constexpr (NCHAN == 3) return;
- if constexpr (NCHAN == 5) {
- proc(*out++, f(inp(), vol[0])); // back left
- proc(*out++, f(inp(), vol[1])); // back right
- return;
- }
-
- proc(*out++, f(inp(), center)); // lfe
- proc(*out++, f(inp(), vol[0])); // back left
- proc(*out++, f(inp(), vol[1])); // back right
- if constexpr (NCHAN == 6) return;
- if constexpr (NCHAN == 7) {
- proc(*out++, f(inp(), center)); // back center
- return;
- }
- // NCHAN == 8
- proc(*out++, f(inp(), vol[0])); // side left
- proc(*out++, f(inp(), vol[1])); // side right
- if constexpr (NCHAN > FCC_8) {
- // Mutes to zero extended surround channels.
- // 7.1.4 has the correct behavior.
- // 22.2 has the behavior that FLC and FRC will be mixed instead
- // of SL and SR and LFE will be center, not left.
- for (int i = 8; i < NCHAN; ++i) {
- // TODO: Consider using android::audio_utils::channels::kSideFromChannelIdx
- proc(*out++, f(inp(), 0.f));
+ constexpr bool USES_CENTER_CHANNEL = usesCenterChannel(MASK);
+ if constexpr (USES_CENTER_CHANNEL) {
+ if constexpr (std::is_floating_point_v<TV>) {
+ center = (vol[0] + vol[1]) * 0.5; // do not use divide
+ } else {
+ center = (vol[0] >> 1) + (vol[1] >> 1); // rounds to 0.
}
}
+
+ using namespace audio_utils::channels;
+
+ // if LFE and LFE2 are both present, they take left and right volume respectively.
+ constexpr unsigned LFE_LFE2 = \
+ AUDIO_CHANNEL_OUT_LOW_FREQUENCY | AUDIO_CHANNEL_OUT_LOW_FREQUENCY_2;
+ constexpr bool has_LFE_LFE2 = (MASK & LFE_LFE2) == LFE_LFE2;
+
+#pragma push_macro("DO_CHANNEL_POSITION")
+#undef DO_CHANNEL_POSITION
+#define DO_CHANNEL_POSITION(BIT_INDEX) \
+ if constexpr ((MASK & (1 << BIT_INDEX)) != 0) { \
+ constexpr auto side = kSideFromChannelIdx[BIT_INDEX]; \
+ if constexpr (side == AUDIO_GEOMETRY_SIDE_LEFT || \
+ has_LFE_LFE2 && (1 << BIT_INDEX) == AUDIO_CHANNEL_OUT_LOW_FREQUENCY) { \
+ proc(*out++, f(inp(), vol[0])); \
+ } else if constexpr (side == AUDIO_GEOMETRY_SIDE_RIGHT || \
+ has_LFE_LFE2 && (1 << BIT_INDEX) == AUDIO_CHANNEL_OUT_LOW_FREQUENCY_2) { \
+ proc(*out++, f(inp(), vol[1])); \
+ } else /* constexpr */ { \
+ proc(*out++, f(inp(), center)); \
+ } \
+ }
+
+ DO_CHANNEL_POSITION(0);
+ DO_CHANNEL_POSITION(1);
+ DO_CHANNEL_POSITION(2);
+ DO_CHANNEL_POSITION(3);
+ DO_CHANNEL_POSITION(4);
+ DO_CHANNEL_POSITION(5);
+ DO_CHANNEL_POSITION(6);
+ DO_CHANNEL_POSITION(7);
+
+ DO_CHANNEL_POSITION(8);
+ DO_CHANNEL_POSITION(9);
+ DO_CHANNEL_POSITION(10);
+ DO_CHANNEL_POSITION(11);
+ DO_CHANNEL_POSITION(12);
+ DO_CHANNEL_POSITION(13);
+ DO_CHANNEL_POSITION(14);
+ DO_CHANNEL_POSITION(15);
+
+ DO_CHANNEL_POSITION(16);
+ DO_CHANNEL_POSITION(17);
+ DO_CHANNEL_POSITION(18);
+ DO_CHANNEL_POSITION(19);
+ DO_CHANNEL_POSITION(20);
+ DO_CHANNEL_POSITION(21);
+ DO_CHANNEL_POSITION(22);
+ DO_CHANNEL_POSITION(23);
+ static_assert(FCC_LIMIT <= FCC_24); // Note: this may need to change.
+#pragma pop_macro("DO_CHANNEL_POSITION")
+}
+
+// These are the channel position masks we expect from the HAL.
+// See audio_channel_out_mask_from_count() but this is constexpr
+constexpr inline audio_channel_mask_t canonicalChannelMaskFromCount(size_t channelCount) {
+ constexpr audio_channel_mask_t canonical[] = {
+ [0] = AUDIO_CHANNEL_NONE,
+ [1] = AUDIO_CHANNEL_OUT_MONO,
+ [2] = AUDIO_CHANNEL_OUT_STEREO,
+ [3] = AUDIO_CHANNEL_OUT_2POINT1,
+ [4] = AUDIO_CHANNEL_OUT_QUAD,
+ [5] = AUDIO_CHANNEL_OUT_PENTA,
+ [6] = AUDIO_CHANNEL_OUT_5POINT1,
+ [7] = AUDIO_CHANNEL_OUT_6POINT1,
+ [8] = AUDIO_CHANNEL_OUT_7POINT1,
+ [12] = AUDIO_CHANNEL_OUT_7POINT1POINT4,
+ [24] = AUDIO_CHANNEL_OUT_22POINT2,
+ };
+ return channelCount < std::size(canonical) ? canonical[channelCount] : AUDIO_CHANNEL_NONE;
+}
+
+template <int MIXTYPE, int NCHAN,
+ typename TO, typename TI, typename TV,
+ typename F>
+void stereoVolumeHelper(TO*& out, const TI*& in, const TV *vol, F f) {
+ static_assert(NCHAN > 0 && NCHAN <= FCC_LIMIT);
+ static_assert(MIXTYPE == MIXTYPE_MULTI_STEREOVOL
+ || MIXTYPE == MIXTYPE_MULTI_SAVEONLY_STEREOVOL
+ || MIXTYPE == MIXTYPE_STEREOEXPAND
+ || MIXTYPE == MIXTYPE_MONOEXPAND);
+ constexpr audio_channel_mask_t MASK{canonicalChannelMaskFromCount(NCHAN)};
+ if constexpr (MASK == AUDIO_CHANNEL_NONE) {
+ ALOGE("%s: Invalid position count %d", __func__, NCHAN);
+ return; // not a valid system mask, ignore.
+ }
+ stereoVolumeHelperWithChannelMask<MIXTYPE, MASK, TO, TI, TV, F>(out, in, vol, f);
}
/*
diff --git a/media/libaudioprocessing/include/media/AudioMixer.h b/media/libaudioprocessing/include/media/AudioMixer.h
index 70eafe3..5a9fa07 100644
--- a/media/libaudioprocessing/include/media/AudioMixer.h
+++ b/media/libaudioprocessing/include/media/AudioMixer.h
@@ -50,6 +50,7 @@
// for haptic
HAPTIC_ENABLED = 0x4007, // Set haptic data from this track should be played or not.
HAPTIC_INTENSITY = 0x4008, // Set the intensity to play haptic data.
+ HAPTIC_MAX_AMPLITUDE = 0x4009, // Set the max amplitude allowed for haptic data.
// for target TIMESTRETCH
PLAYBACK_RATE = 0x4300, // Configure timestretch on this track name;
// parameter 'value' is a pointer to the new playback rate.
@@ -145,6 +146,7 @@
// Haptic
bool mHapticPlaybackEnabled;
os::HapticScale mHapticIntensity;
+ float mHapticMaxAmplitude;
audio_channel_mask_t mHapticChannelMask;
uint32_t mHapticChannelCount;
audio_channel_mask_t mMixerHapticChannelMask;
diff --git a/media/libaudioprocessing/tests/Android.bp b/media/libaudioprocessing/tests/Android.bp
index 3856817..ad402db 100644
--- a/media/libaudioprocessing/tests/Android.bp
+++ b/media/libaudioprocessing/tests/Android.bp
@@ -76,6 +76,7 @@
//
cc_binary {
name: "mixerops_objdump",
+ header_libs: ["libaudioutils_headers"],
srcs: ["mixerops_objdump.cpp"],
}
@@ -84,6 +85,16 @@
//
cc_benchmark {
name: "mixerops_benchmark",
+ header_libs: ["libaudioutils_headers"],
srcs: ["mixerops_benchmark.cpp"],
static_libs: ["libgoogle-benchmark"],
}
+
+//
+// mixerops unit test
+//
+cc_test {
+ name: "mixerops_tests",
+ defaults: ["libaudioprocessing_test_defaults"],
+ srcs: ["mixerops_tests.cpp"],
+}
diff --git a/media/libaudioprocessing/tests/mixerops_benchmark.cpp b/media/libaudioprocessing/tests/mixerops_benchmark.cpp
index 7a4c5c7..f866b1a 100644
--- a/media/libaudioprocessing/tests/mixerops_benchmark.cpp
+++ b/media/libaudioprocessing/tests/mixerops_benchmark.cpp
@@ -16,11 +16,9 @@
#include <inttypes.h>
#include <type_traits>
-#include "../../../../system/media/audio_utils/include/audio_utils/primitives.h"
#define LOG_ALWAYS_FATAL(...)
#include <../AudioMixerOps.h>
-
#include <benchmark/benchmark.h>
using namespace android;
diff --git a/media/libaudioprocessing/tests/mixerops_tests.cpp b/media/libaudioprocessing/tests/mixerops_tests.cpp
new file mode 100644
index 0000000..2500ba9
--- /dev/null
+++ b/media/libaudioprocessing/tests/mixerops_tests.cpp
@@ -0,0 +1,175 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "mixerop_tests"
+#include <log/log.h>
+
+#include <inttypes.h>
+#include <type_traits>
+
+#include <../AudioMixerOps.h>
+#include <gtest/gtest.h>
+
+using namespace android;
+
+// Note: gtest templated tests require typenames, not integers.
+template <int MIXTYPE, int NCHAN>
+class MixerOpsBasicTest {
+public:
+ static void testStereoVolume() {
+ using namespace android::audio_utils::channels;
+
+ constexpr size_t FRAME_COUNT = 1000;
+ constexpr size_t SAMPLE_COUNT = FRAME_COUNT * NCHAN;
+
+ const float in[SAMPLE_COUNT] = {[0 ... (SAMPLE_COUNT - 1)] = 1.f};
+
+ AUDIO_GEOMETRY_SIDE sides[NCHAN];
+ size_t i = 0;
+ unsigned channel = canonicalChannelMaskFromCount(NCHAN);
+ constexpr unsigned LFE_LFE2 =
+ AUDIO_CHANNEL_OUT_LOW_FREQUENCY | AUDIO_CHANNEL_OUT_LOW_FREQUENCY_2;
+ bool has_LFE_LFE2 = (channel & LFE_LFE2) == LFE_LFE2;
+ while (channel != 0) {
+ const int index = __builtin_ctz(channel);
+ if (has_LFE_LFE2 && (1 << index) == AUDIO_CHANNEL_OUT_LOW_FREQUENCY) {
+ sides[i++] = AUDIO_GEOMETRY_SIDE_LEFT; // special case
+ } else if (has_LFE_LFE2 && (1 << index) == AUDIO_CHANNEL_OUT_LOW_FREQUENCY_2) {
+ sides[i++] = AUDIO_GEOMETRY_SIDE_RIGHT; // special case
+ } else {
+ sides[i++] = sideFromChannelIdx(index);
+ }
+ channel &= ~(1 << index);
+ }
+
+ float vola[2] = {1.f, 0.f}; // left volume at max.
+ float out[SAMPLE_COUNT]{};
+ float aux[FRAME_COUNT]{};
+ float volaux = 0.5;
+ {
+ volumeMulti<MIXTYPE, NCHAN>(out, FRAME_COUNT, in, aux, vola, volaux);
+ const float *outp = out;
+ const float *auxp = aux;
+ const float left = vola[0];
+ const float center = (vola[0] + vola[1]) * 0.5;
+ const float right = vola[1];
+ for (size_t i = 0; i < FRAME_COUNT; ++i) {
+ for (size_t j = 0; j < NCHAN; ++j) {
+ const float audio = *outp++;
+ if (sides[j] == AUDIO_GEOMETRY_SIDE_LEFT) {
+ EXPECT_EQ(left, audio);
+ } else if (sides[j] == AUDIO_GEOMETRY_SIDE_CENTER) {
+ EXPECT_EQ(center, audio);
+ } else {
+ EXPECT_EQ(right, audio);
+ }
+ }
+ EXPECT_EQ(volaux, *auxp++); // works if all channels contain 1.f
+ }
+ }
+ float volb[2] = {0.f, 0.5f}; // right volume at half max.
+ {
+ // this accumulates into out, aux.
+ // float out[SAMPLE_COUNT]{};
+ // float aux[FRAME_COUNT]{};
+ volumeMulti<MIXTYPE, NCHAN>(out, FRAME_COUNT, in, aux, volb, volaux);
+ const float *outp = out;
+ const float *auxp = aux;
+ const float left = vola[0] + volb[0];
+ const float center = (vola[0] + vola[1] + volb[0] + volb[1]) * 0.5;
+ const float right = vola[1] + volb[1];
+ for (size_t i = 0; i < FRAME_COUNT; ++i) {
+ for (size_t j = 0; j < NCHAN; ++j) {
+ const float audio = *outp++;
+ if (sides[j] == AUDIO_GEOMETRY_SIDE_LEFT) {
+ EXPECT_EQ(left, audio);
+ } else if (sides[j] == AUDIO_GEOMETRY_SIDE_CENTER) {
+ EXPECT_EQ(center, audio);
+ } else {
+ EXPECT_EQ(right, audio);
+ }
+ }
+ // aux is accumulated so 2x the amplitude
+ EXPECT_EQ(volaux * 2.f, *auxp++); // works if all channels contain 1.f
+ }
+ }
+
+ { // test aux as derived from out.
+ // AUX channel is the weighted sum of all of the output channels prior to volume
+ // adjustment. We must set L and R to the same volume to allow computation
+ // of AUX from the output values.
+ const float volmono = 0.25f;
+ const float vollr[2] = {volmono, volmono}; // all the same.
+ float out[SAMPLE_COUNT]{};
+ float aux[FRAME_COUNT]{};
+ volumeMulti<MIXTYPE, NCHAN>(out, FRAME_COUNT, in, aux, vollr, volaux);
+ const float *outp = out;
+ const float *auxp = aux;
+ for (size_t i = 0; i < FRAME_COUNT; ++i) {
+ float accum = 0.f;
+ for (size_t j = 0; j < NCHAN; ++j) {
+ accum += *outp++;
+ }
+ EXPECT_EQ(accum / NCHAN * volaux / volmono, *auxp++);
+ }
+ }
+ }
+};
+
+TEST(mixerops, stereovolume_1) { // Note: mono not used for output sinks yet.
+ MixerOpsBasicTest<MIXTYPE_MULTI_STEREOVOL, 1>::testStereoVolume();
+}
+TEST(mixerops, stereovolume_2) {
+ MixerOpsBasicTest<MIXTYPE_MULTI_STEREOVOL, 2>::testStereoVolume();
+}
+TEST(mixerops, stereovolume_3) {
+ MixerOpsBasicTest<MIXTYPE_MULTI_STEREOVOL, 3>::testStereoVolume();
+}
+TEST(mixerops, stereovolume_4) {
+ MixerOpsBasicTest<MIXTYPE_MULTI_STEREOVOL, 4>::testStereoVolume();
+}
+TEST(mixerops, stereovolume_5) {
+ MixerOpsBasicTest<MIXTYPE_MULTI_STEREOVOL, 5>::testStereoVolume();
+}
+TEST(mixerops, stereovolume_6) {
+ MixerOpsBasicTest<MIXTYPE_MULTI_STEREOVOL, 6>::testStereoVolume();
+}
+TEST(mixerops, stereovolume_7) {
+ MixerOpsBasicTest<MIXTYPE_MULTI_STEREOVOL, 7>::testStereoVolume();
+}
+TEST(mixerops, stereovolume_8) {
+ MixerOpsBasicTest<MIXTYPE_MULTI_STEREOVOL, 8>::testStereoVolume();
+}
+TEST(mixerops, stereovolume_12) {
+ if constexpr (FCC_LIMIT >= 12) { // NOTE: FCC_LIMIT is an enum, so can't #if
+ MixerOpsBasicTest<MIXTYPE_MULTI_STEREOVOL, 12>::testStereoVolume();
+ }
+}
+TEST(mixerops, stereovolume_24) {
+ if constexpr (FCC_LIMIT >= 24) {
+ MixerOpsBasicTest<MIXTYPE_MULTI_STEREOVOL, 24>::testStereoVolume();
+ }
+}
+TEST(mixerops, channel_equivalence) {
+ // we must match the constexpr function with the system determined channel mask from count.
+ for (size_t i = 0; i < FCC_LIMIT; ++i) {
+ const audio_channel_mask_t actual = canonicalChannelMaskFromCount(i);
+ const audio_channel_mask_t system = audio_channel_out_mask_from_count(i);
+ if (system == AUDIO_CHANNEL_INVALID) continue;
+ EXPECT_EQ(system, actual);
+ }
+}
diff --git a/media/libeffects/downmix/EffectDownmix.cpp b/media/libeffects/downmix/EffectDownmix.cpp
index f500bc3..90bb410 100644
--- a/media/libeffects/downmix/EffectDownmix.cpp
+++ b/media/libeffects/downmix/EffectDownmix.cpp
@@ -19,7 +19,7 @@
#include <log/log.h>
#include "EffectDownmix.h"
-#include <math.h>
+#include <audio_utils/ChannelMix.h>
// Do not submit with DOWNMIX_TEST_CHANNEL_INDEX defined, strictly for testing
//#define DOWNMIX_TEST_CHANNEL_INDEX 0
@@ -35,12 +35,13 @@
} downmix_state_t;
/* parameters for each downmixer */
-typedef struct {
+struct downmix_object_t {
downmix_state_t state;
downmix_type_t type;
bool apply_volume_correction;
uint8_t input_channel_count;
-} downmix_object_t;
+ android::audio_utils::channels::ChannelMix channelMix;
+};
typedef struct downmix_module_s {
const struct effect_interface_s *itfe;
@@ -77,11 +78,6 @@
downmix_object_t *pDownmixer, int32_t param, uint32_t size, void *pValue);
static int Downmix_getParameter(
downmix_object_t *pDownmixer, int32_t param, uint32_t *pSize, void *pValue);
-static void Downmix_foldFromQuad(float *pSrc, float *pDst, size_t numFrames, bool accumulate);
-static void Downmix_foldFrom5Point1(float *pSrc, float *pDst, size_t numFrames, bool accumulate);
-static void Downmix_foldFrom7Point1(float *pSrc, float *pDst, size_t numFrames, bool accumulate);
-static bool Downmix_foldGeneric(
- uint32_t mask, float *pSrc, float *pDst, size_t numFrames, bool accumulate);
// effect_handle_t interface implementation for downmix effect
const struct effect_interface_s gDownmixInterface = {
@@ -315,7 +311,8 @@
audio_buffer_t *inBuffer, audio_buffer_t *outBuffer) {
downmix_object_t *pDownmixer;
- float *pSrc, *pDst;
+ const float *pSrc;
+ float *pDst;
downmix_module_t *pDwmModule = (downmix_module_t *)self;
if (pDwmModule == NULL) {
@@ -344,7 +341,8 @@
const bool accumulate =
(pDwmModule->config.outputCfg.accessMode == EFFECT_BUFFER_ACCESS_ACCUMULATE);
- const uint32_t downmixInputChannelMask = pDwmModule->config.inputCfg.channels;
+ const audio_channel_mask_t downmixInputChannelMask =
+ (audio_channel_mask_t)pDwmModule->config.inputCfg.channels;
switch(pDownmixer->type) {
@@ -368,38 +366,13 @@
}
break;
- case DOWNMIX_TYPE_FOLD:
-#ifdef DOWNMIX_ALWAYS_USE_GENERIC_DOWNMIXER
- // bypass the optimized downmix routines for the common formats
- if (!Downmix_foldGeneric(
- downmixInputChannelMask, pSrc, pDst, numFrames, accumulate)) {
- ALOGE("Multichannel configuration %#x is not supported",
- downmixInputChannelMask);
- return -EINVAL;
- }
- break;
-#endif
- // optimize for the common formats
- switch (downmixInputChannelMask) {
- case AUDIO_CHANNEL_OUT_QUAD_BACK:
- case AUDIO_CHANNEL_OUT_QUAD_SIDE:
- Downmix_foldFromQuad(pSrc, pDst, numFrames, accumulate);
- break;
- case AUDIO_CHANNEL_OUT_5POINT1_BACK:
- case AUDIO_CHANNEL_OUT_5POINT1_SIDE:
- Downmix_foldFrom5Point1(pSrc, pDst, numFrames, accumulate);
- break;
- case AUDIO_CHANNEL_OUT_7POINT1:
- Downmix_foldFrom7Point1(pSrc, pDst, numFrames, accumulate);
- break;
- default:
- if (!Downmix_foldGeneric(
- downmixInputChannelMask, pSrc, pDst, numFrames, accumulate)) {
+ case DOWNMIX_TYPE_FOLD: {
+ if (!pDownmixer->channelMix.process(
+ pSrc, pDst, numFrames, accumulate, downmixInputChannelMask)) {
ALOGE("Multichannel configuration %#x is not supported",
downmixInputChannelMask);
return -EINVAL;
}
- break;
}
break;
@@ -780,7 +753,6 @@
return 0;
} /* end Downmix_setParameter */
-
/*----------------------------------------------------------------------------
* Downmix_getParameter()
*----------------------------------------------------------------------------
@@ -829,299 +801,3 @@
return 0;
} /* end Downmix_getParameter */
-
-/*----------------------------------------------------------------------------
- * Downmix_foldFromQuad()
- *----------------------------------------------------------------------------
- * Purpose:
- * downmix a quad signal to stereo
- *
- * Inputs:
- * pSrc quad audio samples to downmix
- * numFrames the number of quad frames to downmix
- * accumulate whether to mix (when true) the result of the downmix with the contents of pDst,
- * or overwrite pDst (when false)
- *
- * Outputs:
- * pDst downmixed stereo audio samples
- *
- *----------------------------------------------------------------------------
- */
-void Downmix_foldFromQuad(float *pSrc, float *pDst, size_t numFrames, bool accumulate) {
- // sample at index 0 is FL
- // sample at index 1 is FR
- // sample at index 2 is RL
- // sample at index 3 is RR
- if (accumulate) {
- while (numFrames) {
- // FL + RL
- pDst[0] = clamp_float(pDst[0] + ((pSrc[0] + pSrc[2]) / 2.0f));
- // FR + RR
- pDst[1] = clamp_float(pDst[1] + ((pSrc[1] + pSrc[3]) / 2.0f));
- pSrc += 4;
- pDst += 2;
- numFrames--;
- }
- } else { // same code as above but without adding and clamping pDst[i] to itself
- while (numFrames) {
- // FL + RL
- pDst[0] = clamp_float((pSrc[0] + pSrc[2]) / 2.0f);
- // FR + RR
- pDst[1] = clamp_float((pSrc[1] + pSrc[3]) / 2.0f);
- pSrc += 4;
- pDst += 2;
- numFrames--;
- }
- }
-}
-
-/*----------------------------------------------------------------------------
- * Downmix_foldFrom5Point1()
- *----------------------------------------------------------------------------
- * Purpose:
- * downmix a 5.1 signal to stereo
- *
- * Inputs:
- * pSrc 5.1 audio samples to downmix
- * numFrames the number of 5.1 frames to downmix
- * accumulate whether to mix (when true) the result of the downmix with the contents of pDst,
- * or overwrite pDst (when false)
- *
- * Outputs:
- * pDst downmixed stereo audio samples
- *
- *----------------------------------------------------------------------------
- */
-void Downmix_foldFrom5Point1(float *pSrc, float *pDst, size_t numFrames, bool accumulate) {
- float lt, rt, centerPlusLfeContrib; // samples in Q19.12 format
- // sample at index 0 is FL
- // sample at index 1 is FR
- // sample at index 2 is FC
- // sample at index 3 is LFE
- // sample at index 4 is RL
- // sample at index 5 is RR
- // code is mostly duplicated between the two values of accumulate to avoid repeating the test
- // for every sample
- if (accumulate) {
- while (numFrames) {
- // centerPlusLfeContrib = FC(-3dB) + LFE(-3dB)
- centerPlusLfeContrib = (pSrc[2] * MINUS_3_DB_IN_FLOAT)
- + (pSrc[3] * MINUS_3_DB_IN_FLOAT);
- // FL + centerPlusLfeContrib + RL
- lt = pSrc[0] + centerPlusLfeContrib + pSrc[4];
- // FR + centerPlusLfeContrib + RR
- rt = pSrc[1] + centerPlusLfeContrib + pSrc[5];
- // accumulate in destination
- pDst[0] = clamp_float(pDst[0] + (lt / 2.0f));
- pDst[1] = clamp_float(pDst[1] + (rt / 2.0f));
- pSrc += 6;
- pDst += 2;
- numFrames--;
- }
- } else { // same code as above but without adding and clamping pDst[i] to itself
- while (numFrames) {
- // centerPlusLfeContrib = FC(-3dB) + LFE(-3dB)
- centerPlusLfeContrib = (pSrc[2] * MINUS_3_DB_IN_FLOAT)
- + (pSrc[3] * MINUS_3_DB_IN_FLOAT);
- // FL + centerPlusLfeContrib + RL
- lt = pSrc[0] + centerPlusLfeContrib + pSrc[4];
- // FR + centerPlusLfeContrib + RR
- rt = pSrc[1] + centerPlusLfeContrib + pSrc[5];
- // store in destination
- pDst[0] = clamp_float(lt / 2.0f); // differs from when accumulate is true above
- pDst[1] = clamp_float(rt / 2.0f); // differs from when accumulate is true above
- pSrc += 6;
- pDst += 2;
- numFrames--;
- }
- }
-}
-
-/*----------------------------------------------------------------------------
- * Downmix_foldFrom7Point1()
- *----------------------------------------------------------------------------
- * Purpose:
- * downmix a 7.1 signal to stereo
- *
- * Inputs:
- * pSrc 7.1 audio samples to downmix
- * numFrames the number of 7.1 frames to downmix
- * accumulate whether to mix (when true) the result of the downmix with the contents of pDst,
- * or overwrite pDst (when false)
- *
- * Outputs:
- * pDst downmixed stereo audio samples
- *
- *----------------------------------------------------------------------------
- */
-void Downmix_foldFrom7Point1(float *pSrc, float *pDst, size_t numFrames, bool accumulate) {
- float lt, rt, centerPlusLfeContrib; // samples in Q19.12 format
- // sample at index 0 is FL
- // sample at index 1 is FR
- // sample at index 2 is FC
- // sample at index 3 is LFE
- // sample at index 4 is RL
- // sample at index 5 is RR
- // sample at index 6 is SL
- // sample at index 7 is SR
- // code is mostly duplicated between the two values of accumulate to avoid repeating the test
- // for every sample
- if (accumulate) {
- while (numFrames) {
- // centerPlusLfeContrib = FC(-3dB) + LFE(-3dB)
- centerPlusLfeContrib = (pSrc[2] * MINUS_3_DB_IN_FLOAT)
- + (pSrc[3] * MINUS_3_DB_IN_FLOAT);
- // FL + centerPlusLfeContrib + SL + RL
- lt = pSrc[0] + centerPlusLfeContrib + pSrc[6] + pSrc[4];
- // FR + centerPlusLfeContrib + SR + RR
- rt = pSrc[1] + centerPlusLfeContrib + pSrc[7] + pSrc[5];
- //accumulate in destination
- pDst[0] = clamp_float(pDst[0] + (lt / 2.0f));
- pDst[1] = clamp_float(pDst[1] + (rt / 2.0f));
- pSrc += 8;
- pDst += 2;
- numFrames--;
- }
- } else { // same code as above but without adding and clamping pDst[i] to itself
- while (numFrames) {
- // centerPlusLfeContrib = FC(-3dB) + LFE(-3dB)
- centerPlusLfeContrib = (pSrc[2] * MINUS_3_DB_IN_FLOAT)
- + (pSrc[3] * MINUS_3_DB_IN_FLOAT);
- // FL + centerPlusLfeContrib + SL + RL
- lt = pSrc[0] + centerPlusLfeContrib + pSrc[6] + pSrc[4];
- // FR + centerPlusLfeContrib + SR + RR
- rt = pSrc[1] + centerPlusLfeContrib + pSrc[7] + pSrc[5];
- // store in destination
- pDst[0] = clamp_float(lt / 2.0f); // differs from when accumulate is true above
- pDst[1] = clamp_float(rt / 2.0f); // differs from when accumulate is true above
- pSrc += 8;
- pDst += 2;
- numFrames--;
- }
- }
-}
-
-/*----------------------------------------------------------------------------
- * Downmix_foldGeneric()
- *----------------------------------------------------------------------------
- * Purpose:
- * downmix to stereo a multichannel signal of arbitrary channel position mask.
- *
- * Inputs:
- * mask the channel mask of pSrc
- * pSrc multichannel audio buffer to downmix
- * numFrames the number of multichannel frames to downmix
- * accumulate whether to mix (when true) the result of the downmix with the contents of pDst,
- * or overwrite pDst (when false)
- *
- * Outputs:
- * pDst downmixed stereo audio samples
- *
- * Returns: false if multichannel format is not supported
- *
- *----------------------------------------------------------------------------
- */
-bool Downmix_foldGeneric(
- uint32_t mask, float *pSrc, float *pDst, size_t numFrames, bool accumulate) {
-
- if (!Downmix_validChannelMask(mask)) {
- return false;
- }
- const int numChan = audio_channel_count_from_out_mask(mask);
-
- // compute at what index each channel is: samples will be in the following order:
- // FL FR FC LFE BL BR BC SL SR
- //
- // (transfer matrix)
- // FL FR FC LFE BL BR BC SL SR
- // 0.5 0.353 0.353 0.5 0.353 0.5
- // 0.5 0.353 0.353 0.5 0.353 0.5
-
- // derive the indices for the transfer matrix columns that have non-zero values.
- int indexFL = -1;
- int indexFR = -1;
- int indexFC = -1;
- int indexLFE = -1;
- int indexBL = -1;
- int indexBR = -1;
- int indexBC = -1;
- int indexSL = -1;
- int indexSR = -1;
- int index = 0;
- for (unsigned tmp = mask;
- (tmp & (AUDIO_CHANNEL_OUT_7POINT1 | AUDIO_CHANNEL_OUT_BACK_CENTER)) != 0;
- ++index) {
- const unsigned lowestBit = tmp & -(signed)tmp;
- switch (lowestBit) {
- case AUDIO_CHANNEL_OUT_FRONT_LEFT:
- indexFL = index;
- break;
- case AUDIO_CHANNEL_OUT_FRONT_RIGHT:
- indexFR = index;
- break;
- case AUDIO_CHANNEL_OUT_FRONT_CENTER:
- indexFC = index;
- break;
- case AUDIO_CHANNEL_OUT_LOW_FREQUENCY:
- indexLFE = index;
- break;
- case AUDIO_CHANNEL_OUT_BACK_LEFT:
- indexBL = index;
- break;
- case AUDIO_CHANNEL_OUT_BACK_RIGHT:
- indexBR = index;
- break;
- case AUDIO_CHANNEL_OUT_BACK_CENTER:
- indexBC = index;
- break;
- case AUDIO_CHANNEL_OUT_SIDE_LEFT:
- indexSL = index;
- break;
- case AUDIO_CHANNEL_OUT_SIDE_RIGHT:
- indexSR = index;
- break;
- }
- tmp ^= lowestBit;
- }
-
- // With good branch prediction, this should run reasonably fast.
- // Also consider using a transfer matrix form.
- while (numFrames) {
- // compute contribution of FC, BC and LFE
- float centersLfeContrib = 0;
- if (indexFC >= 0) centersLfeContrib = pSrc[indexFC];
- if (indexLFE >= 0) centersLfeContrib += pSrc[indexLFE];
- if (indexBC >= 0) centersLfeContrib += pSrc[indexBC];
- centersLfeContrib *= MINUS_3_DB_IN_FLOAT;
-
- float ch[2];
- ch[0] = centersLfeContrib;
- ch[1] = centersLfeContrib;
-
- // mix in left / right channels
- if (indexFL >= 0) ch[0] += pSrc[indexFL];
- if (indexFR >= 0) ch[1] += pSrc[indexFR];
-
- if (indexSL >= 0) ch[0] += pSrc[indexSL];
- if (indexSR >= 0) ch[1] += pSrc[indexSR]; // note pair checks enforce this if indexSL != 0
-
- if (indexBL >= 0) ch[0] += pSrc[indexBL];
- if (indexBR >= 0) ch[1] += pSrc[indexBR]; // note pair checks enforce this if indexBL != 0
-
- // scale to prevent overflow.
- ch[0] *= 0.5f;
- ch[1] *= 0.5f;
-
- if (accumulate) {
- ch[0] += pDst[0];
- ch[1] += pDst[1];
- }
-
- pDst[0] = clamp_float(ch[0]);
- pDst[1] = clamp_float(ch[1]);
- pSrc += numChan;
- pDst += 2;
- numFrames--;
- }
- return true;
-}
diff --git a/media/libeffects/downmix/benchmark/downmix_benchmark.cpp b/media/libeffects/downmix/benchmark/downmix_benchmark.cpp
index ee169c2..d640e50 100644
--- a/media/libeffects/downmix/benchmark/downmix_benchmark.cpp
+++ b/media/libeffects/downmix/benchmark/downmix_benchmark.cpp
@@ -35,16 +35,14 @@
AUDIO_CHANNEL_OUT_STEREO,
AUDIO_CHANNEL_OUT_2POINT1,
AUDIO_CHANNEL_OUT_2POINT0POINT2,
- AUDIO_CHANNEL_OUT_QUAD,
- AUDIO_CHANNEL_OUT_QUAD_BACK,
+ AUDIO_CHANNEL_OUT_QUAD, // AUDIO_CHANNEL_OUT_QUAD_BACK
AUDIO_CHANNEL_OUT_QUAD_SIDE,
AUDIO_CHANNEL_OUT_SURROUND,
AUDIO_CHANNEL_OUT_2POINT1POINT2,
AUDIO_CHANNEL_OUT_3POINT0POINT2,
AUDIO_CHANNEL_OUT_PENTA,
AUDIO_CHANNEL_OUT_3POINT1POINT2,
- AUDIO_CHANNEL_OUT_5POINT1,
- AUDIO_CHANNEL_OUT_5POINT1_BACK,
+ AUDIO_CHANNEL_OUT_5POINT1, // AUDIO_CHANNEL_OUT_5POINT1_BACK
AUDIO_CHANNEL_OUT_5POINT1_SIDE,
AUDIO_CHANNEL_OUT_6POINT1,
AUDIO_CHANNEL_OUT_5POINT1POINT2,
@@ -62,58 +60,32 @@
static constexpr size_t kFrameCount = 1000;
/*
-Pixel 3XL
-downmix_benchmark:
- #BM_Downmix/0 4723 ns 4708 ns 148694
- #BM_Downmix/1 4717 ns 4702 ns 148873
- #BM_Downmix/2 4803 ns 4788 ns 145893
- #BM_Downmix/3 5056 ns 5041 ns 139110
- #BM_Downmix/4 4710 ns 4696 ns 149625
- #BM_Downmix/5 1514 ns 1509 ns 463694
- #BM_Downmix/6 1513 ns 1509 ns 463451
- #BM_Downmix/7 1516 ns 1511 ns 463899
- #BM_Downmix/8 4445 ns 4431 ns 157831
- #BM_Downmix/9 5081 ns 5065 ns 138412
- #BM_Downmix/10 4354 ns 4341 ns 161247
- #BM_Downmix/11 4411 ns 4397 ns 158893
- #BM_Downmix/12 4434 ns 4420 ns 157992
- #BM_Downmix/13 4845 ns 4830 ns 144873
- #BM_Downmix/14 4851 ns 4835 ns 144954
- #BM_Downmix/15 4884 ns 4870 ns 144233
- #BM_Downmix/16 5832 ns 5813 ns 120565
- #BM_Downmix/17 5241 ns 5224 ns 133927
- #BM_Downmix/18 5044 ns 5028 ns 139131
- #BM_Downmix/19 5244 ns 5227 ns 132315
- #BM_Downmix/20 5943 ns 5923 ns 117759
- #BM_Downmix/21 5990 ns 5971 ns 117263
- #BM_Downmix/22 4468 ns 4454 ns 156689
- #BM_Downmix/23 7306 ns 7286 ns 95911
---
-downmix_benchmark: (generic fold)
- #BM_Downmix/0 4722 ns 4707 ns 149847
- #BM_Downmix/1 4714 ns 4698 ns 148748
- #BM_Downmix/2 4794 ns 4779 ns 145661
- #BM_Downmix/3 5053 ns 5035 ns 139172
- #BM_Downmix/4 4695 ns 4678 ns 149762
- #BM_Downmix/5 4381 ns 4368 ns 159675
- #BM_Downmix/6 4387 ns 4373 ns 160267
- #BM_Downmix/7 4732 ns 4717 ns 148514
- #BM_Downmix/8 4430 ns 4415 ns 158133
- #BM_Downmix/9 5101 ns 5084 ns 138353
- #BM_Downmix/10 4356 ns 4343 ns 160821
- #BM_Downmix/11 4397 ns 4383 ns 159995
- #BM_Downmix/12 4438 ns 4424 ns 158117
- #BM_Downmix/13 5243 ns 5226 ns 133863
- #BM_Downmix/14 5259 ns 5242 ns 131855
- #BM_Downmix/15 5245 ns 5228 ns 133686
- #BM_Downmix/16 5829 ns 5809 ns 120543
- #BM_Downmix/17 5245 ns 5228 ns 133533
- #BM_Downmix/18 5935 ns 5916 ns 118282
- #BM_Downmix/19 5263 ns 5245 ns 133657
- #BM_Downmix/20 5998 ns 5978 ns 114693
- #BM_Downmix/21 5989 ns 5969 ns 117450
- #BM_Downmix/22 4442 ns 4431 ns 157913
- #BM_Downmix/23 7309 ns 7290 ns 95797
+Pixel 4XL
+--------------------------------------------------------
+Benchmark Time CPU Iterations
+--------------------------------------------------------
+BM_Downmix/0 2845 ns 2839 ns 246585 AUDIO_CHANNEL_OUT_MONO
+BM_Downmix/1 2844 ns 2838 ns 246599
+BM_Downmix/2 3727 ns 3719 ns 188227 AUDIO_CHANNEL_OUT_STEREO
+BM_Downmix/3 4609 ns 4600 ns 152148 AUDIO_CHANNEL_OUT_2POINT1
+BM_Downmix/4 3727 ns 3719 ns 188228 AUDIO_CHANNEL_OUT_2POINT0POINT2
+BM_Downmix/5 1787 ns 1784 ns 392384 AUDIO_CHANNEL_OUT_QUAD
+BM_Downmix/6 1787 ns 1783 ns 392527 AUDIO_CHANNEL_OUT_QUAD_SIDE
+BM_Downmix/7 5493 ns 5481 ns 127740 AUDIO_CHANNEL_OUT_SURROUND
+BM_Downmix/8 4610 ns 4600 ns 152168 AUDIO_CHANNEL_OUT_2POINT1POINT2
+BM_Downmix/9 4610 ns 4600 ns 152162 AUDIO_CHANNEL_OUT_3POINT0POINT2
+BM_Downmix/10 6377 ns 6362 ns 110042 AUDIO_CHANNEL_OUT_PENTA
+BM_Downmix/11 5493 ns 5481 ns 127683 AUDIO_CHANNEL_OUT_3POINT1POINT2
+BM_Downmix/12 2758 ns 2752 ns 251488 AUDIO_CHANNEL_OUT_5POINT1
+BM_Downmix/13 2683 ns 2677 ns 261421 AUDIO_CHANNEL_OUT_5POINT1_SIDE
+BM_Downmix/14 8141 ns 8124 ns 86157 AUDIO_CHANNEL_OUT_6POINT1
+BM_Downmix/15 7265 ns 7249 ns 96554 AUDIO_CHANNEL_OUT_5POINT1POINT2
+BM_Downmix/16 3158 ns 3151 ns 222188 AUDIO_CHANNEL_OUT_7POINT1
+BM_Downmix/17 7291 ns 7276 ns 96226 AUDIO_CHANNEL_OUT_5POINT1POINT4
+BM_Downmix/18 9050 ns 9031 ns 77512 AUDIO_CHANNEL_OUT_7POINT1POINT2
+BM_Downmix/19 9056 ns 9036 ns 77467 AUDIO_CHANNEL_OUT_7POINT1POINT4
+BM_Downmix/20 6426 ns 6412 ns 109164 AUDIO_CHANNEL_OUT_13POINT_360RA
+BM_Downmix/21 11743 ns 11716 ns 59762 AUDIO_CHANNEL_OUT_22POINT2
*/
static void BM_Downmix(benchmark::State& state) {
@@ -125,7 +97,7 @@
std::minstd_rand gen(channelMask);
std::uniform_real_distribution<> dis(-1.0f, 1.0f);
std::vector<float> input(kFrameCount * channelCount);
- std::vector<float> output(kFrameCount * 2);
+ std::vector<float> output(kFrameCount * FCC_2);
for (auto& in : input) {
in = dis(gen);
}
@@ -187,7 +159,8 @@
benchmark::ClobberMemory();
}
- state.SetComplexityN(state.range(0));
+ state.SetComplexityN(channelCount);
+ state.SetLabel(audio_channel_out_mask_to_string(channelMask));
if (int status = AUDIO_EFFECT_LIBRARY_INFO_SYM.release_effect(effectHandle); status != 0) {
ALOGE("release_effect returned an error = %d\n", status);
diff --git a/media/libeffects/downmix/tests/downmix_tests.cpp b/media/libeffects/downmix/tests/downmix_tests.cpp
index d4b7a3a..636d8a0 100644
--- a/media/libeffects/downmix/tests/downmix_tests.cpp
+++ b/media/libeffects/downmix/tests/downmix_tests.cpp
@@ -33,16 +33,14 @@
AUDIO_CHANNEL_OUT_STEREO,
AUDIO_CHANNEL_OUT_2POINT1,
AUDIO_CHANNEL_OUT_2POINT0POINT2,
- AUDIO_CHANNEL_OUT_QUAD,
- AUDIO_CHANNEL_OUT_QUAD_BACK,
+ AUDIO_CHANNEL_OUT_QUAD, // AUDIO_CHANNEL_OUT_QUAD_BACK
AUDIO_CHANNEL_OUT_QUAD_SIDE,
AUDIO_CHANNEL_OUT_SURROUND,
AUDIO_CHANNEL_OUT_2POINT1POINT2,
AUDIO_CHANNEL_OUT_3POINT0POINT2,
AUDIO_CHANNEL_OUT_PENTA,
AUDIO_CHANNEL_OUT_3POINT1POINT2,
- AUDIO_CHANNEL_OUT_5POINT1,
- AUDIO_CHANNEL_OUT_5POINT1_BACK,
+ AUDIO_CHANNEL_OUT_5POINT1, // AUDIO_CHANNEL_OUT_5POINT1_BACK
AUDIO_CHANNEL_OUT_5POINT1_SIDE,
AUDIO_CHANNEL_OUT_6POINT1,
AUDIO_CHANNEL_OUT_5POINT1POINT2,
@@ -57,6 +55,33 @@
static constexpr audio_channel_mask_t kConsideredChannels =
(audio_channel_mask_t)(AUDIO_CHANNEL_OUT_7POINT1 | AUDIO_CHANNEL_OUT_BACK_CENTER);
+constexpr inline float kScaleFromChannelIdx[] = {
+ 1.f, // AUDIO_CHANNEL_OUT_FRONT_LEFT = 0x1u,
+ 1.f, // AUDIO_CHANNEL_OUT_FRONT_RIGHT = 0x2u,
+ M_SQRT1_2, // AUDIO_CHANNEL_OUT_FRONT_CENTER = 0x4u,
+ 0.5f, // AUDIO_CHANNEL_OUT_LOW_FREQUENCY = 0x8u,
+ M_SQRT1_2, // AUDIO_CHANNEL_OUT_BACK_LEFT = 0x10u,
+ M_SQRT1_2, // AUDIO_CHANNEL_OUT_BACK_RIGHT = 0x20u,
+ 0, // AUDIO_CHANNEL_OUT_FRONT_LEFT_OF_CENTER = 0x40u,
+ 0, // AUDIO_CHANNEL_OUT_FRONT_RIGHT_OF_CENTER = 0x80u,
+ 0.5f, // AUDIO_CHANNEL_OUT_BACK_CENTER = 0x100u,
+ M_SQRT1_2, // AUDIO_CHANNEL_OUT_SIDE_LEFT = 0x200u,
+ M_SQRT1_2, // AUDIO_CHANNEL_OUT_SIDE_RIGHT = 0x400u,
+ 0, // AUDIO_CHANNEL_OUT_TOP_CENTER = 0x800u,
+ 0, // AUDIO_CHANNEL_OUT_TOP_FRONT_LEFT = 0x1000u,
+ 0, // AUDIO_CHANNEL_OUT_TOP_FRONT_CENTER = 0x2000u,
+ 0, // AUDIO_CHANNEL_OUT_TOP_FRONT_RIGHT = 0x4000u,
+ 0, // AUDIO_CHANNEL_OUT_TOP_BACK_LEFT = 0x8000u,
+ 0, // AUDIO_CHANNEL_OUT_TOP_BACK_CENTER = 0x10000u,
+ 0, // AUDIO_CHANNEL_OUT_TOP_BACK_RIGHT = 0x20000u,
+ 0, // AUDIO_CHANNEL_OUT_TOP_SIDE_LEFT = 0x40000u,
+ 0, // AUDIO_CHANNEL_OUT_TOP_SIDE_RIGHT = 0x80000u,
+ 0, // AUDIO_CHANNEL_OUT_BOTTOM_FRONT_LEFT = 0x100000u,
+ 0, // AUDIO_CHANNEL_OUT_BOTTOM_FRONT_CENTER = 0x200000u,
+ 0, // AUDIO_CHANNEL_OUT_BOTTOM_FRONT_RIGHT = 0x400000u,
+ 0, // AUDIO_CHANNEL_OUT_LOW_FREQUENCY_2 = 0x800000u,
+};
+
// Downmix doesn't change with sample rate
static constexpr size_t kSampleRates[] = {
48000,
@@ -93,8 +118,8 @@
void testBalance(int sampleRate, audio_channel_mask_t channelMask) {
using namespace ::android::audio_utils::channels;
- size_t frames = 100;
- unsigned outChannels = 2;
+ size_t frames = 100; // set to an even number (2, 4, 6 ... ) stream alternates +1, -1.
+ constexpr unsigned outChannels = 2;
unsigned inChannels = audio_channel_count_from_out_mask(channelMask);
std::vector<float> input(frames * inChannels);
std::vector<float> output(frames * outChannels);
@@ -119,7 +144,7 @@
auto stats = channelStatistics(output, 2 /* channels */);
// printf("power: %s %s\n", stats[0].toString().c_str(), stats[1].toString().c_str());
- double power[2] = { stats[0].getVariance(), stats[1].getVariance() };
+ double power[2] = { stats[0].getPopVariance(), stats[1].getPopVariance() };
// Check symmetric power for pair channels on exchange of left/right position.
// to do this, we save previous power measurements.
@@ -139,20 +164,21 @@
EXPECT_EQ(0.f, power[1]);
continue;
}
- constexpr float POWER_TOLERANCE = 0.01; // for variance sum error.
+
+ constexpr float POWER_TOLERANCE = 0.001;
+ const float expectedPower = kScaleFromChannelIdx[index] * kScaleFromChannelIdx[index];
switch (side) {
case AUDIO_GEOMETRY_SIDE_LEFT:
- EXPECT_NEAR(0.25f, power[0], POWER_TOLERANCE);
- EXPECT_EQ(0.f, power[1]);
+ EXPECT_EQ(0.f, power[1]); // always true
+ EXPECT_NEAR(expectedPower, power[0], POWER_TOLERANCE);
break;
case AUDIO_GEOMETRY_SIDE_RIGHT:
- EXPECT_EQ(0.f, power[0]);
- EXPECT_NEAR(0.25f, power[1], POWER_TOLERANCE);
+ EXPECT_EQ(0.f, power[0]); // always true
+ EXPECT_NEAR(expectedPower, power[1], POWER_TOLERANCE);
break;
case AUDIO_GEOMETRY_SIDE_CENTER:
- EXPECT_NEAR(0.125f, power[0], POWER_TOLERANCE);
- EXPECT_NEAR(0.125f, power[1], POWER_TOLERANCE);
- EXPECT_NEAR_EPSILON(power[0], power[1]);
+ EXPECT_NEAR_EPSILON(power[0], power[1]); // always true
+ EXPECT_NEAR(expectedPower, power[0], POWER_TOLERANCE);
break;
}
}
@@ -244,10 +270,11 @@
::testing::Combine(
::testing::Range(0, (int)std::size(kSampleRates)),
::testing::Range(0, (int)std::size(kChannelPositionMasks))
- ));
-
-int main(int argc, /* const */ char** argv) {
- ::testing::InitGoogleTest(&argc, argv);
- int status = RUN_ALL_TESTS();
- return status;
-}
+ ),
+ [](const testing::TestParamInfo<DownmixTest::ParamType>& info) {
+ const int index = std::get<1>(info.param);
+ const audio_channel_mask_t channelMask = kChannelPositionMasks[index];
+ const std::string name = std::string(audio_channel_out_mask_to_string(channelMask))
+ + "_" + std::to_string(std::get<0>(info.param)) + "_" + std::to_string(index);
+ return name;
+ });
diff --git a/media/libeffects/hapticgenerator/Android.bp b/media/libeffects/hapticgenerator/Android.bp
index a660957..03ce329 100644
--- a/media/libeffects/hapticgenerator/Android.bp
+++ b/media/libeffects/hapticgenerator/Android.bp
@@ -45,6 +45,7 @@
shared_libs: [
"libaudioutils",
+ "libbase",
"libbinder",
"liblog",
"libutils",
diff --git a/media/libeffects/hapticgenerator/EffectHapticGenerator.cpp b/media/libeffects/hapticgenerator/EffectHapticGenerator.cpp
index 65a20a7..3137e13 100644
--- a/media/libeffects/hapticgenerator/EffectHapticGenerator.cpp
+++ b/media/libeffects/hapticgenerator/EffectHapticGenerator.cpp
@@ -22,12 +22,15 @@
#include <algorithm>
#include <memory>
+#include <string>
#include <utility>
#include <errno.h>
#include <inttypes.h>
#include <math.h>
+#include <android-base/parsedouble.h>
+#include <android-base/properties.h>
#include <audio_effects/effect_hapticgenerator.h>
#include <audio_utils/format.h>
#include <system/audio.h>
@@ -35,6 +38,7 @@
static constexpr float DEFAULT_RESONANT_FREQUENCY = 150.0f;
static constexpr float DEFAULT_BSF_ZERO_Q = 8.0f;
static constexpr float DEFAULT_BSF_POLE_Q = 4.0f;
+static constexpr float DEFAULT_DISTORTION_OUTPUT_GAIN = 1.5f;
// This is the only symbol that needs to be exported
__attribute__ ((visibility ("default")))
@@ -81,6 +85,15 @@
namespace {
+float getFloatProperty(const std::string& key, float defaultValue) {
+ float result;
+ std::string value = android::base::GetProperty(key, "");
+ if (!value.empty() && android::base::ParseFloat(value, &result)) {
+ return result;
+ }
+ return defaultValue;
+}
+
int HapticGenerator_Init(struct HapticGeneratorContext *context) {
context->itfe = &gHapticGeneratorInterface;
@@ -114,7 +127,9 @@
context->param.distortionCornerFrequency = 300.0f;
context->param.distortionInputGain = 0.3f;
context->param.distortionCubeThreshold = 0.1f;
- context->param.distortionOutputGain = 1.5f;
+ context->param.distortionOutputGain = getFloatProperty(
+ "vendor.audio.hapticgenerator.distortion.output.gain", DEFAULT_DISTORTION_OUTPUT_GAIN);
+ ALOGD("Using distortion output gain as %f", context->param.distortionOutputGain);
context->state = HAPTICGENERATOR_STATE_INITIALIZED;
return 0;
@@ -287,15 +302,17 @@
break;
}
case HG_PARAM_VIBRATOR_INFO: {
- if (value == nullptr || size != 2 * sizeof(float)) {
+ if (value == nullptr || size != 3 * sizeof(float)) {
return -EINVAL;
}
const float resonantFrequency = *(float*) value;
const float qFactor = *((float *) value + 1);
+ const float maxAmplitude = *((float *) value + 2);
context->param.resonantFrequency =
isnan(resonantFrequency) ? DEFAULT_RESONANT_FREQUENCY : resonantFrequency;
context->param.bsfZeroQ = isnan(qFactor) ? DEFAULT_BSF_POLE_Q : qFactor;
context->param.bsfPoleQ = context->param.bsfZeroQ / 2.0f;
+ context->param.maxHapticAmplitude = maxAmplitude;
if (context->processorsRecord.bpf != nullptr) {
context->processorsRecord.bpf->setCoefficients(
@@ -448,7 +465,8 @@
float* hapticOutBuffer = HapticGenerator_runProcessingChain(
context->processingChain, context->inputBuffer.data(),
context->outputBuffer.data(), inBuffer->frameCount);
- os::scaleHapticData(hapticOutBuffer, hapticSampleCount, context->param.maxHapticIntensity);
+ os::scaleHapticData(hapticOutBuffer, hapticSampleCount, context->param.maxHapticIntensity,
+ context->param.maxHapticAmplitude);
// For haptic data, the haptic playback thread will copy the data from effect input buffer,
// which contains haptic data at the end of the buffer, directly to sink buffer.
diff --git a/media/libeffects/hapticgenerator/EffectHapticGenerator.h b/media/libeffects/hapticgenerator/EffectHapticGenerator.h
index 96b744a..85e961f 100644
--- a/media/libeffects/hapticgenerator/EffectHapticGenerator.h
+++ b/media/libeffects/hapticgenerator/EffectHapticGenerator.h
@@ -51,6 +51,7 @@
// A map from track id to haptic intensity.
std::map<int, os::HapticScale> id2Intensity;
os::HapticScale maxHapticIntensity; // max intensity will be used to scale haptic data.
+ float maxHapticAmplitude; // max amplitude will be used to limit haptic data absolute values.
float resonantFrequency;
float bpfQ;
diff --git a/media/libheadtracking/Android.bp b/media/libheadtracking/Android.bp
new file mode 100644
index 0000000..015fd7e
--- /dev/null
+++ b/media/libheadtracking/Android.bp
@@ -0,0 +1,78 @@
+package {
+ // See: http://go/android-license-faq
+ // A large-scale-change added 'default_applicable_licenses' to import
+ // all of the 'license_kinds' from "frameworks_av_license"
+ // to get the below license kinds:
+ // SPDX-license-identifier-Apache-2.0
+ default_applicable_licenses: ["frameworks_av_license"],
+}
+
+cc_library {
+ name: "libheadtracking",
+ host_supported: true,
+ srcs: [
+ "HeadTrackingProcessor.cpp",
+ "ModeSelector.cpp",
+ "Pose.cpp",
+ "PoseDriftCompensator.cpp",
+ "PoseRateLimiter.cpp",
+ "QuaternionUtil.cpp",
+ "ScreenHeadFusion.cpp",
+ "Twist.cpp",
+ ],
+ export_include_dirs: [
+ "include",
+ ],
+ header_libs: [
+ "libeigen",
+ ],
+ export_header_lib_headers: [
+ "libeigen",
+ ],
+}
+
+cc_library {
+ name: "libheadtracking-binding",
+ srcs: [
+ "SensorPoseProvider.cpp",
+ ],
+ shared_libs: [
+ "libheadtracking",
+ "libandroid",
+ "liblog",
+ "libsensor",
+ ],
+ export_shared_lib_headers: [
+ "libheadtracking",
+ ],
+}
+
+cc_binary {
+ name: "SensorPoseProvider-example",
+ srcs: [
+ "SensorPoseProvider-example.cpp",
+ ],
+ shared_libs: [
+ "libandroid",
+ "libheadtracking",
+ "libheadtracking-binding",
+ "libsensor",
+ ],
+}
+
+cc_test_host {
+ name: "libheadtracking-test",
+ srcs: [
+ "HeadTrackingProcessor-test.cpp",
+ "ModeSelector-test.cpp",
+ "Pose-test.cpp",
+ "PoseDriftCompensator-test.cpp",
+ "PoseRateLimiter-test.cpp",
+ "QuaternionUtil-test.cpp",
+ "ScreenHeadFusion-test.cpp",
+ "Twist-test.cpp",
+ ],
+ shared_libs: [
+ "libheadtracking",
+ ],
+}
diff --git a/media/libheadtracking/HeadTrackingProcessor-test.cpp b/media/libheadtracking/HeadTrackingProcessor-test.cpp
new file mode 100644
index 0000000..1739c6d
--- /dev/null
+++ b/media/libheadtracking/HeadTrackingProcessor-test.cpp
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "media/HeadTrackingProcessor.h"
+
+#include <gtest/gtest.h>
+
+#include "QuaternionUtil.h"
+#include "TestUtil.h"
+
+namespace android {
+namespace media {
+namespace {
+
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+using Options = HeadTrackingProcessor::Options;
+
+TEST(HeadTrackingProcessor, Initial) {
+ for (auto mode : {HeadTrackingMode::STATIC, HeadTrackingMode::WORLD_RELATIVE,
+ HeadTrackingMode::SCREEN_RELATIVE}) {
+ std::unique_ptr<HeadTrackingProcessor> processor =
+ createHeadTrackingProcessor(Options{}, mode);
+ processor->calculate(0);
+ EXPECT_EQ(processor->getActualMode(), HeadTrackingMode::STATIC);
+ EXPECT_EQ(processor->getHeadToStagePose(), Pose3f());
+ }
+}
+
+TEST(HeadTrackingProcessor, BasicComposition) {
+ const Pose3f worldToHead{{1, 2, 3}, Quaternionf::UnitRandom()};
+ const Pose3f worldToScreen{{4, 5, 6}, Quaternionf::UnitRandom()};
+ const Pose3f screenToStage{{7, 8, 9}, Quaternionf::UnitRandom()};
+ const float physicalToLogical = M_PI_2;
+
+ std::unique_ptr<HeadTrackingProcessor> processor =
+ createHeadTrackingProcessor(Options{}, HeadTrackingMode::SCREEN_RELATIVE);
+
+ // Establish a baseline for the drift compensators.
+ processor->setWorldToHeadPose(0, Pose3f(), Twist3f());
+ processor->setWorldToScreenPose(0, Pose3f());
+
+ processor->setWorldToHeadPose(0, worldToHead, Twist3f());
+ processor->setWorldToScreenPose(0, worldToScreen);
+ processor->setScreenToStagePose(screenToStage);
+ processor->setDisplayOrientation(physicalToLogical);
+ processor->calculate(0);
+ ASSERT_EQ(processor->getActualMode(), HeadTrackingMode::SCREEN_RELATIVE);
+ EXPECT_EQ(processor->getHeadToStagePose(), worldToHead.inverse() * worldToScreen *
+ Pose3f(rotateY(-physicalToLogical)) *
+ screenToStage);
+
+ processor->setDesiredMode(HeadTrackingMode::WORLD_RELATIVE);
+ processor->calculate(0);
+ ASSERT_EQ(processor->getActualMode(), HeadTrackingMode::WORLD_RELATIVE);
+ EXPECT_EQ(processor->getHeadToStagePose(), worldToHead.inverse() * screenToStage);
+
+ processor->setDesiredMode(HeadTrackingMode::STATIC);
+ processor->calculate(0);
+ ASSERT_EQ(processor->getActualMode(), HeadTrackingMode::STATIC);
+ EXPECT_EQ(processor->getHeadToStagePose(), screenToStage);
+}
+
+TEST(HeadTrackingProcessor, Prediction) {
+ const Pose3f worldToHead{{1, 2, 3}, Quaternionf::UnitRandom()};
+ const Twist3f headTwist{{4, 5, 6}, quaternionToRotationVector(Quaternionf::UnitRandom()) / 10};
+ const Pose3f worldToScreen{{4, 5, 6}, Quaternionf::UnitRandom()};
+
+ std::unique_ptr<HeadTrackingProcessor> processor = createHeadTrackingProcessor(
+ Options{.predictionDuration = 2.f}, HeadTrackingMode::WORLD_RELATIVE);
+
+ // Establish a baseline for the drift compensators.
+ processor->setWorldToHeadPose(0, Pose3f(), Twist3f());
+ processor->setWorldToScreenPose(0, Pose3f());
+
+ processor->setWorldToHeadPose(0, worldToHead, headTwist);
+ processor->setWorldToScreenPose(0, worldToScreen);
+ processor->calculate(0);
+ ASSERT_EQ(processor->getActualMode(), HeadTrackingMode::WORLD_RELATIVE);
+ EXPECT_EQ(processor->getHeadToStagePose(), (worldToHead * integrate(headTwist, 2.f)).inverse());
+
+ processor->setDesiredMode(HeadTrackingMode::SCREEN_RELATIVE);
+ processor->calculate(0);
+ ASSERT_EQ(processor->getActualMode(), HeadTrackingMode::SCREEN_RELATIVE);
+ EXPECT_EQ(processor->getHeadToStagePose(),
+ (worldToHead * integrate(headTwist, 2.f)).inverse() * worldToScreen);
+
+ processor->setDesiredMode(HeadTrackingMode::STATIC);
+ processor->calculate(0);
+ ASSERT_EQ(processor->getActualMode(), HeadTrackingMode::STATIC);
+ EXPECT_EQ(processor->getHeadToStagePose(), Pose3f());
+}
+
+TEST(HeadTrackingProcessor, SmoothModeSwitch) {
+ const Pose3f targetHeadToWorld = Pose3f({4, 0, 0}, rotateZ(M_PI / 2));
+
+ std::unique_ptr<HeadTrackingProcessor> processor = createHeadTrackingProcessor(
+ Options{.maxTranslationalVelocity = 1}, HeadTrackingMode::STATIC);
+
+ // Establish a baseline for the drift compensators.
+ processor->setWorldToHeadPose(0, Pose3f(), Twist3f());
+ processor->setWorldToScreenPose(0, Pose3f());
+
+ processor->calculate(0);
+
+ processor->setDesiredMode(HeadTrackingMode::WORLD_RELATIVE);
+ processor->setWorldToHeadPose(0, targetHeadToWorld.inverse(), Twist3f());
+
+ // We're expecting a gradual move to the target.
+ processor->calculate(0);
+ EXPECT_EQ(HeadTrackingMode::WORLD_RELATIVE, processor->getActualMode());
+ EXPECT_EQ(processor->getHeadToStagePose(), Pose3f());
+
+ processor->calculate(2);
+ EXPECT_EQ(HeadTrackingMode::WORLD_RELATIVE, processor->getActualMode());
+ EXPECT_EQ(processor->getHeadToStagePose(), Pose3f({2, 0, 0}, rotateZ(M_PI / 4)));
+
+ processor->calculate(4);
+ EXPECT_EQ(HeadTrackingMode::WORLD_RELATIVE, processor->getActualMode());
+ EXPECT_EQ(processor->getHeadToStagePose(), targetHeadToWorld);
+
+ // Now that we've reached the target, we should no longer be rate limiting.
+ processor->setWorldToHeadPose(4, Pose3f(), Twist3f());
+ processor->calculate(5);
+ EXPECT_EQ(HeadTrackingMode::WORLD_RELATIVE, processor->getActualMode());
+ EXPECT_EQ(processor->getHeadToStagePose(), Pose3f());
+}
+
+} // namespace
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/HeadTrackingProcessor.cpp b/media/libheadtracking/HeadTrackingProcessor.cpp
new file mode 100644
index 0000000..ee60fa5
--- /dev/null
+++ b/media/libheadtracking/HeadTrackingProcessor.cpp
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"){}
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "media/HeadTrackingProcessor.h"
+
+#include "ModeSelector.h"
+#include "PoseDriftCompensator.h"
+#include "QuaternionUtil.h"
+#include "ScreenHeadFusion.h"
+
+namespace android {
+namespace media {
+namespace {
+
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+
+class HeadTrackingProcessorImpl : public HeadTrackingProcessor {
+ public:
+ HeadTrackingProcessorImpl(const Options& options, HeadTrackingMode initialMode)
+ : mOptions(options),
+ mHeadPoseDriftCompensator(PoseDriftCompensator::Options{
+ .translationalDriftTimeConstant = options.translationalDriftTimeConstant,
+ .rotationalDriftTimeConstant = options.rotationalDriftTimeConstant,
+ }),
+ mScreenPoseDriftCompensator(PoseDriftCompensator::Options{
+ .translationalDriftTimeConstant = options.translationalDriftTimeConstant,
+ .rotationalDriftTimeConstant = options.rotationalDriftTimeConstant,
+ }),
+ mModeSelector(ModeSelector::Options{.freshnessTimeout = options.freshnessTimeout},
+ initialMode),
+ mRateLimiter(PoseRateLimiter::Options{
+ .maxTranslationalVelocity = options.maxTranslationalVelocity,
+ .maxRotationalVelocity = options.maxRotationalVelocity}) {}
+
+ void setDesiredMode(HeadTrackingMode mode) override { mModeSelector.setDesiredMode(mode); }
+
+ void setWorldToHeadPose(int64_t timestamp, const Pose3f& worldToHead,
+ const Twist3f& headTwist) override {
+ Pose3f predictedWorldToHead =
+ worldToHead * integrate(headTwist, mOptions.predictionDuration);
+ mHeadPoseDriftCompensator.setInput(timestamp, predictedWorldToHead);
+ mWorldToHeadTimestamp = timestamp;
+ }
+
+ void setWorldToScreenPose(int64_t timestamp, const Pose3f& worldToScreen) override {
+ mScreenPoseDriftCompensator.setInput(timestamp, worldToScreen);
+ mWorldToScreenTimestamp = timestamp;
+ }
+
+ void setScreenToStagePose(const Pose3f& screenToStage) override {
+ mModeSelector.setScreenToStagePose(screenToStage);
+ }
+
+ void setDisplayOrientation(float physicalToLogicalAngle) override {
+ if (mPhysicalToLogicalAngle != physicalToLogicalAngle) {
+ mRateLimiter.enable();
+ }
+ mPhysicalToLogicalAngle = physicalToLogicalAngle;
+ }
+
+ void calculate(int64_t timestamp) override {
+ if (mWorldToHeadTimestamp.has_value()) {
+ const Pose3f worldToHead = mHeadPoseDriftCompensator.getOutput();
+ mScreenHeadFusion.setWorldToHeadPose(mWorldToHeadTimestamp.value(), worldToHead);
+ mModeSelector.setWorldToHeadPose(mWorldToHeadTimestamp.value(), worldToHead);
+ }
+
+ if (mWorldToScreenTimestamp.has_value()) {
+ const Pose3f worldToLogicalScreen = mScreenPoseDriftCompensator.getOutput() *
+ Pose3f(rotateY(-mPhysicalToLogicalAngle));
+ mScreenHeadFusion.setWorldToScreenPose(mWorldToScreenTimestamp.value(),
+ worldToLogicalScreen);
+ }
+
+ auto maybeScreenToHead = mScreenHeadFusion.calculate();
+ if (maybeScreenToHead.has_value()) {
+ mModeSelector.setScreenToHeadPose(maybeScreenToHead->timestamp,
+ maybeScreenToHead->pose);
+ } else {
+ mModeSelector.setScreenToHeadPose(timestamp, std::nullopt);
+ }
+
+ HeadTrackingMode prevMode = mModeSelector.getActualMode();
+ mModeSelector.calculate(timestamp);
+ if (mModeSelector.getActualMode() != prevMode) {
+ // Mode has changed, enable rate limiting.
+ mRateLimiter.enable();
+ }
+ mRateLimiter.setTarget(mModeSelector.getHeadToStagePose());
+ mHeadToStagePose = mRateLimiter.calculatePose(timestamp);
+ }
+
+ Pose3f getHeadToStagePose() const override { return mHeadToStagePose; }
+
+ HeadTrackingMode getActualMode() const override { return mModeSelector.getActualMode(); }
+
+ void recenter() override {
+ mHeadPoseDriftCompensator.recenter();
+ mScreenPoseDriftCompensator.recenter();
+ mRateLimiter.enable();
+ }
+
+ private:
+ const Options mOptions;
+ float mPhysicalToLogicalAngle = 0;
+ std::optional<int64_t> mWorldToHeadTimestamp;
+ std::optional<int64_t> mWorldToScreenTimestamp;
+ Pose3f mHeadToStagePose;
+ PoseDriftCompensator mHeadPoseDriftCompensator;
+ PoseDriftCompensator mScreenPoseDriftCompensator;
+ ScreenHeadFusion mScreenHeadFusion;
+ ModeSelector mModeSelector;
+ PoseRateLimiter mRateLimiter;
+};
+
+} // namespace
+
+std::unique_ptr<HeadTrackingProcessor> createHeadTrackingProcessor(
+ const HeadTrackingProcessor::Options& options, HeadTrackingMode initialMode) {
+ return std::make_unique<HeadTrackingProcessorImpl>(options, initialMode);
+}
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/ModeSelector-test.cpp b/media/libheadtracking/ModeSelector-test.cpp
new file mode 100644
index 0000000..6247d84
--- /dev/null
+++ b/media/libheadtracking/ModeSelector-test.cpp
@@ -0,0 +1,149 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ModeSelector.h"
+
+#include <gtest/gtest.h>
+
+#include "QuaternionUtil.h"
+#include "TestUtil.h"
+
+namespace android {
+namespace media {
+namespace {
+
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+
+TEST(ModeSelector, Initial) {
+ ModeSelector::Options options;
+ ModeSelector selector(options);
+
+ selector.calculate(0);
+ EXPECT_EQ(HeadTrackingMode::STATIC, selector.getActualMode());
+ EXPECT_EQ(selector.getHeadToStagePose(), Pose3f());
+}
+
+TEST(ModeSelector, InitialWorldRelative) {
+ const Pose3f worldToHead({1, 2, 3}, Quaternionf::UnitRandom());
+
+ ModeSelector::Options options;
+ ModeSelector selector(options, HeadTrackingMode::WORLD_RELATIVE);
+
+ selector.setWorldToHeadPose(0, worldToHead);
+ selector.calculate(0);
+ EXPECT_EQ(HeadTrackingMode::WORLD_RELATIVE, selector.getActualMode());
+ EXPECT_EQ(selector.getHeadToStagePose(), worldToHead.inverse());
+}
+
+TEST(ModeSelector, InitialScreenRelative) {
+ const Pose3f screenToHead({1, 2, 3}, Quaternionf::UnitRandom());
+
+ ModeSelector::Options options;
+ ModeSelector selector(options, HeadTrackingMode::SCREEN_RELATIVE);
+
+ selector.setScreenToHeadPose(0, screenToHead);
+ selector.calculate(0);
+ EXPECT_EQ(HeadTrackingMode::SCREEN_RELATIVE, selector.getActualMode());
+ EXPECT_EQ(selector.getHeadToStagePose(), screenToHead.inverse());
+}
+
+TEST(ModeSelector, WorldRelative) {
+ const Pose3f worldToHead({1, 2, 3}, Quaternionf::UnitRandom());
+ const Pose3f screenToStage({4, 5, 6}, Quaternionf::UnitRandom());
+
+ ModeSelector::Options options;
+ ModeSelector selector(options);
+
+ selector.setScreenToStagePose(screenToStage);
+
+ selector.setDesiredMode(HeadTrackingMode::WORLD_RELATIVE);
+ selector.setWorldToHeadPose(0, worldToHead);
+ selector.calculate(0);
+ EXPECT_EQ(HeadTrackingMode::WORLD_RELATIVE, selector.getActualMode());
+ EXPECT_EQ(selector.getHeadToStagePose(), worldToHead.inverse() * screenToStage);
+}
+
+TEST(ModeSelector, WorldRelativeStale) {
+ const Pose3f worldToHead({1, 2, 3}, Quaternionf::UnitRandom());
+ const Pose3f screenToStage({4, 5, 6}, Quaternionf::UnitRandom());
+
+ ModeSelector::Options options{.freshnessTimeout = 100};
+ ModeSelector selector(options);
+
+ selector.setScreenToStagePose(screenToStage);
+
+ selector.setDesiredMode(HeadTrackingMode::WORLD_RELATIVE);
+ selector.setWorldToHeadPose(0, worldToHead);
+ selector.calculate(101);
+ EXPECT_EQ(HeadTrackingMode::STATIC, selector.getActualMode());
+ EXPECT_EQ(selector.getHeadToStagePose(), screenToStage);
+}
+
+TEST(ModeSelector, ScreenRelative) {
+ const Pose3f screenToHead({1, 2, 3}, Quaternionf::UnitRandom());
+ const Pose3f screenToStage({4, 5, 6}, Quaternionf::UnitRandom());
+
+ ModeSelector::Options options;
+ ModeSelector selector(options);
+
+ selector.setScreenToStagePose(screenToStage);
+
+ selector.setDesiredMode(HeadTrackingMode::SCREEN_RELATIVE);
+ selector.setScreenToHeadPose(0, screenToHead);
+ selector.calculate(0);
+ EXPECT_EQ(HeadTrackingMode::SCREEN_RELATIVE, selector.getActualMode());
+ EXPECT_EQ(selector.getHeadToStagePose(), screenToHead.inverse() * screenToStage);
+}
+
+TEST(ModeSelector, ScreenRelativeStaleToWorldRelative) {
+ const Pose3f screenToHead({1, 2, 3}, Quaternionf::UnitRandom());
+ const Pose3f screenToStage({4, 5, 6}, Quaternionf::UnitRandom());
+ const Pose3f worldToHead({7, 8, 9}, Quaternionf::UnitRandom());
+
+ ModeSelector::Options options{.freshnessTimeout = 100};
+ ModeSelector selector(options);
+
+ selector.setScreenToStagePose(screenToStage);
+
+ selector.setDesiredMode(HeadTrackingMode::SCREEN_RELATIVE);
+ selector.setScreenToHeadPose(0, screenToHead);
+ selector.setWorldToHeadPose(50, worldToHead);
+ selector.calculate(101);
+ EXPECT_EQ(HeadTrackingMode::WORLD_RELATIVE, selector.getActualMode());
+ EXPECT_EQ(selector.getHeadToStagePose(), worldToHead.inverse() * screenToStage);
+}
+
+TEST(ModeSelector, ScreenRelativeInvalidToWorldRelative) {
+ const Pose3f screenToStage({4, 5, 6}, Quaternionf::UnitRandom());
+ const Pose3f worldToHead({7, 8, 9}, Quaternionf::UnitRandom());
+
+ ModeSelector::Options options;
+ ModeSelector selector(options);
+
+ selector.setScreenToStagePose(screenToStage);
+
+ selector.setDesiredMode(HeadTrackingMode::SCREEN_RELATIVE);
+ selector.setScreenToHeadPose(50, std::nullopt);
+ selector.setWorldToHeadPose(50, worldToHead);
+ selector.calculate(101);
+ EXPECT_EQ(HeadTrackingMode::WORLD_RELATIVE, selector.getActualMode());
+ EXPECT_EQ(selector.getHeadToStagePose(), worldToHead.inverse() * screenToStage);
+}
+
+} // namespace
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/ModeSelector.cpp b/media/libheadtracking/ModeSelector.cpp
new file mode 100644
index 0000000..16e1712
--- /dev/null
+++ b/media/libheadtracking/ModeSelector.cpp
@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ModeSelector.h"
+
+namespace android {
+namespace media {
+
+ModeSelector::ModeSelector(const Options& options, HeadTrackingMode initialMode)
+ : mOptions(options), mDesiredMode(initialMode), mActualMode(initialMode) {}
+
+void ModeSelector::setDesiredMode(HeadTrackingMode mode) {
+ mDesiredMode = mode;
+}
+
+void ModeSelector::setScreenToStagePose(const Pose3f& screenToStage) {
+ mScreenToStage = screenToStage;
+}
+
+void ModeSelector::setScreenToHeadPose(int64_t timestamp,
+ const std::optional<Pose3f>& screenToHead) {
+ mScreenToHead = screenToHead;
+ mScreenToHeadTimestamp = timestamp;
+}
+
+void ModeSelector::setWorldToHeadPose(int64_t timestamp, const Pose3f& worldToHead) {
+ mWorldToHead = worldToHead;
+ mWorldToHeadTimestamp = timestamp;
+}
+
+void ModeSelector::calculateActualMode(int64_t timestamp) {
+ bool isValidScreenToHead = mScreenToHead.has_value() &&
+ timestamp - mScreenToHeadTimestamp < mOptions.freshnessTimeout;
+ bool isValidWorldToHead = mWorldToHead.has_value() &&
+ timestamp - mWorldToHeadTimestamp < mOptions.freshnessTimeout;
+
+ HeadTrackingMode mode = mDesiredMode;
+
+ // Optional downgrade from screen-relative to world-relative.
+ if (mode == HeadTrackingMode::SCREEN_RELATIVE) {
+ if (!isValidScreenToHead) {
+ mode = HeadTrackingMode::WORLD_RELATIVE;
+ }
+ }
+
+ // Optional downgrade from world-relative to static.
+ if (mode == HeadTrackingMode::WORLD_RELATIVE) {
+ if (!isValidWorldToHead) {
+ mode = HeadTrackingMode::STATIC;
+ }
+ }
+
+ mActualMode = mode;
+}
+
+void ModeSelector::calculate(int64_t timestamp) {
+ calculateActualMode(timestamp);
+
+ switch (mActualMode) {
+ case HeadTrackingMode::STATIC:
+ mHeadToStage = mScreenToStage;
+ break;
+
+ case HeadTrackingMode::WORLD_RELATIVE:
+ mHeadToStage = mWorldToHead.value().inverse() * mScreenToStage;
+ break;
+
+ case HeadTrackingMode::SCREEN_RELATIVE:
+ mHeadToStage = mScreenToHead.value().inverse() * mScreenToStage;
+ break;
+ }
+}
+
+Pose3f ModeSelector::getHeadToStagePose() const {
+ return mHeadToStage;
+}
+
+HeadTrackingMode ModeSelector::getActualMode() const {
+ return mActualMode;
+}
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/ModeSelector.h b/media/libheadtracking/ModeSelector.h
new file mode 100644
index 0000000..17a5142
--- /dev/null
+++ b/media/libheadtracking/ModeSelector.h
@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <optional>
+
+#include "media/HeadTrackingMode.h"
+#include "media/Pose.h"
+
+#include "PoseRateLimiter.h"
+
+namespace android {
+namespace media {
+
+/**
+ * Head-tracking mode selector.
+ *
+ * This class is responsible for production of the determining pose for audio virtualization, based
+ * on a number of available sources and a selectable mode.
+ *
+ * Typical flow is:
+ * ModeSelector selector(...);
+ * while (...) {
+ * // Set inputs.
+ * selector.setFoo(...);
+ * selector.setBar(...);
+ *
+ * // Update outputs based on inputs.
+ * selector.calculate(...);
+ *
+ * // Get outputs.
+ * Pose3f pose = selector.getHeadToStagePose();
+ * }
+ *
+ * This class is not thread-safe, but thread-compatible.
+ *
+ * For details on the frames of reference involved, their composition and the definitions to the
+ * different modes, refer to:
+ * go/immersive-audio-frames
+ *
+ * The actual mode may deviate from the desired mode in the following cases:
+ * - When we cannot get a valid and fresh estimate of the screen-to-head pose, we will fall back
+ * from screen-relative to world-relative.
+ * - When we cannot get a fresh estimate of the world-to-head pose, we will fall back from
+ * world-relative to static.
+ *
+ * All the timestamps used here are of arbitrary units and origin. They just need to be consistent
+ * between all the calls and with the Options provided for determining freshness and rate limiting.
+ */
+class ModeSelector {
+ public:
+ struct Options {
+ int64_t freshnessTimeout = std::numeric_limits<int64_t>::max();
+ };
+
+ ModeSelector(const Options& options, HeadTrackingMode initialMode = HeadTrackingMode::STATIC);
+
+ /** Sets the desired head-tracking mode. */
+ void setDesiredMode(HeadTrackingMode mode);
+
+ /**
+ * Set the screen-to-stage pose, used in all modes.
+ */
+ void setScreenToStagePose(const Pose3f& screenToStage);
+
+ /**
+ * Set the screen-to-head pose, used in screen-relative mode.
+ * The timestamp needs to reflect how fresh the sample is (not necessarily which point in time
+ * it applies to). nullopt can be used if it is determined that the listener is not in front of
+ * the screen.
+ */
+ void setScreenToHeadPose(int64_t timestamp, const std::optional<Pose3f>& screenToHead);
+
+ /**
+ * Set the world-to-head pose, used in world-relative mode.
+ * The timestamp needs to reflect how fresh the sample is (not necessarily which point in time
+ * it applies to).
+ */
+ void setWorldToHeadPose(int64_t timestamp, const Pose3f& worldToHead);
+
+ /**
+ * Process all the previous inputs and update the outputs.
+ */
+ void calculate(int64_t timestamp);
+
+ /**
+ * Get the aggregate head-to-stage pose (primary output of this module).
+ */
+ Pose3f getHeadToStagePose() const;
+
+ /**
+ * Get the actual head-tracking mode (which may deviate from the desired one as mentioned in the
+ * class documentation above).
+ */
+ HeadTrackingMode getActualMode() const;
+
+ private:
+ const Options mOptions;
+
+ HeadTrackingMode mDesiredMode;
+ Pose3f mScreenToStage;
+ std::optional<Pose3f> mScreenToHead;
+ int64_t mScreenToHeadTimestamp;
+ std::optional<Pose3f> mWorldToHead;
+ int64_t mWorldToHeadTimestamp;
+
+ HeadTrackingMode mActualMode;
+ Pose3f mHeadToStage;
+
+ void calculateActualMode(int64_t timestamp);
+};
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/OWNERS b/media/libheadtracking/OWNERS
new file mode 100644
index 0000000..e5d0370
--- /dev/null
+++ b/media/libheadtracking/OWNERS
@@ -0,0 +1,2 @@
+ytai@google.com
+elaurent@google.com
diff --git a/media/libheadtracking/Pose-test.cpp b/media/libheadtracking/Pose-test.cpp
new file mode 100644
index 0000000..a9e18ce
--- /dev/null
+++ b/media/libheadtracking/Pose-test.cpp
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "media/Pose.h"
+
+#include <gtest/gtest.h>
+
+#include "QuaternionUtil.h"
+#include "TestUtil.h"
+
+using android::media::Pose3f;
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+
+namespace android {
+namespace media {
+namespace {
+
+TEST(Pose, CtorDefault) {
+ Pose3f pose;
+ EXPECT_EQ(pose.translation(), Vector3f::Zero());
+ EXPECT_EQ(pose.rotation(), Quaternionf::Identity());
+}
+
+TEST(Pose, CtorRotation) {
+ Quaternionf rot = Quaternionf::UnitRandom();
+ Pose3f pose(rot);
+ EXPECT_EQ(pose.translation(), Vector3f::Zero());
+ EXPECT_EQ(pose.rotation(), rot);
+}
+
+TEST(Pose, CtorTranslation) {
+ Vector3f trans{1, 2, 3};
+ Pose3f pose(trans);
+ EXPECT_EQ(pose.translation(), trans);
+ EXPECT_EQ(pose.rotation(), Quaternionf::Identity());
+}
+
+TEST(Pose, CtorTranslationRotation) {
+ Quaternionf rot = Quaternionf::UnitRandom();
+ Vector3f trans{1, 2, 3};
+ Pose3f pose(trans, rot);
+ EXPECT_EQ(pose.translation(), trans);
+ EXPECT_EQ(pose.rotation(), rot);
+}
+
+TEST(Pose, Inverse) {
+ Pose3f pose({1, 2, 3}, Quaternionf::UnitRandom());
+ EXPECT_EQ(pose.inverse() * pose, Pose3f());
+ EXPECT_EQ(pose * pose.inverse(), Pose3f());
+}
+
+TEST(Pose, IsApprox) {
+ constexpr float eps = std::numeric_limits<float>::epsilon();
+
+ EXPECT_EQ(Pose3f({1, 2, 3}, rotationVectorToQuaternion({4, 5, 6})),
+ Pose3f({1 + eps, 2 + eps, 3 + eps},
+ rotationVectorToQuaternion({4 + eps, 5 + eps, 6 + eps})));
+
+ EXPECT_NE(Pose3f({1, 2, 3}, rotationVectorToQuaternion({4, 5, 6})),
+ Pose3f({1.01, 2, 3}, rotationVectorToQuaternion({4, 5, 6})));
+
+ EXPECT_NE(Pose3f({1, 2, 3}, rotationVectorToQuaternion({4, 5, 6})),
+ Pose3f({1, 2, 3}, rotationVectorToQuaternion({4.01, 5, 6})));
+}
+
+TEST(Pose, Compose) {
+ Pose3f p1({1, 2, 3}, rotateZ(M_PI_2));
+ Pose3f p2({4, 5, 6}, rotateX(M_PI_2));
+ Pose3f p3({-4, 6, 9}, p1.rotation() * p2.rotation());
+ EXPECT_EQ(p1 * p2, p3);
+}
+
+TEST(Pose, MoveWithRateLimit_NoLimit) {
+ Pose3f from({1, 1, 1}, Quaternionf::Identity());
+ Pose3f to({1, 1, 2}, rotateZ(M_PI_2));
+ auto result = moveWithRateLimit(from, to, 1, 10, 10);
+ EXPECT_EQ(std::get<0>(result), to);
+ EXPECT_FALSE(std::get<1>(result));
+}
+
+TEST(Pose, MoveWithRateLimit_TranslationLimit) {
+ Pose3f from({1, 1, 1}, Quaternionf::Identity());
+ Pose3f to({1, 1, 2}, rotateZ(M_PI_2));
+ auto result = moveWithRateLimit(from, to, 1, 0.5f, 10);
+ Pose3f expected({1, 1, 1.5f}, rotateZ(M_PI_4));
+ EXPECT_EQ(std::get<0>(result), expected);
+ EXPECT_TRUE(std::get<1>(result));
+}
+
+TEST(Pose, MoveWithRateLimit_RotationLimit) {
+ Pose3f from({1, 1, 1}, Quaternionf::Identity());
+ Pose3f to({1, 1, 2}, rotateZ(M_PI_2));
+ auto result = moveWithRateLimit(from, to, 1, 10, M_PI_4);
+ Pose3f expected({1, 1, 1.5f}, rotateZ(M_PI_4));
+ EXPECT_EQ(std::get<0>(result), expected);
+ EXPECT_TRUE(std::get<1>(result));
+}
+
+TEST(Pose, FloatVectorRoundTrip1) {
+ // Rotation vector magnitude must be less than Pi.
+ std::vector<float> vec = { 1, 2, 3, 0.4, 0.5, 0.6};
+ std::optional<Pose3f> pose = Pose3f::fromVector(vec);
+ ASSERT_TRUE(pose.has_value());
+ std::vector<float> reconstructed = pose->toVector();
+ EXPECT_EQ(vec, reconstructed);
+}
+
+TEST(Pose, FloatVectorRoundTrip2) {
+ Pose3f pose({1, 2, 3}, Quaternionf::UnitRandom());
+ std::vector<float> vec = pose.toVector();
+ std::optional<Pose3f> reconstructed = Pose3f::fromVector(vec);
+ ASSERT_TRUE(reconstructed.has_value());
+ EXPECT_EQ(pose, reconstructed.value());
+}
+
+TEST(Pose, FloatVectorInvalid) {
+ EXPECT_FALSE(Pose3f::fromVector({}).has_value());
+ EXPECT_FALSE(Pose3f::fromVector({1, 2, 3, 4, 5}).has_value());
+ EXPECT_FALSE(Pose3f::fromVector({1, 2, 3, 4, 5, 6, 7}).has_value());
+}
+
+} // namespace
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/Pose.cpp b/media/libheadtracking/Pose.cpp
new file mode 100644
index 0000000..47241ce
--- /dev/null
+++ b/media/libheadtracking/Pose.cpp
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "media/Pose.h"
+#include "media/Twist.h"
+#include "QuaternionUtil.h"
+
+namespace android {
+namespace media {
+
+using Eigen::Vector3f;
+
+std::optional<Pose3f> Pose3f::fromVector(const std::vector<float>& vec) {
+ if (vec.size() != 6) {
+ return std::nullopt;
+ }
+ return Pose3f({vec[0], vec[1], vec[2]}, rotationVectorToQuaternion({vec[3], vec[4], vec[5]}));
+}
+
+std::vector<float> Pose3f::toVector() const {
+ Eigen::Vector3f rot = quaternionToRotationVector(mRotation);
+ return {mTranslation[0], mTranslation[1], mTranslation[2], rot[0], rot[1], rot[2]};
+}
+
+std::tuple<Pose3f, bool> moveWithRateLimit(const Pose3f& from, const Pose3f& to, float t,
+ float maxTranslationalVelocity,
+ float maxRotationalVelocity) {
+ // Never rate limit if both limits are set to infinity.
+ if (isinf(maxTranslationalVelocity) && isinf(maxRotationalVelocity)) {
+ return {to, false};
+ }
+ // Always rate limit if t is 0 (required to avoid division by 0).
+ if (t == 0) {
+ return {from, true};
+ }
+
+ Pose3f fromToTo = from.inverse() * to;
+ Twist3f twist = differentiate(fromToTo, t);
+ float angularRotationalRatio = twist.scalarRotationalVelocity() / maxRotationalVelocity;
+ float translationalVelocityRatio =
+ twist.scalarTranslationalVelocity() / maxTranslationalVelocity;
+ float maxRatio = std::max(angularRotationalRatio, translationalVelocityRatio);
+ if (maxRatio <= 1) {
+ return {to, false};
+ }
+ return {from * integrate(twist, t / maxRatio), true};
+}
+
+std::ostream& operator<<(std::ostream& os, const Pose3f& pose) {
+ os << "translation: " << pose.translation().transpose()
+ << " quaternion: " << pose.rotation().coeffs().transpose();
+ return os;
+}
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/PoseDriftCompensator-test.cpp b/media/libheadtracking/PoseDriftCompensator-test.cpp
new file mode 100644
index 0000000..df0a05f
--- /dev/null
+++ b/media/libheadtracking/PoseDriftCompensator-test.cpp
@@ -0,0 +1,146 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+#include <cmath>
+
+#include "PoseDriftCompensator.h"
+#include "QuaternionUtil.h"
+#include "TestUtil.h"
+
+namespace android {
+namespace media {
+namespace {
+
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+using Options = PoseDriftCompensator::Options;
+
+TEST(PoseDriftCompensator, Initial) {
+ PoseDriftCompensator comp(Options{});
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+}
+
+TEST(PoseDriftCompensator, NoDrift) {
+ Pose3f pose1({1, 2, 3}, Quaternionf::UnitRandom());
+ Pose3f pose2({4, 5, 6}, Quaternionf::UnitRandom());
+ PoseDriftCompensator comp(Options{});
+
+ // First pose sets the baseline.
+ comp.setInput(1000, pose1);
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+
+ comp.setInput(2000, pose2);
+ EXPECT_EQ(comp.getOutput(), pose1.inverse() * pose2);
+
+ // Recentering resets the baseline.
+ comp.recenter();
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+
+ comp.setInput(3000, pose1);
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+
+ comp.setInput(4000, pose2);
+ EXPECT_EQ(comp.getOutput(), pose1.inverse() * pose2);
+}
+
+TEST(PoseDriftCompensator, NoDriftZeroTime) {
+ Pose3f pose1({1, 2, 3}, Quaternionf::UnitRandom());
+ Pose3f pose2({4, 5, 6}, Quaternionf::UnitRandom());
+ PoseDriftCompensator comp(Options{});
+
+ comp.setInput(1000, pose1);
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+
+ comp.setInput(1000, pose2);
+ EXPECT_EQ(comp.getOutput(), pose1.inverse() * pose2);
+
+ comp.recenter();
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+
+ comp.setInput(1000, pose1);
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+
+ comp.setInput(1000, pose2);
+ EXPECT_EQ(comp.getOutput(), pose1.inverse() * pose2);
+}
+
+TEST(PoseDriftCompensator, Asymptotic) {
+ Pose3f pose({1, 2, 3}, Quaternionf::UnitRandom());
+
+ PoseDriftCompensator comp(
+ Options{.translationalDriftTimeConstant = 1, .rotationalDriftTimeConstant = 1});
+
+ // Set the same pose for a long time.
+ for (int64_t t = 0; t < 1000; ++t) {
+ comp.setInput(t, pose);
+ }
+
+ // Output would have faded to approx. identity.
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+}
+
+TEST(PoseDriftCompensator, Fast) {
+ Pose3f pose1({1, 2, 3}, Quaternionf::UnitRandom());
+ Pose3f pose2({4, 5, 6}, Quaternionf::UnitRandom());
+ PoseDriftCompensator comp(
+ Options{.translationalDriftTimeConstant = 1e7, .rotationalDriftTimeConstant = 1e7});
+
+ comp.setInput(0, pose1);
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+
+ comp.setInput(1, pose2);
+ EXPECT_EQ(comp.getOutput(), pose1.inverse() * pose2);
+
+ comp.recenter();
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+
+ comp.setInput(2, pose1);
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+
+ comp.setInput(3, pose2);
+ EXPECT_EQ(comp.getOutput(), pose1.inverse() * pose2);
+}
+
+TEST(PoseDriftCompensator, Drift) {
+ Pose3f pose1({1, 2, 3}, rotateZ(-M_PI * 3 / 4));
+ PoseDriftCompensator comp(
+ Options{.translationalDriftTimeConstant = 500, .rotationalDriftTimeConstant = 1000});
+
+ // Establish a baseline.
+ comp.setInput(1000, Pose3f());
+
+ // Initial pose is used as is.
+ comp.setInput(1000, pose1);
+ EXPECT_EQ(comp.getOutput(), pose1);
+
+ // After 1000 ticks, our rotation should be exp(-1) and translation exp(-2) from identity.
+ comp.setInput(2000, pose1);
+ EXPECT_EQ(comp.getOutput(),
+ Pose3f(Vector3f{1, 2, 3} * std::expf(-2), rotateZ(-M_PI * 3 / 4 * std::expf(-1))));
+
+ // As long as the input stays the same, we'll continue to advance towards identity.
+ comp.setInput(3000, pose1);
+ EXPECT_EQ(comp.getOutput(),
+ Pose3f(Vector3f{1, 2, 3} * std::expf(-4), rotateZ(-M_PI * 3 / 4 * std::expf(-2))));
+
+ comp.recenter();
+ EXPECT_EQ(comp.getOutput(), Pose3f());
+}
+
+} // namespace
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/PoseDriftCompensator.cpp b/media/libheadtracking/PoseDriftCompensator.cpp
new file mode 100644
index 0000000..0e90cad
--- /dev/null
+++ b/media/libheadtracking/PoseDriftCompensator.cpp
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"){}
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "PoseDriftCompensator.h"
+
+#include <cmath>
+
+#include "QuaternionUtil.h"
+
+namespace android {
+namespace media {
+
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+
+PoseDriftCompensator::PoseDriftCompensator(const Options& options) : mOptions(options) {}
+
+void PoseDriftCompensator::setInput(int64_t timestamp, const Pose3f& input) {
+ if (mTimestamp.has_value()) {
+ // Avoid computation upon first input (only sets the initial state).
+ Pose3f prevInputToInput = mPrevInput.inverse() * input;
+ mOutput = scale(mOutput, timestamp - mTimestamp.value()) * prevInputToInput;
+ }
+ mPrevInput = input;
+ mTimestamp = timestamp;
+}
+
+void PoseDriftCompensator::recenter() {
+ mTimestamp.reset();
+ mOutput = Pose3f();
+}
+
+Pose3f PoseDriftCompensator::getOutput() const {
+ return mOutput;
+}
+
+Pose3f PoseDriftCompensator::scale(const Pose3f& pose, int64_t dt) {
+ // Translation.
+ Vector3f translation = pose.translation();
+ translation *= std::expf(-static_cast<float>(dt) / mOptions.translationalDriftTimeConstant);
+
+ // Rotation.
+ Vector3f rotationVec = quaternionToRotationVector(pose.rotation());
+ rotationVec *= std::expf(-static_cast<float>(dt) / mOptions.rotationalDriftTimeConstant);
+
+ return Pose3f(translation, rotationVectorToQuaternion(rotationVec));
+}
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/PoseDriftCompensator.h b/media/libheadtracking/PoseDriftCompensator.h
new file mode 100644
index 0000000..a71483b
--- /dev/null
+++ b/media/libheadtracking/PoseDriftCompensator.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <optional>
+
+#include "media/Pose.h"
+
+namespace android {
+namespace media {
+
+/**
+ * Drift compensator for a stream of poses.
+ *
+ * This is effectively a high-pass filter for a pose stream, removing any DC-offset / bias. The
+ * provided input stream will be "pulled" toward identity with an exponential decay filter with a
+ * configurable time constant. Rotation and translation are handled separately.
+ *
+ * Typical usage:
+ * PoseDriftCompensator comp(...);
+ *
+ * while (...) {
+ * comp.setInput(...);
+ * Pose3f output = comp.getOutput();
+ * }
+ *
+ * There doesn't need to be a 1:1 correspondence between setInput() and getOutput() calls. The
+ * output timestamp is always that of the last setInput() call. Calling recenter() will reset the
+ * bias to the current output, causing the output to be identity.
+ *
+ * The initial bias point is identity.
+ *
+ * This implementation is thread-compatible, but not thread-safe.
+ */
+class PoseDriftCompensator {
+ public:
+ struct Options {
+ float translationalDriftTimeConstant = std::numeric_limits<float>::infinity();
+ float rotationalDriftTimeConstant = std::numeric_limits<float>::infinity();
+ };
+
+ explicit PoseDriftCompensator(const Options& options);
+
+ void setInput(int64_t timestamp, const Pose3f& input);
+
+ void recenter();
+
+ Pose3f getOutput() const;
+
+ private:
+ const Options mOptions;
+
+ Pose3f mPrevInput;
+ Pose3f mOutput;
+ std::optional<int64_t> mTimestamp;
+
+ Pose3f scale(const Pose3f& pose, int64_t dt);
+};
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/PoseProcessingGraph.png b/media/libheadtracking/PoseProcessingGraph.png
new file mode 100644
index 0000000..8e6dfd2
--- /dev/null
+++ b/media/libheadtracking/PoseProcessingGraph.png
Binary files differ
diff --git a/media/libheadtracking/PoseRateLimiter-test.cpp b/media/libheadtracking/PoseRateLimiter-test.cpp
new file mode 100644
index 0000000..f306183
--- /dev/null
+++ b/media/libheadtracking/PoseRateLimiter-test.cpp
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include "PoseRateLimiter.h"
+#include "QuaternionUtil.h"
+#include "TestUtil.h"
+
+namespace android {
+namespace media {
+namespace {
+
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+using Options = PoseRateLimiter::Options;
+
+TEST(PoseRateLimiter, Initial) {
+ Pose3f target({1, 2, 3}, Quaternionf::UnitRandom());
+ PoseRateLimiter limiter(Options{.maxTranslationalVelocity = 10, .maxRotationalVelocity = 10});
+ limiter.setTarget(target);
+ EXPECT_EQ(limiter.calculatePose(1000), target);
+}
+
+TEST(PoseRateLimiter, UnlimitedZeroTime) {
+ Pose3f target1({1, 2, 3}, Quaternionf::UnitRandom());
+ Pose3f target2({4, 5, 6}, Quaternionf::UnitRandom());
+ PoseRateLimiter limiter(Options{});
+ limiter.setTarget(target1);
+ EXPECT_EQ(limiter.calculatePose(0), target1);
+ limiter.setTarget(target2);
+ EXPECT_EQ(limiter.calculatePose(0), target2);
+ limiter.setTarget(target1);
+ EXPECT_EQ(limiter.calculatePose(0), target1);
+}
+
+TEST(PoseRateLimiter, Limited) {
+ Pose3f pose1({1, 2, 3}, Quaternionf::Identity());
+ Pose3f pose2({1, 2, 8}, rotateZ(M_PI * 5 / 8));
+ PoseRateLimiter limiter(Options{.maxTranslationalVelocity = 1, .maxRotationalVelocity = 10});
+ limiter.setTarget(pose2);
+ EXPECT_EQ(limiter.calculatePose(1000), pose2);
+
+ // Rate limiting is inactive. Should track despite the violation.
+ limiter.setTarget(pose1);
+ EXPECT_EQ(limiter.calculatePose(1001), pose1);
+
+ // Enable rate limiting and observe gradual motion from pose1 to pose2.
+ limiter.enable();
+ limiter.setTarget(pose2);
+ EXPECT_EQ(limiter.calculatePose(1002), Pose3f({1, 2, 4}, rotateZ(M_PI * 1 / 8)));
+ limiter.setTarget(pose2);
+ EXPECT_EQ(limiter.calculatePose(1003), Pose3f({1, 2, 5}, rotateZ(M_PI * 2 / 8)));
+ // Skip a tick.
+ limiter.setTarget(pose2);
+ EXPECT_EQ(limiter.calculatePose(1005), Pose3f({1, 2, 7}, rotateZ(M_PI * 4 / 8)));
+ limiter.setTarget(pose2);
+ EXPECT_EQ(limiter.calculatePose(1006), pose2);
+
+ // We reached the target, so rate limiting should now be disabled.
+ limiter.setTarget(pose1);
+ EXPECT_EQ(limiter.calculatePose(1007), pose1);
+}
+
+TEST(PoseRateLimiter, Reset) {
+ Pose3f pose1({1, 2, 3}, Quaternionf::Identity());
+ Pose3f pose2({1, 2, 8}, rotateZ(M_PI * 5 / 8));
+ PoseRateLimiter limiter(Options{.maxTranslationalVelocity = 1, .maxRotationalVelocity = 10});
+ limiter.setTarget(pose1);
+ EXPECT_EQ(limiter.calculatePose(1000), pose1);
+
+ // Enable rate limiting and observe gradual motion from pose1 to pose2.
+ limiter.enable();
+ limiter.setTarget(pose2);
+ EXPECT_EQ(limiter.calculatePose(1001), Pose3f({1, 2, 4}, rotateZ(M_PI * 1 / 8)));
+
+ // Reset the pose and disable rate limiting.
+ limiter.reset(pose2);
+ EXPECT_EQ(limiter.calculatePose(1002), pose2);
+
+ // Rate limiting should now be disabled.
+ limiter.setTarget(pose1);
+ EXPECT_EQ(limiter.calculatePose(1003), pose1);
+}
+
+} // namespace
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/PoseRateLimiter.cpp b/media/libheadtracking/PoseRateLimiter.cpp
new file mode 100644
index 0000000..380e22b
--- /dev/null
+++ b/media/libheadtracking/PoseRateLimiter.cpp
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"){}
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "PoseRateLimiter.h"
+
+namespace android {
+namespace media {
+
+PoseRateLimiter::PoseRateLimiter(const Options& options) : mOptions(options), mLimiting(false) {}
+
+void PoseRateLimiter::enable() {
+ mLimiting = true;
+}
+
+void PoseRateLimiter::reset(const Pose3f& target) {
+ mLimiting = false;
+ mTargetPose = target;
+}
+
+void PoseRateLimiter::setTarget(const Pose3f& target) {
+ mTargetPose = target;
+}
+
+Pose3f PoseRateLimiter::calculatePose(int64_t timestamp) {
+ assert(mTargetPose.has_value());
+ Pose3f pose;
+ if (mLimiting && mOutput.has_value()) {
+ std::tie(pose, mLimiting) = moveWithRateLimit(
+ mOutput->pose, mTargetPose.value(), timestamp - mOutput->timestamp,
+ mOptions.maxTranslationalVelocity, mOptions.maxRotationalVelocity);
+ } else {
+ pose = mTargetPose.value();
+ }
+ mOutput = Point{pose, timestamp};
+ return pose;
+}
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/PoseRateLimiter.h b/media/libheadtracking/PoseRateLimiter.h
new file mode 100644
index 0000000..aa2fe80
--- /dev/null
+++ b/media/libheadtracking/PoseRateLimiter.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <optional>
+
+#include "media/Pose.h"
+
+namespace android {
+namespace media {
+
+/**
+ * Limits a stream of poses to a given maximum translational and rotational velocities.
+ *
+ * Normal operation:
+ *
+ * Pose3f output;
+ * PoseRateLimiter limiter(...);
+ *
+ * // Limiting is disabled. Output will be the same as last input.
+ * limiter.setTarget(...);
+ * output = limiter.calculatePose(...);
+ * limiter.setTarget(...);
+ * output = limiter.calculatePose(...);
+ *
+ * // Enable limiting. Output will no longer be necessarily the same as last input.
+ * limiter.enable();
+ * limiter.setTarget(...);
+ * output = limiter.calculatePose(...);
+ * limiter.setTarget(...);
+ * output = limiter.calculatePose(...);
+ *
+ * // When eventually the output has been able to catch up with the last input, the limited will be
+ * // automatically disabled again and the output will match the input again.
+ * limiter.setTarget(...);
+ * output = limiter.calculatePose(...);
+ *
+ * As shown above, the limiter is turned on manually via enable(), but turns off automatically as
+ * soon as the output is able to catch up to the input. The intention is that rate limiting will be
+ * turned on at specific times to smooth out any artificial discontinuities introduced to the pose
+ * stream, but the rest of the time will be a simple passthrough.
+
+ * setTarget(...) and calculatePose(...) don't have to be ordered in any particular way. However,
+ * setTarget or reset() must be called at least once prior to the first calculatePose().
+ *
+ * Calling reset() instead of setTarget() forces the output to the given pose and disables rate
+ * limiting.
+ *
+ * This implementation is thread-compatible, but not thread-safe.
+ */
+class PoseRateLimiter {
+ public:
+ struct Options {
+ float maxTranslationalVelocity = std::numeric_limits<float>::infinity();
+ float maxRotationalVelocity = std::numeric_limits<float>::infinity();
+ };
+
+ explicit PoseRateLimiter(const Options& options);
+
+ void enable();
+
+ void reset(const Pose3f& target);
+ void setTarget(const Pose3f& target);
+
+ Pose3f calculatePose(int64_t timestamp);
+
+ private:
+ struct Point {
+ Pose3f pose;
+ int64_t timestamp;
+ };
+
+ const Options mOptions;
+ bool mLimiting;
+ std::optional<Pose3f> mTargetPose;
+ std::optional<Point> mOutput;
+};
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/QuaternionUtil-test.cpp b/media/libheadtracking/QuaternionUtil-test.cpp
new file mode 100644
index 0000000..e79e54a
--- /dev/null
+++ b/media/libheadtracking/QuaternionUtil-test.cpp
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include "QuaternionUtil.h"
+#include "TestUtil.h"
+
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+
+namespace android {
+namespace media {
+namespace {
+
+TEST(QuaternionUtil, RotationVectorToQuaternion) {
+ // 90 degrees around Z.
+ Vector3f rot = {0, 0, M_PI_2};
+ Quaternionf quat = rotationVectorToQuaternion(rot);
+ ASSERT_EQ(quat * Vector3f(1, 0, 0), Vector3f(0, 1, 0));
+ ASSERT_EQ(quat * Vector3f(0, 1, 0), Vector3f(-1, 0, 0));
+ ASSERT_EQ(quat * Vector3f(0, 0, 1), Vector3f(0, 0, 1));
+}
+
+TEST(QuaternionUtil, QuaternionToRotationVector) {
+ Quaternionf quat = Quaternionf::FromTwoVectors(Vector3f(1, 0, 0), Vector3f(0, 1, 0));
+ Vector3f rot = quaternionToRotationVector(quat);
+ ASSERT_EQ(rot, Vector3f(0, 0, M_PI_2));
+}
+
+TEST(QuaternionUtil, RoundTripFromQuaternion) {
+ Quaternionf quaternion = Quaternionf::UnitRandom();
+ EXPECT_EQ(quaternion, rotationVectorToQuaternion(quaternionToRotationVector(quaternion)));
+}
+
+TEST(QuaternionUtil, RoundTripFromVector) {
+ Vector3f vec{0.1, 0.2, 0.3};
+ EXPECT_EQ(vec, quaternionToRotationVector(rotationVectorToQuaternion(vec)));
+}
+
+} // namespace
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/QuaternionUtil.cpp b/media/libheadtracking/QuaternionUtil.cpp
new file mode 100644
index 0000000..5d090de
--- /dev/null
+++ b/media/libheadtracking/QuaternionUtil.cpp
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "QuaternionUtil.h"
+
+#include <cassert>
+
+namespace android {
+namespace media {
+
+using Eigen::NumTraits;
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+
+namespace {
+
+Vector3f LogSU2(const Quaternionf& q) {
+ // Implementation of the logarithmic map of SU(2) using atan.
+ // This follows Hertzberg et al. "Integrating Generic Sensor Fusion Algorithms
+ // with Sound State Representations through Encapsulation of Manifolds", Eq.
+ // (31)
+ // We use asin and acos instead of atan to enable the use of Eigen Autodiff
+ // with SU2.
+ const float sign_of_w = q.w() < 0.f ? -1.f : 1.f;
+ const float abs_w = sign_of_w * q.w();
+ const Vector3f v = sign_of_w * q.vec();
+ const float squared_norm_of_v = v.squaredNorm();
+
+ assert(abs(1.f - abs_w * abs_w - squared_norm_of_v) < NumTraits<float>::dummy_precision());
+
+ if (squared_norm_of_v > NumTraits<float>::dummy_precision()) {
+ const float norm_of_v = sqrt(squared_norm_of_v);
+ if (abs_w > NumTraits<float>::dummy_precision()) {
+ // asin(x) = acos(x) at x = 1/sqrt(2).
+ if (norm_of_v <= float(M_SQRT1_2)) {
+ return (asin(norm_of_v) / norm_of_v) * v;
+ }
+ return (acos(abs_w) / norm_of_v) * v;
+ }
+ return (M_PI_2 / norm_of_v) * v;
+ }
+
+ // Taylor expansion at squared_norm_of_v == 0
+ return (1.f / abs_w - squared_norm_of_v / (3.f * pow(abs_w, 3))) * v;
+}
+
+Quaternionf ExpSU2(const Vector3f& delta) {
+ Quaternionf q_delta;
+ const float theta_squared = delta.squaredNorm();
+ if (theta_squared > NumTraits<float>::dummy_precision()) {
+ const float theta = sqrt(theta_squared);
+ q_delta.w() = cos(theta);
+ q_delta.vec() = (sin(theta) / theta) * delta;
+ } else {
+ // taylor expansions around theta == 0
+ q_delta.w() = 1.f - 0.5f * theta_squared;
+ q_delta.vec() = (1.f - 1.f / 6.f * theta_squared) * delta;
+ }
+ return q_delta;
+}
+
+} // namespace
+
+Quaternionf rotationVectorToQuaternion(const Vector3f& rotationVector) {
+ // SU(2) is a double cover of SO(3), thus we have to half the tangent vector
+ // delta
+ const Vector3f half_delta = 0.5f * rotationVector;
+ return ExpSU2(half_delta);
+}
+
+Vector3f quaternionToRotationVector(const Quaternionf& quaternion) {
+ // SU(2) is a double cover of SO(3), thus we have to multiply the tangent
+ // vector delta by two
+ return 2.f * LogSU2(quaternion);
+}
+
+Quaternionf rotateX(float angle) {
+ return rotationVectorToQuaternion(Vector3f(1, 0, 0) * angle);
+}
+
+Quaternionf rotateY(float angle) {
+ return rotationVectorToQuaternion(Vector3f(0, 1, 0) * angle);
+}
+
+Quaternionf rotateZ(float angle) {
+ return rotationVectorToQuaternion(Vector3f(0, 0, 1) * angle);
+}
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/QuaternionUtil.h b/media/libheadtracking/QuaternionUtil.h
new file mode 100644
index 0000000..f7a2ca9
--- /dev/null
+++ b/media/libheadtracking/QuaternionUtil.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <Eigen/Geometry>
+
+namespace android {
+namespace media {
+
+/**
+ * Converts a rotation vector to an equivalent quaternion.
+ * The rotation vector is given as a 3-vector whose direction represents the rotation axis and its
+ * magnitude the rotation angle (in radians) around that axis.
+ */
+Eigen::Quaternionf rotationVectorToQuaternion(const Eigen::Vector3f& rotationVector);
+
+/**
+ * Converts a quaternion to an equivalent rotation vector.
+ * The rotation vector is given as a 3-vector whose direction represents the rotation axis and its
+ * magnitude the rotation angle (in radians) around that axis.
+ */
+Eigen::Vector3f quaternionToRotationVector(const Eigen::Quaternionf& quaternion);
+
+/**
+ * Returns a quaternion representing a rotation around the X-axis with the given amount (in
+ * radians).
+ */
+Eigen::Quaternionf rotateX(float angle);
+
+/**
+ * Returns a quaternion representing a rotation around the Y-axis with the given amount (in
+ * radians).
+ */
+Eigen::Quaternionf rotateY(float angle);
+
+/**
+ * Returns a quaternion representing a rotation around the Z-axis with the given amount (in
+ * radians).
+ */
+Eigen::Quaternionf rotateZ(float angle);
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/README.md b/media/libheadtracking/README.md
new file mode 100644
index 0000000..3d5b71a
--- /dev/null
+++ b/media/libheadtracking/README.md
@@ -0,0 +1,185 @@
+# Head-Tracking Library For Immersive Audio
+
+This library handles the processing of head-tracking information, necessary for
+Immersive Audio functionality. It goes from bare sensor reading into the final
+pose fed into a virtualizer.
+
+## Basic Usage
+
+The main entry point into this library is the `HeadTrackingProcessor` class.
+This class is provided with the following inputs:
+
+- Head pose, relative to some arbitrary world frame.
+- Screen pose, relative to some arbitrary world frame.
+- Display orientation, defined as the angle between the "physical" screen and
+ the "logical" screen.
+- Transform between the screen and the sound stage.
+- Desired operational mode:
+ - Static: only the sound stage pose is taken into account. This will result
+ in an experience where the sound stage moved with the listener's head.
+ - World-relative: both the head pose and stage pose are taken into account.
+ This will result in an experience where the sound stage is perceived to be
+ located at a fixed place in the world.
+ - Screen-relative: the head pose, screen pose and stage pose are all taken
+ into account. This will result in an experience where the sound stage is
+ perceived to be located at a fixed place relative to the screen.
+
+Once inputs are provided, the `calculate()` method will make the following
+output available:
+
+- Stage pose, relative to the head. This aggregates all the inputs mentioned
+ above and is ready to be fed into a virtualizer.
+- Actual operational mode. May deviate from the desired one in cases where the
+ desired mode cannot be calculated (for example, as result of dropped messages
+ from one of the sensors).
+
+A `recenter()` operation is also available, which indicates to the system that
+whatever pose the screen and head are currently at should be considered as the
+"center" pose, or frame of reference.
+
+## Pose-Related Conventions
+
+### Naming and Composition
+
+When referring to poses in code, it is always good practice to follow
+conventional naming, which highlights the reference and target frames clearly:
+
+Bad:
+
+```
+Pose3f headPose;
+```
+
+Good:
+
+```
+Pose3f worldToHead; // “world” is the reference frame,
+ // “head” is the target frame.
+```
+
+By following this convention, it is easy to follow correct composition of poses,
+by making sure adjacent frames are identical:
+
+```
+Pose3f aToD = aToB * bToC * cToD;
+```
+
+And similarly, inverting the transform simply flips the reference and target:
+
+```
+Pose3f aToB = bToA.inverse();
+```
+
+### Twist
+
+“Twist” is to pose what velocity is to distance: it is the time-derivative of a
+pose, representing the change in pose over a short period of time. Its naming
+convention always states one frame, e.g.:
+Twist3f headTwist;
+
+This means that this twist represents the head-at-time-T to head-at-time-T+dt
+transform. Twists are not composable in the same way as poses.
+
+### Frames of Interest
+
+The frames of interest in this library are defined as follows:
+
+#### Head
+
+This is the listener’s head. The origin is at the center point between the
+ear-drums, the X-axis goes from left ear to right ear, Y-axis goes from the back
+of the head towards the face and Z-axis goes from the bottom of the head to the
+top.
+
+#### Screen
+
+This is the primary screen that the user will be looking at, which is relevant
+for some Immersive Audio use-cases, such as watching a movie. We will follow a
+different convention for this frame than what the Sensor framework uses. The
+origin is at the center of the screen. X-axis goes from left to right, Z-axis
+goes from the screen bottom to the screen top, Y-axis goes “into” the screen (
+from the direction of the viewer). The up/down/left/right of the screen are
+defined as the logical directions used for display. So when flipping the display
+orientation between “landscape” and “portrait”, the frame of reference will
+change with respect to the physical screen.
+
+#### Stage
+
+This is the frame of reference used by the virtualizer for positioning sound
+objects. It is not associated with any physical frame. In a typical
+multi-channel scenario, the listener is at the origin, the X-axis goes from left
+to right, Y-axis from back to front and Z-axis from down to up. For example, a
+front-right speaker is located at positive X, Y and Z=0, a height speaker will
+have a positive Z.
+
+#### World
+
+It is sometimes convenient to use an intermediate frame when dealing with
+head-to-screen transforms. The “world” frame is an arbitrary frame of reference
+in the physical world, relative to which we can measure the head pose and screen
+pose. In (very common) cases when we can’t establish such an absolute frame, we
+can take each measurement relative to a separate, arbitrary frame and high-pass
+the result.
+
+## Processing Description
+
+
+
+The diagram above illustrates the processing that takes place from the inputs to
+the outputs.
+
+### Predictor
+
+The Predictor block gets pose + twist (pose derivative) and extrapolates to
+obtain a predicted head pose (w/ given latency).
+
+### Drift / Bias Compensator
+
+The Drift / Bias Compensator blocks serve two purposes:
+
+- Compensate for floating reference axes by applying a high-pass filter, which
+ slowly pulls the pose toward identity.
+- Establish the reference frame for the poses by having the ability to set the
+ current pose as the reference for future poses (recentering). Effectively,
+ this is resetting the filter state to identity.
+
+### Orientation Compensation
+
+The Orientation Compensation block applies the display orientation to the screen
+pose to obtain the pose of the “logical screen” frame, in which the Y-axis is
+pointing in the direction of the logical screen “up” rather than the physical
+one.
+
+### Screen-Relative Pose
+
+The Screen-Relative Pose block is provided with a head pose and a screen pose
+and estimates the pose of the head relative to the screen. Optionally, this
+module may indicate that the user is likely not in front of the screen via the
+“valid” output.
+
+### Mode Selector
+
+The Mode Selector block aggregates the various sources of pose information into
+a head-to-stage pose that is going to feed the virtualizer. It is controlled by
+the “desired mode” signal that indicates whether the preference is to be in
+either static, world-relative or screen-relative.
+
+The actual mode may diverge from the desired mode. It is determined as follows:
+
+- If the desired mode is static, the actual mode is static.
+- If the desired mode is world-relative:
+ - If head poses are fresh, the actual mode is world-relative.
+ - Otherwise the actual mode is static.
+- If the desired mode is screen-relative:
+ - If head and screen poses are fresh and the ‘valid’ signal is asserted, the
+ actual mode is screen-relative.
+ - Otherwise, apply the same rules as the desired mode being world-relative.
+
+### Rate Limiter
+
+A Rate Limiter block is applied to the final output to smooth out any abrupt
+transitions caused by any of the following events:
+
+- Mode switch.
+- Display orientation switch.
+- Recenter operation.
diff --git a/media/libheadtracking/ScreenHeadFusion-test.cpp b/media/libheadtracking/ScreenHeadFusion-test.cpp
new file mode 100644
index 0000000..ecf27f5
--- /dev/null
+++ b/media/libheadtracking/ScreenHeadFusion-test.cpp
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include "ScreenHeadFusion.h"
+#include "TestUtil.h"
+
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+
+namespace android {
+namespace media {
+namespace {
+
+TEST(ScreenHeadFusion, Init) {
+ ScreenHeadFusion fusion;
+ EXPECT_FALSE(fusion.calculate().has_value());
+}
+
+TEST(ScreenHeadFusion, Calculate_NoHead) {
+ ScreenHeadFusion fusion;
+ fusion.setWorldToScreenPose(0, Pose3f());
+ EXPECT_FALSE(fusion.calculate().has_value());
+}
+
+TEST(ScreenHeadFusion, Calculate_NoScreen) {
+ ScreenHeadFusion fusion;
+ fusion.setWorldToHeadPose(0, Pose3f());
+ EXPECT_FALSE(fusion.calculate().has_value());
+}
+
+TEST(ScreenHeadFusion, Calculate) {
+ Pose3f worldToScreen1({1, 2, 3}, Quaternionf::UnitRandom());
+ Pose3f worldToHead1({4, 5, 6}, Quaternionf::UnitRandom());
+ Pose3f worldToScreen2({11, 12, 13}, Quaternionf::UnitRandom());
+ Pose3f worldToHead2({14, 15, 16}, Quaternionf::UnitRandom());
+
+ ScreenHeadFusion fusion;
+ fusion.setWorldToHeadPose(123, worldToHead1);
+ fusion.setWorldToScreenPose(456, worldToScreen1);
+ auto result = fusion.calculate();
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(123, result->timestamp);
+ EXPECT_EQ(worldToScreen1.inverse() * worldToHead1, result->pose);
+
+ fusion.setWorldToHeadPose(567, worldToHead2);
+ result = fusion.calculate();
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(456, result->timestamp);
+ EXPECT_EQ(worldToScreen1.inverse() * worldToHead2, result->pose);
+
+ fusion.setWorldToScreenPose(678, worldToScreen2);
+ result = fusion.calculate();
+ ASSERT_TRUE(result.has_value());
+ EXPECT_EQ(567, result->timestamp);
+ EXPECT_EQ(worldToScreen2.inverse() * worldToHead2, result->pose);
+}
+
+} // namespace
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/ScreenHeadFusion.cpp b/media/libheadtracking/ScreenHeadFusion.cpp
new file mode 100644
index 0000000..f023570
--- /dev/null
+++ b/media/libheadtracking/ScreenHeadFusion.cpp
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"){}
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ScreenHeadFusion.h"
+
+namespace android {
+namespace media {
+
+void ScreenHeadFusion::setWorldToHeadPose(int64_t timestamp, const Pose3f& worldToHead) {
+ mWorldToHead = TimestampedPose{.timestamp = timestamp, .pose = worldToHead};
+}
+
+void ScreenHeadFusion::setWorldToScreenPose(int64_t timestamp, const Pose3f& worldToScreen) {
+ mWorldToScreen = TimestampedPose{.timestamp = timestamp, .pose = worldToScreen};
+}
+
+std::optional<ScreenHeadFusion::TimestampedPose> ScreenHeadFusion::calculate() {
+ // TODO: this is temporary, simplistic logic.
+ if (!mWorldToHead.has_value() || !mWorldToScreen.has_value()) {
+ return std::nullopt;
+ }
+ return TimestampedPose{
+ .timestamp = std::min(mWorldToHead->timestamp, mWorldToScreen->timestamp),
+ .pose = mWorldToScreen->pose.inverse() * mWorldToHead->pose};
+}
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/ScreenHeadFusion.h b/media/libheadtracking/ScreenHeadFusion.h
new file mode 100644
index 0000000..ee81100
--- /dev/null
+++ b/media/libheadtracking/ScreenHeadFusion.h
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <optional>
+
+#include "media/Pose.h"
+
+namespace android {
+namespace media {
+
+/**
+ * Combines world-to-head pose with world-to-screen pose to obtain screen-to-head.
+ *
+ * Input poses may arrive separately. The last pose of each kind is taken into account. The
+ * timestamp of the output is the ealier (older) timestamp of the two inputs.
+ *
+ * Output may be nullopt in the following cases:
+ * - Either one of the inputs has not yet been provided.
+ * - It is estimated that the user is no longer facing the screen.
+ *
+ * Typical usage:
+ *
+ * ScreenHeadFusion fusion(...);
+ * fusion.setWorldToHeadPose(...);
+ * fusion.setWorldToScreenPose(...);
+ * auto output = fusion.calculate();
+ *
+ * This class is not thread-safe, but thread-compatible.
+ */
+class ScreenHeadFusion {
+ public:
+ struct TimestampedPose {
+ int64_t timestamp;
+ Pose3f pose;
+ };
+
+ void setWorldToHeadPose(int64_t timestamp, const Pose3f& worldToHead);
+
+ void setWorldToScreenPose(int64_t timestamp, const Pose3f& worldToScreen);
+
+ /**
+ * Returns the screen-to-head pose, or nullopt if invalid.
+ */
+ std::optional<TimestampedPose> calculate();
+
+ private:
+ std::optional<TimestampedPose> mWorldToHead;
+ std::optional<TimestampedPose> mWorldToScreen;
+};
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/SensorPoseProvider-example.cpp b/media/libheadtracking/SensorPoseProvider-example.cpp
new file mode 100644
index 0000000..30c6537
--- /dev/null
+++ b/media/libheadtracking/SensorPoseProvider-example.cpp
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <unistd.h>
+#include <iostream>
+
+#include <android/sensor.h>
+#include <hardware/sensors.h>
+
+#include <media/SensorPoseProvider.h>
+
+using android::media::Pose3f;
+using android::media::SensorPoseProvider;
+using android::media::Twist3f;
+
+const char kPackageName[] = "SensorPoseProvider-example";
+
+class Listener : public SensorPoseProvider::Listener {
+ public:
+ void onPose(int64_t timestamp, int32_t handle, const Pose3f& pose,
+ const std::optional<Twist3f>& twist) override {
+ std::cout << "onPose t=" << timestamp << " sensor=" << handle << " pose=" << pose
+ << " twist=";
+ if (twist.has_value()) {
+ std::cout << twist.value();
+ } else {
+ std::cout << "<none>";
+ }
+ std::cout << std::endl;
+ }
+};
+
+int main() {
+ ASensorManager* sensor_manager = ASensorManager_getInstanceForPackage(kPackageName);
+ if (!sensor_manager) {
+ std::cerr << "Failed to get a sensor manager" << std::endl;
+ return 1;
+ }
+
+ const ASensor* headSensor =
+ ASensorManager_getDefaultSensor(sensor_manager, SENSOR_TYPE_GAME_ROTATION_VECTOR);
+ const ASensor* screenSensor =
+ ASensorManager_getDefaultSensor(sensor_manager, SENSOR_TYPE_ROTATION_VECTOR);
+
+ Listener listener;
+
+ std::unique_ptr<SensorPoseProvider> provider =
+ SensorPoseProvider::create(kPackageName, &listener);
+ int32_t headHandle = provider->startSensor(headSensor, std::chrono::milliseconds(500));
+ sleep(2);
+ provider->startSensor(screenSensor, std::chrono::milliseconds(500));
+ sleep(2);
+ provider->stopSensor(headHandle);
+ sleep(2);
+ return 0;
+}
diff --git a/media/libheadtracking/SensorPoseProvider.cpp b/media/libheadtracking/SensorPoseProvider.cpp
new file mode 100644
index 0000000..8e6c2ff
--- /dev/null
+++ b/media/libheadtracking/SensorPoseProvider.cpp
@@ -0,0 +1,236 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <media/SensorPoseProvider.h>
+
+#define LOG_TAG "SensorPoseProvider"
+
+#include <inttypes.h>
+
+#include <future>
+#include <iostream>
+#include <map>
+#include <thread>
+
+#include <android/looper.h>
+#include <log/log_main.h>
+
+namespace android {
+namespace media {
+namespace {
+
+/**
+ * RAII-wrapper around ASensorEventQueue, which destroys it on destruction.
+ */
+class EventQueueGuard {
+ public:
+ EventQueueGuard(ASensorManager* manager, ASensorEventQueue* queue)
+ : mManager(manager), mQueue(queue) {}
+
+ ~EventQueueGuard() {
+ if (mQueue) {
+ int ret = ASensorManager_destroyEventQueue(mManager, mQueue);
+ if (ret) {
+ ALOGE("Failed to destroy event queue: %s\n", strerror(ret));
+ }
+ }
+ }
+
+ EventQueueGuard(const EventQueueGuard&) = delete;
+ EventQueueGuard& operator=(const EventQueueGuard&) = delete;
+
+ [[nodiscard]] ASensorEventQueue* get() const { return mQueue; }
+
+ private:
+ ASensorManager* const mManager;
+ ASensorEventQueue* mQueue;
+};
+
+/**
+ * RAII-wrapper around an enabled sensor, which disables it upon destruction.
+ */
+class SensorEnableGuard {
+ public:
+ SensorEnableGuard(ASensorEventQueue* queue, const ASensor* sensor)
+ : mQueue(queue), mSensor(sensor) {}
+
+ ~SensorEnableGuard() {
+ if (mSensor) {
+ int ret = ASensorEventQueue_disableSensor(mQueue, mSensor);
+ if (ret) {
+ ALOGE("Failed to disable sensor: %s\n", strerror(ret));
+ }
+ }
+ }
+
+ SensorEnableGuard(const SensorEnableGuard&) = delete;
+ SensorEnableGuard& operator=(const SensorEnableGuard&) = delete;
+
+ // Enable moving.
+ SensorEnableGuard(SensorEnableGuard&& other) : mQueue(other.mQueue), mSensor(other.mSensor) {
+ other.mSensor = nullptr;
+ }
+
+ private:
+ ASensorEventQueue* const mQueue;
+ const ASensor* mSensor;
+};
+
+/**
+ * Streams the required events to a PoseListener, based on events originating from the Sensor stack.
+ */
+class SensorPoseProviderImpl : public SensorPoseProvider {
+ public:
+ static std::unique_ptr<SensorPoseProvider> create(const char* packageName, Listener* listener) {
+ std::unique_ptr<SensorPoseProviderImpl> result(
+ new SensorPoseProviderImpl(packageName, listener));
+ return result->waitInitFinished() ? std::move(result) : nullptr;
+ }
+
+ ~SensorPoseProviderImpl() override {
+ ALooper_wake(mLooper);
+ mThread.join();
+ }
+
+ int32_t startSensor(const ASensor* sensor, std::chrono::microseconds samplingPeriod) override {
+ int32_t handle = ASensor_getHandle(sensor);
+
+ // Enable the sensor.
+ if (ASensorEventQueue_registerSensor(mQueue->get(), sensor, samplingPeriod.count(), 0)) {
+ ALOGE("Failed to enable sensor");
+ return INVALID_HANDLE;
+ }
+
+ mEnabledSensors.emplace(handle, SensorEnableGuard(mQueue->get(), sensor));
+ return handle;
+ }
+
+ void stopSensor(int handle) override { mEnabledSensors.erase(handle); }
+
+ private:
+ ALooper* mLooper;
+ Listener* const mListener;
+ std::thread mThread;
+ std::map<int32_t, SensorEnableGuard> mEnabledSensors;
+ std::unique_ptr<EventQueueGuard> mQueue;
+
+ // We must do some of the initialization operations on the worker thread, because the API relies
+ // on the thread-local looper. In addition, as a matter of convenience, we store some of the
+ // state on the stack.
+ // For that reason, we use a two-step initialization approach, where the ctor mostly just starts
+ // the worker thread and that thread would notify, via the promise below whenever initialization
+ // is finished, and whether it was successful.
+ std::promise<bool> mInitPromise;
+
+ SensorPoseProviderImpl(const char* packageName, Listener* listener)
+ : mListener(listener),
+ mThread([this, p = std::string(packageName)] { threadFunc(p.c_str()); }) {}
+
+ void initFinished(bool success) { mInitPromise.set_value(success); }
+
+ bool waitInitFinished() { return mInitPromise.get_future().get(); }
+
+ void threadFunc(const char* packageName) {
+ // Obtain looper.
+ mLooper = ALooper_prepare(ALOOPER_PREPARE_ALLOW_NON_CALLBACKS);
+
+ // The number 19 is arbitrary, only useful if using multiple objects on the same looper.
+ constexpr int kIdent = 19;
+
+ // Obtain sensor manager.
+ ASensorManager* sensor_manager = ASensorManager_getInstanceForPackage(packageName);
+ if (!sensor_manager) {
+ ALOGE("Failed to get a sensor manager");
+ initFinished(false);
+ return;
+ }
+
+ // Create event queue.
+ ASensorEventQueue* queue =
+ ASensorManager_createEventQueue(sensor_manager, mLooper, kIdent, nullptr, nullptr);
+
+ if (queue == nullptr) {
+ ALOGE("Failed to create a sensor event queue");
+ initFinished(false);
+ return;
+ }
+
+ mQueue.reset(new EventQueueGuard(sensor_manager, queue));
+
+ initFinished(true);
+
+ while (true) {
+ int ret = ALooper_pollOnce(-1 /* no timeout */, nullptr, nullptr, nullptr);
+
+ switch (ret) {
+ case ALOOPER_POLL_WAKE:
+ // Normal way to exit.
+ return;
+
+ case kIdent:
+ // Possible events on our queue.
+ break;
+
+ default:
+ ALOGE("Unexpected status out of ALooper_pollOnce: %d", ret);
+ }
+
+ // Process an event.
+ ASensorEvent event;
+ ssize_t size = ASensorEventQueue_getEvents(queue, &event, 1);
+ if (size < 0 || size > 1) {
+ ALOGE("Unexpected return value from ASensorEventQueue_getEvents: %zd", size);
+ break;
+ }
+ if (size == 0) {
+ // No events.
+ continue;
+ }
+
+ handleEvent(event);
+ }
+ }
+
+ void handleEvent(const ASensorEvent& event) {
+ auto value = parseEvent(event);
+ mListener->onPose(event.timestamp, event.sensor, std::get<0>(value), std::get<1>(value));
+ }
+
+ static std::tuple<Pose3f, std::optional<Twist3f>> parseEvent(const ASensorEvent& event) {
+ // TODO(ytai): Add more types.
+ switch (event.type) {
+ case ASENSOR_TYPE_ROTATION_VECTOR:
+ case ASENSOR_TYPE_GAME_ROTATION_VECTOR: {
+ Eigen::Quaternionf quat(event.data[3], event.data[0], event.data[1], event.data[2]);
+ return std::make_tuple(Pose3f(quat), std::optional<Twist3f>());
+ }
+
+ default:
+ ALOGE("Unsupported sensor type: %" PRId32, event.type);
+ return std::make_tuple(Pose3f(), std::optional<Twist3f>());
+ }
+ }
+};
+
+} // namespace
+
+std::unique_ptr<SensorPoseProvider> SensorPoseProvider::create(const char* packageName,
+ Listener* listener) {
+ return SensorPoseProviderImpl::create(packageName, listener);
+}
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/TestUtil.h b/media/libheadtracking/TestUtil.h
new file mode 100644
index 0000000..4636d86
--- /dev/null
+++ b/media/libheadtracking/TestUtil.h
@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <gtest/gtest.h>
+
+#include "media/Pose.h"
+#include "media/Twist.h"
+
+namespace {
+
+constexpr float kPoseComparisonPrecision = 1e-5;
+
+} // namespace
+
+// These specializations make {EXPECT,ASSERT}_{EQ,NE} work correctly for Pose3f, Twist3f, Vector3f
+// and Quaternionf.
+namespace testing {
+namespace internal {
+
+template <>
+inline AssertionResult CmpHelperEQ<android::media::Pose3f, android::media::Pose3f>(
+ const char* lhs_expression, const char* rhs_expression, const android::media::Pose3f& lhs,
+ const android::media::Pose3f& rhs) {
+ if (lhs.isApprox(rhs, kPoseComparisonPrecision)) {
+ return AssertionSuccess();
+ }
+
+ return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs);
+}
+
+template <>
+inline AssertionResult CmpHelperNE<android::media::Pose3f, android::media::Pose3f>(
+ const char* lhs_expression, const char* rhs_expression, const android::media::Pose3f& lhs,
+ const android::media::Pose3f& rhs) {
+ if (!lhs.isApprox(rhs, kPoseComparisonPrecision)) {
+ return AssertionSuccess();
+ }
+
+ return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs);
+}
+
+template <>
+inline AssertionResult CmpHelperEQ<android::media::Twist3f, android::media::Twist3f>(
+ const char* lhs_expression, const char* rhs_expression, const android::media::Twist3f& lhs,
+ const android::media::Twist3f& rhs) {
+ if (lhs.isApprox(rhs, kPoseComparisonPrecision)) {
+ return AssertionSuccess();
+ }
+
+ return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs);
+}
+
+template <>
+inline AssertionResult CmpHelperNE<android::media::Twist3f, android::media::Twist3f>(
+ const char* lhs_expression, const char* rhs_expression, const android::media::Twist3f& lhs,
+ const android::media::Twist3f& rhs) {
+ if (!lhs.isApprox(rhs, kPoseComparisonPrecision)) {
+ return AssertionSuccess();
+ }
+
+ return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs);
+}
+
+template <>
+inline AssertionResult CmpHelperEQ<Eigen::Vector3f, Eigen::Vector3f>(const char* lhs_expression,
+ const char* rhs_expression,
+ const Eigen::Vector3f& lhs,
+ const Eigen::Vector3f& rhs) {
+ if (lhs.isApprox(rhs)) {
+ return AssertionSuccess();
+ }
+
+ return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs);
+}
+
+template <>
+inline AssertionResult CmpHelperNE<Eigen::Vector3f, Eigen::Vector3f>(const char* lhs_expression,
+ const char* rhs_expression,
+ const Eigen::Vector3f& lhs,
+ const Eigen::Vector3f& rhs) {
+ if (!lhs.isApprox(rhs)) {
+ return AssertionSuccess();
+ }
+
+ return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs);
+}
+
+template <>
+inline AssertionResult CmpHelperEQ<Eigen::Quaternionf, Eigen::Quaternionf>(
+ const char* lhs_expression, const char* rhs_expression, const Eigen::Quaternionf& lhs,
+ const Eigen::Quaternionf& rhs) {
+ // Negating the coefs results in an equivalent quaternion.
+ if (lhs.isApprox(rhs) || lhs.isApprox(Eigen::Quaternionf(-rhs.coeffs()))) {
+ return AssertionSuccess();
+ }
+
+ return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs);
+}
+
+template <>
+inline AssertionResult CmpHelperNE<Eigen::Quaternionf, Eigen::Quaternionf>(
+ const char* lhs_expression, const char* rhs_expression, const Eigen::Quaternionf& lhs,
+ const Eigen::Quaternionf& rhs) {
+ // Negating the coefs results in an equivalent quaternion.
+ if (!(lhs.isApprox(rhs) || lhs.isApprox(Eigen::Quaternionf(-rhs.coeffs())))) {
+ return AssertionSuccess();
+ }
+
+ return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs);
+}
+
+} // namespace internal
+} // namespace testing
diff --git a/media/libheadtracking/Twist-test.cpp b/media/libheadtracking/Twist-test.cpp
new file mode 100644
index 0000000..7984e1e
--- /dev/null
+++ b/media/libheadtracking/Twist-test.cpp
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "media/Twist.h"
+
+#include <gtest/gtest.h>
+
+#include "QuaternionUtil.h"
+#include "TestUtil.h"
+
+using Eigen::Quaternionf;
+using Eigen::Vector3f;
+
+namespace android {
+namespace media {
+namespace {
+
+TEST(Twist, DefaultCtor) {
+ Twist3f twist;
+ EXPECT_EQ(twist.translationalVelocity(), Vector3f::Zero());
+ EXPECT_EQ(twist.rotationalVelocity(), Vector3f::Zero());
+ EXPECT_FLOAT_EQ(twist.scalarRotationalVelocity(), 0);
+ EXPECT_FLOAT_EQ(twist.scalarTranslationalVelocity(), 0);
+}
+
+TEST(Twist, FullCtor) {
+ Vector3f rot{1, 2, 3};
+ Vector3f trans{4, 5, 6};
+ Twist3f twist(trans, rot);
+ EXPECT_EQ(twist.translationalVelocity(), trans);
+ EXPECT_EQ(twist.rotationalVelocity(), rot);
+ EXPECT_FLOAT_EQ(twist.scalarRotationalVelocity(), std::sqrt(14.f));
+ EXPECT_FLOAT_EQ(twist.scalarTranslationalVelocity(), std::sqrt(77.f));
+}
+
+TEST(Twist, Integrate) {
+ Vector3f trans{1, 2, 3};
+ // 45 deg/sec around Z.
+ Vector3f rot{0, 0, M_PI_4};
+ Twist3f twist(trans, rot);
+ Pose3f pose = integrate(twist, 2.f);
+
+ EXPECT_EQ(pose, Pose3f(Vector3f{2, 4, 6}, rotateZ(M_PI_2)));
+}
+
+TEST(Twist, Differentiate) {
+ Pose3f pose(Vector3f{2, 4, 6}, rotateZ(M_PI_2));
+ Twist3f twist = differentiate(pose, 2.f);
+ EXPECT_EQ(twist, Twist3f(Vector3f(1, 2, 3), Vector3f(0, 0, M_PI_4)));
+}
+
+} // namespace
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/Twist.cpp b/media/libheadtracking/Twist.cpp
new file mode 100644
index 0000000..664c4d5
--- /dev/null
+++ b/media/libheadtracking/Twist.cpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "media/Twist.h"
+
+#include "QuaternionUtil.h"
+
+namespace android {
+namespace media {
+
+Pose3f integrate(const Twist3f& twist, float dt) {
+ Eigen::Vector3f translation = twist.translationalVelocity() * dt;
+ Eigen::Vector3f rotationVector = twist.rotationalVelocity() * dt;
+ return Pose3f(translation, rotationVectorToQuaternion(rotationVector));
+}
+
+Twist3f differentiate(const Pose3f& pose, float dt) {
+ Eigen::Vector3f translationalVelocity = pose.translation() / dt;
+ Eigen::Vector3f rotationalVelocity = quaternionToRotationVector(pose.rotation()) / dt;
+ return Twist3f(translationalVelocity, rotationalVelocity);
+}
+
+std::ostream& operator<<(std::ostream& os, const Twist3f& twist) {
+ os << "translation: " << twist.translationalVelocity().transpose()
+ << " rotation vector: " << twist.rotationalVelocity().transpose();
+ return os;
+}
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/include/media/HeadTrackingMode.h b/media/libheadtracking/include/media/HeadTrackingMode.h
new file mode 100644
index 0000000..38496e8
--- /dev/null
+++ b/media/libheadtracking/include/media/HeadTrackingMode.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+namespace android {
+namespace media {
+
+/**
+ * Mode of head-tracking.
+ */
+enum class HeadTrackingMode {
+ /** No head-tracking - screen-to-head pose is assumed to be identity. */
+ STATIC,
+ /** Head tracking enabled - world-to-screen pose is assumed to be identity. */
+ WORLD_RELATIVE,
+ /** Full screen-to-head tracking enabled. */
+ SCREEN_RELATIVE,
+};
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/include/media/HeadTrackingProcessor.h b/media/libheadtracking/include/media/HeadTrackingProcessor.h
new file mode 100644
index 0000000..23de540
--- /dev/null
+++ b/media/libheadtracking/include/media/HeadTrackingProcessor.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <limits>
+
+#include "HeadTrackingMode.h"
+#include "Pose.h"
+#include "Twist.h"
+
+namespace android {
+namespace media {
+
+/**
+ * Main entry-point for this library.
+ * This interfaces encompasses all the processing required for determining the head-to-stage pose
+ * used for audio virtualization.
+ * The usage involves periodic setting of the inputs, calling calculate() and obtaining the outputs.
+ * This class is not thread-safe, but thread-compatible.
+ */
+class HeadTrackingProcessor {
+ public:
+ virtual ~HeadTrackingProcessor() = default;
+
+ struct Options {
+ float maxTranslationalVelocity = std::numeric_limits<float>::infinity();
+ float maxRotationalVelocity = std::numeric_limits<float>::infinity();
+ float translationalDriftTimeConstant = std::numeric_limits<float>::infinity();
+ float rotationalDriftTimeConstant = std::numeric_limits<float>::infinity();
+ int64_t freshnessTimeout = std::numeric_limits<int64_t>::max();
+ float predictionDuration = 0;
+ };
+
+ /** Sets the desired head-tracking mode. */
+ virtual void setDesiredMode(HeadTrackingMode mode) = 0;
+
+ /**
+ * Sets the world-to-head pose and head twist (velocity).
+ * headTwist is given in the head coordinate frame.
+ */
+ virtual void setWorldToHeadPose(int64_t timestamp, const Pose3f& worldToHead,
+ const Twist3f& headTwist) = 0;
+
+ /**
+ * Sets the world-to-screen pose.
+ */
+ virtual void setWorldToScreenPose(int64_t timestamp, const Pose3f& worldToScreen) = 0;
+
+ /**
+ * Set the screen-to-stage pose, used in all modes.
+ */
+ virtual void setScreenToStagePose(const Pose3f& screenToStage) = 0;
+
+ /**
+ * Sets the display orientation.
+ * Orientation is expressed in the angle of rotation from the physical "up" side of the screen
+ * to the logical "up" side of the content displayed the screen. Counterclockwise angles, as
+ * viewed while facing the screen are positive.
+ */
+ virtual void setDisplayOrientation(float physicalToLogicalAngle) = 0;
+
+ /**
+ * Process all the previous inputs and update the outputs.
+ */
+ virtual void calculate(int64_t timestamp) = 0;
+
+ /**
+ * Get the aggregate head-to-stage pose (primary output of this module).
+ */
+ virtual Pose3f getHeadToStagePose() const = 0;
+
+ /**
+ * Get the actual head-tracking mode (which may deviate from the desired one as mentioned in the
+ * class documentation above).
+ */
+ virtual HeadTrackingMode getActualMode() const = 0;
+
+ /**
+ * This causes the current poses for both the head and screen to be considered "center".
+ */
+ virtual void recenter() = 0;
+};
+
+/**
+ * Creates an instance featuring a default implementation of the HeadTrackingProcessor interface.
+ */
+std::unique_ptr<HeadTrackingProcessor> createHeadTrackingProcessor(
+ const HeadTrackingProcessor::Options& options,
+ HeadTrackingMode initialMode = HeadTrackingMode::STATIC);
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/include/media/Pose.h b/media/libheadtracking/include/media/Pose.h
new file mode 100644
index 0000000..e660bb9
--- /dev/null
+++ b/media/libheadtracking/include/media/Pose.h
@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <optional>
+#include <vector>
+#include <Eigen/Geometry>
+
+namespace android {
+namespace media {
+
+/**
+ * A 6-DoF pose.
+ * This class represents a proper rigid transformation (translation + rotation) between a reference
+ * frame and a target frame,
+ *
+ * See https://en.wikipedia.org/wiki/Six_degrees_of_freedom
+ */
+class Pose3f {
+ public:
+ /** Typical precision for isApprox comparisons. */
+ static constexpr float kDummyPrecision = 1e-5f;
+
+ Pose3f(const Eigen::Vector3f& translation, const Eigen::Quaternionf& rotation)
+ : mTranslation(translation), mRotation(rotation) {}
+
+ explicit Pose3f(const Eigen::Vector3f& translation)
+ : Pose3f(translation, Eigen::Quaternionf::Identity()) {}
+
+ explicit Pose3f(const Eigen::Quaternionf& rotation)
+ : Pose3f(Eigen::Vector3f::Zero(), rotation) {}
+
+ Pose3f() : Pose3f(Eigen::Vector3f::Zero(), Eigen::Quaternionf::Identity()) {}
+
+ Pose3f(const Pose3f& other) { *this = other; }
+
+ /**
+ * Create instance from a vector-of-floats representation.
+ * The vector is expected to have exactly 6 elements, where the first three are a translation
+ * vector and the last three are a rotation vector.
+ *
+ * Returns nullopt if the input vector is illegal.
+ */
+ static std::optional<Pose3f> fromVector(const std::vector<float>& vec);
+
+ /**
+ * Convert instance to a vector-of-floats representation.
+ * The vector will have exactly 6 elements, where the first three are a translation vector and
+ * the last three are a rotation vector.
+ */
+ std::vector<float> toVector() const;
+
+ Pose3f& operator=(const Pose3f& other) {
+ mTranslation = other.mTranslation;
+ mRotation = other.mRotation;
+ return *this;
+ }
+
+ Eigen::Vector3f translation() const { return mTranslation; };
+ Eigen::Quaternionf rotation() const { return mRotation; };
+
+ /**
+ * Reverses the reference and target frames.
+ */
+ Pose3f inverse() const {
+ Eigen::Quaternionf invRotation = mRotation.inverse();
+ return Pose3f(-(invRotation * translation()), invRotation);
+ }
+
+ /**
+ * Composes (chains) together two poses. By convention, this only makes sense if the target
+ * frame of the left-hand pose is the same the reference frame of the right-hand pose.
+ * Note that this operator is not commutative.
+ */
+ Pose3f operator*(const Pose3f& other) const {
+ Pose3f result = *this;
+ result *= other;
+ return result;
+ }
+
+ Pose3f& operator*=(const Pose3f& other) {
+ mTranslation += mRotation * other.mTranslation;
+ mRotation *= other.mRotation;
+ return *this;
+ }
+
+ /**
+ * This is an imprecise "fuzzy" comparison, which is only to be used for validity-testing
+ * purposes.
+ */
+ bool isApprox(const Pose3f& other, float prec = kDummyPrecision) const {
+ return (mTranslation - other.mTranslation).norm() < prec &&
+ // Quaternions are equivalent under sign inversion.
+ ((mRotation.coeffs() - other.mRotation.coeffs()).norm() < prec ||
+ (mRotation.coeffs() + other.mRotation.coeffs()).norm() < prec);
+ }
+
+ private:
+ Eigen::Vector3f mTranslation;
+ Eigen::Quaternionf mRotation;
+};
+
+/**
+ * Pretty-printer for Pose3f.
+ */
+std::ostream& operator<<(std::ostream& os, const Pose3f& pose);
+
+/**
+ * Move between the 'from' pose and the 'to' pose, while making sure velocity limits are enforced.
+ * If velocity limits are not violated, returns the 'to' pose and false.
+ * If velocity limits are violated, returns pose farthest along the path that can be reached within
+ * the limits, and true.
+ */
+std::tuple<Pose3f, bool> moveWithRateLimit(const Pose3f& from, const Pose3f& to, float t,
+ float maxTranslationalVelocity,
+ float maxRotationalVelocity);
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/include/media/SensorPoseProvider.h b/media/libheadtracking/include/media/SensorPoseProvider.h
new file mode 100644
index 0000000..7b79704
--- /dev/null
+++ b/media/libheadtracking/include/media/SensorPoseProvider.h
@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <chrono>
+#include <memory>
+#include <optional>
+
+#include <android/sensor.h>
+
+#include "Pose.h"
+#include "Twist.h"
+
+namespace android {
+namespace media {
+
+/**
+ * A utility providing streaming of pose data from motion sensors provided by the Sensor Framework.
+ *
+ * A live instance of this interface keeps around some resources required for accessing sensor
+ * readings (e.g. a thread and a queue). Those would be released when the instance is deleted.
+ *
+ * Once alive, individual sensors can be subscribed to using startSensor() and updates can be
+ * stopped via stopSensor(). Those two methods should not be called concurrently and correct usage
+ * is assumed.
+ */
+class SensorPoseProvider {
+ public:
+ static constexpr int32_t INVALID_HANDLE = ASENSOR_INVALID;
+
+ /**
+ * Interface for consuming pose-related sensor events.
+ *
+ * The listener will be provided with a stream of events, each including:
+ * - A handle of the sensor responsible for the event.
+ * - Timestamp.
+ * - Pose.
+ * - Optional twist (time-derivative of pose).
+ *
+ * Sensors having only orientation data will have the translation part of the pose set to
+ * identity.
+ *
+ * Events are delivered in a serialized manner (i.e. callbacks do not need to be reentrant).
+ * Callbacks should not block.
+ */
+ class Listener {
+ public:
+ virtual ~Listener() = default;
+
+ virtual void onPose(int64_t timestamp, int32_t handle, const Pose3f& pose,
+ const std::optional<Twist3f>& twist) = 0;
+ };
+
+ /**
+ * Creates a new SensorPoseProvider instance.
+ * Events will be delivered to the listener as long as the returned instance is kept alive.
+ * @param packageName Client's package name.
+ * @param listener The listener that will get the events.
+ * @return The new instance, or nullptr in case of failure.
+ */
+ static std::unique_ptr<SensorPoseProvider> create(const char* packageName, Listener* listener);
+
+ virtual ~SensorPoseProvider() = default;
+
+ /**
+ * Start receiving pose updates from a given sensor.
+ * @param sensor The sensor to subscribe to.
+ * @param samplingPeriod Sampling interval, in microseconds. Actual rate might be slightly
+ * different.
+ * @return A handle, which can be later used for stopSensor(). INVALID_HANDLE would be returned
+ * in case of error.
+ */
+ virtual int32_t startSensor(const ASensor* sensor,
+ std::chrono::microseconds samplingPeriod) = 0;
+
+ /**
+ * Stop a sensor, previously started with startSensor(). It is not required to stop all sensors
+ * before deleting the SensorPoseProvider instance.
+ * @param handle The sensor handle, as returned from startSensor().
+ */
+ virtual void stopSensor(int32_t handle) = 0;
+};
+
+} // namespace media
+} // namespace android
diff --git a/media/libheadtracking/include/media/Twist.h b/media/libheadtracking/include/media/Twist.h
new file mode 100644
index 0000000..e2fc203
--- /dev/null
+++ b/media/libheadtracking/include/media/Twist.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <Eigen/Geometry>
+
+#include "Pose.h"
+
+namespace android {
+namespace media {
+
+/**
+ * A 6-DoF twist.
+ * This class represents the translational and rotational velocity of a rigid object, typically
+ * relative to its own coordinate-frame.
+ * It is created by two 3-vectors, one representing linear motion per time-unit and the other, a
+ * rotation-vector in radians per time-unit (right-handed).
+ */
+class Twist3f {
+ public:
+ Twist3f(const Eigen::Vector3f& translationalVelocity, const Eigen::Vector3f& rotationalVelocity)
+ : mTranslationalVelocity(translationalVelocity), mRotationalVelocity(rotationalVelocity) {}
+
+ Twist3f() : Twist3f(Eigen::Vector3f::Zero(), Eigen::Vector3f::Zero()) {}
+
+ Twist3f(const Twist3f& other) { *this = other; }
+
+ Twist3f& operator=(const Twist3f& other) {
+ mTranslationalVelocity = other.mTranslationalVelocity;
+ mRotationalVelocity = other.mRotationalVelocity;
+ return *this;
+ }
+
+ Eigen::Vector3f translationalVelocity() const { return mTranslationalVelocity; }
+ Eigen::Vector3f rotationalVelocity() const { return mRotationalVelocity; }
+
+ float scalarTranslationalVelocity() const { return mTranslationalVelocity.norm(); }
+ float scalarRotationalVelocity() const { return mRotationalVelocity.norm(); }
+
+ bool isApprox(const Twist3f& other,
+ float prec = Eigen::NumTraits<float>::dummy_precision()) const {
+ return mTranslationalVelocity.isApprox(other.mTranslationalVelocity, prec) &&
+ mRotationalVelocity.isApprox(other.mRotationalVelocity, prec);
+ }
+
+ private:
+ Eigen::Vector3f mTranslationalVelocity;
+ Eigen::Vector3f mRotationalVelocity;
+};
+
+/**
+ * Integrate a twist over time to obtain a pose.
+ * dt is the time over which to integration.
+ * The resulting pose represents the transformation between the starting point and the ending point
+ * of the motion over the time period.
+ */
+Pose3f integrate(const Twist3f& twist, float dt);
+
+/**
+ * Differentiate pose to obtain a twist.
+ * dt is the time of the motion between the reference and the target frames of the pose.
+ */
+Twist3f differentiate(const Pose3f& pose, float dt);
+
+/**
+ * Pretty-printer for twist.
+ */
+std::ostream& operator<<(std::ostream& os, const Twist3f& twist);
+
+} // namespace media
+} // namespace android
diff --git a/media/libmediahelper/TypeConverter.cpp b/media/libmediahelper/TypeConverter.cpp
index d3a517f..97b5b95 100644
--- a/media/libmediahelper/TypeConverter.cpp
+++ b/media/libmediahelper/TypeConverter.cpp
@@ -50,6 +50,8 @@
MAKE_STRING_FROM_ENUM(AUDIO_FLAG_MUTE_HAPTIC),
MAKE_STRING_FROM_ENUM(AUDIO_FLAG_NO_SYSTEM_CAPTURE),
MAKE_STRING_FROM_ENUM(AUDIO_FLAG_CAPTURE_PRIVATE),
+ MAKE_STRING_FROM_ENUM(AUDIO_FLAG_CONTENT_SPATIALIZED),
+ MAKE_STRING_FROM_ENUM(AUDIO_FLAG_NEVER_SPATIALIZE),
TERMINATOR
};
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index 1aa1848..1e64538 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -5395,21 +5395,21 @@
err = mOMXNode->getParameter(
(OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAacDrcPresentation,
&presentation, sizeof(presentation));
- if (err != OK) {
- return err;
+ if (err == OK) {
+ notify->setInt32("aac-encoded-target-level",
+ presentation.nEncodedTargetLevel);
+ notify->setInt32("aac-drc-cut-level", presentation.nDrcCut);
+ notify->setInt32("aac-drc-boost-level", presentation.nDrcBoost);
+ notify->setInt32("aac-drc-heavy-compression",
+ presentation.nHeavyCompression);
+ notify->setInt32("aac-target-ref-level",
+ presentation.nTargetReferenceLevel);
+ notify->setInt32("aac-drc-effect-type",
+ presentation.nDrcEffectType);
+ notify->setInt32("aac-drc-album-mode", presentation.nDrcAlbumMode);
+ notify->setInt32("aac-drc-output-loudness",
+ presentation.nDrcOutputLoudness);
}
- notify->setInt32("aac-encoded-target-level",
- presentation.nEncodedTargetLevel);
- notify->setInt32("aac-drc-cut-level", presentation.nDrcCut);
- notify->setInt32("aac-drc-boost-level", presentation.nDrcBoost);
- notify->setInt32("aac-drc-heavy-compression",
- presentation.nHeavyCompression);
- notify->setInt32("aac-target-ref-level",
- presentation.nTargetReferenceLevel);
- notify->setInt32("aac-drc-effect-type", presentation.nDrcEffectType);
- notify->setInt32("aac-drc-album-mode", presentation.nDrcAlbumMode);
- notify->setInt32("aac-drc-output-loudness",
- presentation.nDrcOutputLoudness);
}
}
break;
diff --git a/media/ndk/NdkMediaFormat.cpp b/media/ndk/NdkMediaFormat.cpp
index 51f6c78..69ab242 100644
--- a/media/ndk/NdkMediaFormat.cpp
+++ b/media/ndk/NdkMediaFormat.cpp
@@ -354,6 +354,11 @@
EXPORT const char* AMEDIAFORMAT_KEY_MIME = "mime";
EXPORT const char* AMEDIAFORMAT_KEY_MPEG_USER_DATA = "mpeg-user-data";
EXPORT const char* AMEDIAFORMAT_KEY_MPEG2_STREAM_HEADER = "mpeg2-stream-header";
+EXPORT const char* AMEDIAFORMAT_KEY_MPEGH_COMPATIBLE_SETS = "mpegh-compatible-sets";
+EXPORT const char* AMEDIAFORMAT_KEY_MPEGH_PROFILE_LEVEL_INDICATION =
+ "mpegh-profile-level-indication";
+EXPORT const char* AMEDIAFORMAT_KEY_MPEGH_REFERENCE_CHANNEL_LAYOUT =
+ "mpegh-reference-channel-layout";
EXPORT const char* AMEDIAFORMAT_KEY_OPERATING_RATE = "operating-rate";
EXPORT const char* AMEDIAFORMAT_KEY_PCM_ENCODING = "pcm-encoding";
EXPORT const char* AMEDIAFORMAT_KEY_PRIORITY = "priority";
diff --git a/media/ndk/include/media/NdkMediaFormat.h b/media/ndk/include/media/NdkMediaFormat.h
index fbd855d..2d2fcc0 100644
--- a/media/ndk/include/media/NdkMediaFormat.h
+++ b/media/ndk/include/media/NdkMediaFormat.h
@@ -320,6 +320,34 @@
extern const char* AMEDIAFORMAT_VIDEO_QP_P_MAX __INTRODUCED_IN(31);
extern const char* AMEDIAFORMAT_VIDEO_QP_P_MIN __INTRODUCED_IN(31);
+/**
+ * MPEG-H audio profile and level compatibility.
+ *
+ * See FDAmd_2 of ISO_IEC_23008-3;2019 MHAProfileAndLevelCompatibilitySetBox.
+ *
+ * Available since API level 32.
+ */
+extern const char* AMEDIAFORMAT_KEY_MPEGH_COMPATIBLE_SETS __INTRODUCED_IN(32);
+
+/**
+ * MPEG-H audio profile level indication.
+ *
+ * See ISO_IEC_23008-3;2019 MHADecoderConfigurationRecord mpegh3daProfileLevelIndication.
+ *
+ * Available since API level 32.
+ */
+extern const char* AMEDIAFORMAT_KEY_MPEGH_PROFILE_LEVEL_INDICATION __INTRODUCED_IN(32);
+
+/**
+ * MPEG-H audio reference channel layout.
+ *
+ * See ISO_IEC_23008-3;2019 MHADecoderConfigurationRecord referenceChannelLayout
+ * and ISO_IEC_23001‐8 ChannelConfiguration value.
+ *
+ * Available since API level 32.
+ */
+extern const char* AMEDIAFORMAT_KEY_MPEGH_REFERENCE_CHANNEL_LAYOUT __INTRODUCED_IN(32);
+
__END_DECLS
#endif // _NDK_MEDIA_FORMAT_H
diff --git a/media/ndk/libmediandk.map.txt b/media/ndk/libmediandk.map.txt
index 7e9e57e..6f275c7 100644
--- a/media/ndk/libmediandk.map.txt
+++ b/media/ndk/libmediandk.map.txt
@@ -126,6 +126,9 @@
AMEDIAFORMAT_KEY_MIME; # var introduced=21
AMEDIAFORMAT_KEY_MPEG_USER_DATA; # var introduced=28
AMEDIAFORMAT_KEY_MPEG2_STREAM_HEADER; # var introduced=29
+ AMEDIAFORMAT_KEY_MPEGH_COMPATIBLE_SETS; # var introduced=32
+ AMEDIAFORMAT_KEY_MPEGH_PROFILE_LEVEL_INDICATION; # var introduced=32
+ AMEDIAFORMAT_KEY_MPEGH_REFERENCE_CHANNEL_LAYOUT; # var introduced=32
AMEDIAFORMAT_KEY_OPERATING_RATE; # var introduced=28
AMEDIAFORMAT_KEY_PCM_BIG_ENDIAN; # var introduced=29
AMEDIAFORMAT_KEY_PCM_ENCODING; # var introduced=28
diff --git a/media/utils/Android.bp b/media/utils/Android.bp
index bfe73d5..73c4e3b 100644
--- a/media/utils/Android.bp
+++ b/media/utils/Android.bp
@@ -81,6 +81,36 @@
export_include_dirs: ["include"],
}
+cc_library {
+ name: "libmediautils_vendor",
+ vendor_available: true, // required for platform/hardware/interfaces
+ srcs: [
+ "MemoryLeakTrackUtil.cpp",
+ ],
+
+ cflags: [
+ "-Wall",
+ "-Wextra",
+ "-Werror",
+ ],
+ shared_libs: [
+ "liblog",
+ "libutils",
+ ],
+
+ static_libs: [
+ "libc_malloc_debug_backtrace",
+ ],
+
+ header_libs: [
+ "bionic_libc_platform_headers",
+ ],
+
+ local_include_dirs: ["include"],
+ export_include_dirs: ["include"],
+}
+
+
cc_library_headers {
name: "libmediautils_headers",
vendor_available: true, // required for platform/hardware/interfaces
diff --git a/services/audioflinger/AudioFlinger.cpp b/services/audioflinger/AudioFlinger.cpp
index 65a163f..a77a27d 100644
--- a/services/audioflinger/AudioFlinger.cpp
+++ b/services/audioflinger/AudioFlinger.cpp
@@ -336,11 +336,11 @@
}
// getDefaultVibratorInfo_l must be called with AudioFlinger lock held.
-const media::AudioVibratorInfo* AudioFlinger::getDefaultVibratorInfo_l() {
+std::optional<media::AudioVibratorInfo> AudioFlinger::getDefaultVibratorInfo_l() {
if (mAudioVibratorInfos.empty()) {
- return nullptr;
+ return {};
}
- return &mAudioVibratorInfos.front();
+ return mAudioVibratorInfos.front();
}
AudioFlinger::~AudioFlinger()
@@ -695,7 +695,7 @@
// dump all hardware devs
for (size_t i = 0; i < mAudioHwDevs.size(); i++) {
sp<DeviceHalInterface> dev = mAudioHwDevs.valueAt(i)->hwDevice();
- dev->dump(fd);
+ dev->dump(fd, args);
}
mPatchPanel.dump(fd);
@@ -2455,6 +2455,10 @@
ThreadBase *thread = (ThreadBase *)mRecordThreads.valueAt(i).get();
thread->systemReady();
}
+ for (size_t i = 0; i < mMmapThreads.size(); i++) {
+ ThreadBase *thread = (ThreadBase *)mMmapThreads.valueAt(i).get();
+ thread->systemReady();
+ }
return NO_ERROR;
}
@@ -2501,7 +2505,8 @@
sp<AudioFlinger::ThreadBase> AudioFlinger::openOutput_l(audio_module_handle_t module,
audio_io_handle_t *output,
- audio_config_t *config,
+ audio_config_t *halConfig,
+ audio_config_base_t *mixerConfig __unused,
audio_devices_t deviceType,
const String8& address,
audio_output_flags_t flags)
@@ -2529,16 +2534,16 @@
// Check only for Normal Mixing mode
if (kEnableExtendedPrecision) {
// Specify format (uncomment one below to choose)
- //config->format = AUDIO_FORMAT_PCM_FLOAT;
- //config->format = AUDIO_FORMAT_PCM_24_BIT_PACKED;
- //config->format = AUDIO_FORMAT_PCM_32_BIT;
- //config->format = AUDIO_FORMAT_PCM_8_24_BIT;
- // ALOGV("openOutput_l() upgrading format to %#08x", config->format);
+ //halConfig->format = AUDIO_FORMAT_PCM_FLOAT;
+ //halConfig->format = AUDIO_FORMAT_PCM_24_BIT_PACKED;
+ //halConfig->format = AUDIO_FORMAT_PCM_32_BIT;
+ //halConfig->format = AUDIO_FORMAT_PCM_8_24_BIT;
+ // ALOGV("openOutput_l() upgrading format to %#08x", halConfig->format);
}
if (kEnableExtendedChannels) {
// Specify channel mask (uncomment one below to choose)
- //config->channel_mask = audio_channel_out_mask_from_count(4); // for USB 4ch
- //config->channel_mask = audio_channel_mask_from_representation_and_bits(
+ //halConfig->channel_mask = audio_channel_out_mask_from_count(4); // for USB 4ch
+ //halConfig->channel_mask = audio_channel_mask_from_representation_and_bits(
// AUDIO_CHANNEL_REPRESENTATION_INDEX, (1 << 4) - 1); // another 4ch example
}
}
@@ -2549,7 +2554,7 @@
*output,
deviceType,
flags,
- config,
+ halConfig,
address.string());
mHardwareStatus = AUDIO_HW_IDLE;
@@ -2564,13 +2569,20 @@
return thread;
} else {
sp<PlaybackThread> thread;
- if (flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) {
+ //TODO: b/193496180 use spatializer flag at audio HAL when available
+ if (flags == (audio_output_flags_t)(AUDIO_OUTPUT_FLAG_FAST
+ | AUDIO_OUTPUT_FLAG_DEEP_BUFFER)) {
+ thread = new SpatializerThread(this, outputStream, *output,
+ mSystemReady, mixerConfig);
+ ALOGD("openOutput_l() created virtualizer output: ID %d thread %p",
+ *output, thread.get());
+ } else if (flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) {
thread = new OffloadThread(this, outputStream, *output, mSystemReady);
ALOGV("openOutput_l() created offload output: ID %d thread %p",
*output, thread.get());
} else if ((flags & AUDIO_OUTPUT_FLAG_DIRECT)
- || !isValidPcmSinkFormat(config->format)
- || !isValidPcmSinkChannelMask(config->channel_mask)) {
+ || !isValidPcmSinkFormat(halConfig->format)
+ || !isValidPcmSinkChannelMask(halConfig->channel_mask)) {
thread = new DirectOutputThread(this, outputStream, *output, mSystemReady);
ALOGV("openOutput_l() created direct output: ID %d thread %p",
*output, thread.get());
@@ -2597,8 +2609,10 @@
{
audio_module_handle_t module = VALUE_OR_RETURN_STATUS(
aidl2legacy_int32_t_audio_module_handle_t(request.module));
- audio_config_t config = VALUE_OR_RETURN_STATUS(
- aidl2legacy_AudioConfig_audio_config_t(request.config));
+ audio_config_t halConfig = VALUE_OR_RETURN_STATUS(
+ aidl2legacy_AudioConfig_audio_config_t(request.halConfig));
+ audio_config_base_t mixerConfig = VALUE_OR_RETURN_STATUS(
+ aidl2legacy_AudioConfigBase_audio_config_base_t(request.mixerConfig));
sp<DeviceDescriptorBase> device = VALUE_OR_RETURN_STATUS(
aidl2legacy_DeviceDescriptorBase(request.device));
audio_output_flags_t flags = VALUE_OR_RETURN_STATUS(
@@ -2611,9 +2625,9 @@
"Channels %#x, flags %#x",
this, module,
device->toString().c_str(),
- config.sample_rate,
- config.format,
- config.channel_mask,
+ halConfig.sample_rate,
+ halConfig.format,
+ halConfig.channel_mask,
flags);
audio_devices_t deviceType = device->type();
@@ -2625,7 +2639,8 @@
Mutex::Autolock _l(mLock);
- sp<ThreadBase> thread = openOutput_l(module, &output, &config, deviceType, address, flags);
+ sp<ThreadBase> thread = openOutput_l(module, &output, &halConfig,
+ &mixerConfig, deviceType, address, flags);
if (thread != 0) {
if ((flags & AUDIO_OUTPUT_FLAG_MMAP_NOIRQ) == 0) {
PlaybackThread *playbackThread = (PlaybackThread *)thread.get();
@@ -2650,7 +2665,8 @@
mmapThread->ioConfigChanged(AUDIO_OUTPUT_OPENED);
}
response->output = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_io_handle_t_int32_t(output));
- response->config = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_config_t_AudioConfig(config));
+ response->config =
+ VALUE_OR_RETURN_STATUS(legacy2aidl_audio_config_t_AudioConfig(halConfig));
response->latencyMs = VALUE_OR_RETURN_STATUS(convertIntegral<int32_t>(latencyMs));
response->flags = VALUE_OR_RETURN_STATUS(
legacy2aidl_audio_output_flags_t_int32_t_mask(flags));
@@ -3744,7 +3760,7 @@
ALOGV("%s device type %#x address %s", __func__, device.mType, device.getAddress());
handle = mDeviceEffectManager.createEffect_l(
&descOut, device, client, effectClient, mPatchPanel.patches_l(),
- &enabledOut, &lStatus, probe);
+ &enabledOut, &lStatus, probe, request.notifyFramesProcessed);
if (lStatus != NO_ERROR && lStatus != ALREADY_EXISTS) {
// remove local strong reference to Client with mClientLock held
Mutex::Autolock _cl(mClientLock);
@@ -3797,7 +3813,8 @@
io = mPlaybackThreads.keyAt(0);
}
ALOGV("createEffect() got io %d for effect %s", io, descOut.name);
- } else if (checkPlaybackThread_l(io) != nullptr) {
+ } else if (checkPlaybackThread_l(io) != nullptr
+ && sessionId != AUDIO_SESSION_OUTPUT_STAGE) {
// allow only one effect chain per sessionId on mPlaybackThreads.
for (size_t i = 0; i < mPlaybackThreads.size(); i++) {
const audio_io_handle_t checkIo = mPlaybackThreads.keyAt(i);
@@ -3863,7 +3880,8 @@
}
}
handle = thread->createEffect_l(client, effectClient, priority, sessionId,
- &descOut, &enabledOut, &lStatus, pinned, probe);
+ &descOut, &enabledOut, &lStatus, pinned, probe,
+ request.notifyFramesProcessed);
if (lStatus != NO_ERROR && lStatus != ALREADY_EXISTS) {
// remove local strong reference to Client with mClientLock held
Mutex::Autolock _cl(mClientLock);
@@ -4178,6 +4196,7 @@
case TransactionCode::LIST_AUDIO_PATCHES:
case TransactionCode::SET_AUDIO_PORT_CONFIG:
case TransactionCode::SET_RECORD_SILENCED:
+ case TransactionCode::AUDIO_POLICY_READY:
ALOGW("%s: transaction %d received from PID %d",
__func__, code, IPCThreadState::self()->getCallingPid());
// return status only for non void methods
diff --git a/services/audioflinger/AudioFlinger.h b/services/audioflinger/AudioFlinger.h
index fff61f8..8fcd6e4 100644
--- a/services/audioflinger/AudioFlinger.h
+++ b/services/audioflinger/AudioFlinger.h
@@ -269,6 +269,9 @@
/* Indicate JAVA services are ready (scheduling, power management ...) */
virtual status_t systemReady();
+ virtual status_t audioPolicyReady() { mAudioPolicyReady.store(true); return NO_ERROR; }
+ bool isAudioPolicyReady() const { return mAudioPolicyReady.load(); }
+
virtual status_t getMicrophones(std::vector<media::MicrophoneInfo> *microphones);
@@ -309,7 +312,7 @@
void updateDownStreamPatches_l(const struct audio_patch *patch,
const std::set<audio_io_handle_t> streams);
- const media::AudioVibratorInfo* getDefaultVibratorInfo_l();
+ std::optional<media::AudioVibratorInfo> getDefaultVibratorInfo_l();
private:
// FIXME The 400 is temporarily too high until a leak of writers in media.log is fixed.
@@ -735,7 +738,8 @@
const String8& outputDeviceAddress);
sp<ThreadBase> openOutput_l(audio_module_handle_t module,
audio_io_handle_t *output,
- audio_config_t *config,
+ audio_config_t *halConfig,
+ audio_config_base_t *mixerConfig,
audio_devices_t deviceType,
const String8& address,
audio_output_flags_t flags);
@@ -986,6 +990,7 @@
DeviceEffectManager mDeviceEffectManager;
bool mSystemReady;
+ std::atomic_bool mAudioPolicyReady{};
mediautils::UidInfo mUidInfo;
diff --git a/services/audioflinger/DeviceEffectManager.cpp b/services/audioflinger/DeviceEffectManager.cpp
index cecd52b..53ac5cb 100644
--- a/services/audioflinger/DeviceEffectManager.cpp
+++ b/services/audioflinger/DeviceEffectManager.cpp
@@ -77,7 +77,8 @@
const std::map<audio_patch_handle_t, PatchPanel::Patch>& patches,
int *enabled,
status_t *status,
- bool probe) {
+ bool probe,
+ bool notifyFramesProcessed) {
sp<DeviceEffectProxy> effect;
sp<EffectHandle> handle;
status_t lStatus;
@@ -95,10 +96,12 @@
effect = iter->second;
} else {
effect = new DeviceEffectProxy(device, mMyCallback,
- descriptor, mAudioFlinger.nextUniqueId(AUDIO_UNIQUE_ID_USE_EFFECT));
+ descriptor, mAudioFlinger.nextUniqueId(AUDIO_UNIQUE_ID_USE_EFFECT),
+ notifyFramesProcessed);
}
// create effect handle and connect it to effect module
- handle = new EffectHandle(effect, client, effectClient, 0 /*priority*/);
+ handle = new EffectHandle(effect, client, effectClient, 0 /*priority*/,
+ notifyFramesProcessed);
lStatus = handle->initCheck();
if (lStatus == NO_ERROR) {
lStatus = effect->addHandle(handle.get());
diff --git a/services/audioflinger/DeviceEffectManager.h b/services/audioflinger/DeviceEffectManager.h
index a05f5fe..5cfe2fb 100644
--- a/services/audioflinger/DeviceEffectManager.h
+++ b/services/audioflinger/DeviceEffectManager.h
@@ -37,7 +37,8 @@
const std::map<audio_patch_handle_t, PatchPanel::Patch>& patches,
int *enabled,
status_t *status,
- bool probe);
+ bool probe,
+ bool notifyFramesProcessed);
void createAudioPatch(audio_patch_handle_t handle, const PatchPanel::Patch& patch);
void releaseAudioPatch(audio_patch_handle_t handle);
@@ -163,8 +164,13 @@
bool isOffloadOrMmap() const override { return false; }
uint32_t sampleRate() const override { return 0; }
- audio_channel_mask_t channelMask() const override { return AUDIO_CHANNEL_NONE; }
- uint32_t channelCount() const override { return 0; }
+ audio_channel_mask_t inChannelMask(int id __unused) const override {
+ return AUDIO_CHANNEL_NONE;
+ }
+ uint32_t inChannelCount(int id __unused) const override { return 0; }
+ audio_channel_mask_t outChannelMask() const override { return AUDIO_CHANNEL_NONE; }
+ uint32_t outChannelCount() const override { return 0; }
+
audio_channel_mask_t hapticChannelMask() const override { return AUDIO_CHANNEL_NONE; }
size_t frameCount() const override { return 0; }
uint32_t latency() const override { return 0; }
@@ -190,6 +196,10 @@
wp<EffectChain> chain() const override { return nullptr; }
+ bool isAudioPolicyReady() const override {
+ return mManager.audioFlinger().isAudioPolicyReady();
+ }
+
int newEffectId() { return mManager.audioFlinger().nextUniqueId(AUDIO_UNIQUE_ID_USE_EFFECT); }
status_t addEffectToHal(audio_port_handle_t deviceId,
diff --git a/services/audioflinger/Effects.cpp b/services/audioflinger/Effects.cpp
index b267d88..b80943e 100644
--- a/services/audioflinger/Effects.cpp
+++ b/services/audioflinger/Effects.cpp
@@ -242,6 +242,12 @@
{
Mutex::Autolock _l(mLock);
+
+ if ((isInternal_l() && !mPolicyRegistered)
+ || !getCallback()->isAudioPolicyReady()) {
+ return NO_ERROR;
+ }
+
// register effect when first handle is attached and unregister when last handle is removed
if (mPolicyRegistered != mHandles.size() > 0) {
doRegister = true;
@@ -642,6 +648,13 @@
mState = IDLE;
}
break;
+ case ACTIVE:
+ for (size_t i = 0; i < mHandles.size(); i++) {
+ if (!mHandles[i]->disconnected()) {
+ mHandles[i]->framesProcessed(mConfig.inputCfg.buffer.frameCount);
+ }
+ }
+ break;
default: //IDLE , ACTIVE, DESTROYED
break;
}
@@ -875,9 +888,9 @@
// similar to output EFFECT_FLAG_TYPE_INSERT/REPLACE,
// in which case input channel masks should be used here.
callback = getCallback();
- channelMask = callback->channelMask();
+ channelMask = callback->inChannelMask(mId);
mConfig.inputCfg.channels = channelMask;
- mConfig.outputCfg.channels = channelMask;
+ mConfig.outputCfg.channels = callback->outChannelMask();
if ((mDescriptor.flags & EFFECT_FLAG_TYPE_MASK) == EFFECT_FLAG_TYPE_AUXILIARY) {
if (mConfig.inputCfg.channels != AUDIO_CHANNEL_OUT_MONO) {
@@ -1600,7 +1613,7 @@
return status;
}
-status_t AudioFlinger::EffectModule::setVibratorInfo(const media::AudioVibratorInfo* vibratorInfo)
+status_t AudioFlinger::EffectModule::setVibratorInfo(const media::AudioVibratorInfo& vibratorInfo)
{
if (mStatus != NO_ERROR) {
return mStatus;
@@ -1610,15 +1623,17 @@
return INVALID_OPERATION;
}
+ const size_t paramCount = 3;
std::vector<uint8_t> request(
- sizeof(effect_param_t) + sizeof(int32_t) + 2 * sizeof(float));
+ sizeof(effect_param_t) + sizeof(int32_t) + paramCount * sizeof(float));
effect_param_t *param = (effect_param_t*) request.data();
param->psize = sizeof(int32_t);
- param->vsize = 2 * sizeof(float);
+ param->vsize = paramCount * sizeof(float);
*(int32_t*)param->data = HG_PARAM_VIBRATOR_INFO;
float* vibratorInfoPtr = reinterpret_cast<float*>(param->data + sizeof(int32_t));
- vibratorInfoPtr[0] = vibratorInfo->resonantFrequency;
- vibratorInfoPtr[1] = vibratorInfo->qFactor;
+ vibratorInfoPtr[0] = vibratorInfo.resonantFrequency;
+ vibratorInfoPtr[1] = vibratorInfo.qFactor;
+ vibratorInfoPtr[2] = vibratorInfo.maxAmplitude;
std::vector<uint8_t> response;
status_t status = command(EFFECT_CMD_SET_PARAM, request, sizeof(int32_t), &response);
if (status == NO_ERROR) {
@@ -1708,10 +1723,11 @@
AudioFlinger::EffectHandle::EffectHandle(const sp<EffectBase>& effect,
const sp<AudioFlinger::Client>& client,
const sp<media::IEffectClient>& effectClient,
- int32_t priority)
+ int32_t priority, bool notifyFramesProcessed)
: BnEffect(),
mEffect(effect), mEffectClient(effectClient), mClient(client), mCblk(NULL),
- mPriority(priority), mHasControl(false), mEnabled(false), mDisconnected(false)
+ mPriority(priority), mHasControl(false), mEnabled(false), mDisconnected(false),
+ mNotifyFramesProcessed(notifyFramesProcessed)
{
ALOGV("constructor %p client %p", this, client.get());
@@ -2020,6 +2036,13 @@
}
}
+void AudioFlinger::EffectHandle::framesProcessed(int32_t frames) const
+{
+ if (mEffectClient != 0 && mNotifyFramesProcessed) {
+ mEffectClient->framesProcessed(frames);
+ }
+}
+
void AudioFlinger::EffectHandle::dumpToBuffer(char* buffer, size_t size)
{
bool locked = mCblk != NULL && AudioFlinger::dumpTryLock(mCblk->lock);
@@ -2048,11 +2071,11 @@
mNewLeftVolume(UINT_MAX), mNewRightVolume(UINT_MAX),
mEffectCallback(new EffectCallback(wp<EffectChain>(this), thread))
{
- mStrategy = AudioSystem::getStrategyForStream(AUDIO_STREAM_MUSIC);
sp<ThreadBase> p = thread.promote();
if (p == nullptr) {
return;
}
+ mStrategy = p->getStrategyForStream(AUDIO_STREAM_MUSIC);
mMaxTailBuffers = ((kProcessTailDurationMs * p->sampleRate()) / 1000) /
p->frameCount();
}
@@ -2125,8 +2148,8 @@
if (mInBuffer == NULL) {
return;
}
- const size_t frameSize =
- audio_bytes_per_sample(EFFECT_BUFFER_FORMAT) * mEffectCallback->channelCount();
+ const size_t frameSize = audio_bytes_per_sample(EFFECT_BUFFER_FORMAT)
+ * mEffectCallback->inChannelCount(mEffects[0]->id());
memset(mInBuffer->audioBuffer()->raw, 0, mEffectCallback->frameCount() * frameSize);
mInBuffer->commit();
@@ -2236,6 +2259,9 @@
numSamples * sizeof(int32_t), &halBuffer);
#endif
if (result != OK) return result;
+
+ effect->configure();
+
effect->setInBuffer(halBuffer);
// auxiliary effects output samples to chain input buffer for further processing
// by insert effects
@@ -2303,6 +2329,10 @@
}
}
+ mEffects.insertAt(effect, idx_insert);
+
+ effect->configure();
+
// always read samples from chain input buffer
effect->setInBuffer(mInBuffer);
@@ -2310,14 +2340,13 @@
// output buffer, otherwise to chain input buffer
if (idx_insert == size) {
if (idx_insert != 0) {
- mEffects[idx_insert-1]->setOutBuffer(mInBuffer);
mEffects[idx_insert-1]->configure();
+ mEffects[idx_insert-1]->setOutBuffer(mInBuffer);
}
effect->setOutBuffer(mOutBuffer);
} else {
effect->setOutBuffer(mInBuffer);
}
- mEffects.insertAt(effect, idx_insert);
ALOGV("addEffect_l() effect %p, added in chain %p at rank %zu", effect.get(), this,
idx_insert);
@@ -2350,14 +2379,21 @@
if (type != EFFECT_FLAG_TYPE_AUXILIARY) {
if (i == size - 1 && i != 0) {
- mEffects[i - 1]->setOutBuffer(mOutBuffer);
mEffects[i - 1]->configure();
+ mEffects[i - 1]->setOutBuffer(mOutBuffer);
}
}
mEffects.removeAt(i);
+
+ // make sure the input buffer configuration for the new first effect in the chain
+ // is updated if needed (can switch from HAL channel mask to mixer channel mask)
+ if (i == 0 && size > 1) {
+ mEffects[0]->configure();
+ mEffects[0]->setInBuffer(mInBuffer);
+ }
+
ALOGV("removeEffect_l() effect %p, removed from chain %p at rank %zu", effect.get(),
this, i);
-
break;
}
}
@@ -2932,7 +2968,43 @@
return t->sampleRate();
}
-audio_channel_mask_t AudioFlinger::EffectChain::EffectCallback::channelMask() const {
+audio_channel_mask_t AudioFlinger::EffectChain::EffectCallback::inChannelMask(int id) const {
+ sp<ThreadBase> t = thread().promote();
+ if (t == nullptr) {
+ return AUDIO_CHANNEL_NONE;
+ }
+ sp<EffectChain> c = chain().promote();
+ if (c == nullptr) {
+ return AUDIO_CHANNEL_NONE;
+ }
+
+ if (c->sessionId() != AUDIO_SESSION_OUTPUT_STAGE
+ || c->isFirstEffect(id)) {
+ return t->mixerChannelMask();
+ } else {
+ return t->channelMask();
+ }
+}
+
+uint32_t AudioFlinger::EffectChain::EffectCallback::inChannelCount(int id) const {
+ sp<ThreadBase> t = thread().promote();
+ if (t == nullptr) {
+ return 0;
+ }
+ sp<EffectChain> c = chain().promote();
+ if (c == nullptr) {
+ return 0;
+ }
+
+ if (c->sessionId() != AUDIO_SESSION_OUTPUT_STAGE
+ || c->isFirstEffect(id)) {
+ return audio_channel_count_from_out_mask(t->mixerChannelMask());
+ } else {
+ return t->channelCount();
+ }
+}
+
+audio_channel_mask_t AudioFlinger::EffectChain::EffectCallback::outChannelMask() const {
sp<ThreadBase> t = thread().promote();
if (t == nullptr) {
return AUDIO_CHANNEL_NONE;
@@ -2940,7 +3012,7 @@
return t->channelMask();
}
-uint32_t AudioFlinger::EffectChain::EffectCallback::channelCount() const {
+uint32_t AudioFlinger::EffectChain::EffectCallback::outChannelCount() const {
sp<ThreadBase> t = thread().promote();
if (t == nullptr) {
return 0;
@@ -3143,7 +3215,8 @@
} else {
mHalEffect->setDevices({mDevice});
}
- *handle = new EffectHandle(mHalEffect, nullptr, nullptr, 0 /*priority*/);
+ *handle = new EffectHandle(mHalEffect, nullptr, nullptr, 0 /*priority*/,
+ mNotifyFramesProcessed);
status = (*handle)->initCheck();
if (status == OK) {
status = mHalEffect->addHandle((*handle).get());
@@ -3169,7 +3242,8 @@
int enabled;
*handle = thread->createEffect_l(nullptr, nullptr, 0, AUDIO_SESSION_DEVICE,
const_cast<effect_descriptor_t *>(&mDescriptor),
- &enabled, &status, false, false /*probe*/);
+ &enabled, &status, false, false /*probe*/,
+ mNotifyFramesProcessed);
ALOGV("%s thread->createEffect_l status %d", __func__, status);
} else {
status = BAD_VALUE;
@@ -3364,7 +3438,8 @@
return proxy->sampleRate();
}
-audio_channel_mask_t AudioFlinger::DeviceEffectProxy::ProxyCallback::channelMask() const {
+audio_channel_mask_t AudioFlinger::DeviceEffectProxy::ProxyCallback::inChannelMask(
+ int id __unused) const {
sp<DeviceEffectProxy> proxy = mProxy.promote();
if (proxy == nullptr) {
return AUDIO_CHANNEL_OUT_STEREO;
@@ -3372,7 +3447,23 @@
return proxy->channelMask();
}
-uint32_t AudioFlinger::DeviceEffectProxy::ProxyCallback::channelCount() const {
+uint32_t AudioFlinger::DeviceEffectProxy::ProxyCallback::inChannelCount(int id __unused) const {
+ sp<DeviceEffectProxy> proxy = mProxy.promote();
+ if (proxy == nullptr) {
+ return 2;
+ }
+ return proxy->channelCount();
+}
+
+audio_channel_mask_t AudioFlinger::DeviceEffectProxy::ProxyCallback::outChannelMask() const {
+ sp<DeviceEffectProxy> proxy = mProxy.promote();
+ if (proxy == nullptr) {
+ return AUDIO_CHANNEL_OUT_STEREO;
+ }
+ return proxy->channelMask();
+}
+
+uint32_t AudioFlinger::DeviceEffectProxy::ProxyCallback::outChannelCount() const {
sp<DeviceEffectProxy> proxy = mProxy.promote();
if (proxy == nullptr) {
return 2;
diff --git a/services/audioflinger/Effects.h b/services/audioflinger/Effects.h
index a727e04..389ff7b 100644
--- a/services/audioflinger/Effects.h
+++ b/services/audioflinger/Effects.h
@@ -34,8 +34,10 @@
virtual bool isOffloadOrDirect() const = 0;
virtual bool isOffloadOrMmap() const = 0;
virtual uint32_t sampleRate() const = 0;
- virtual audio_channel_mask_t channelMask() const = 0;
- virtual uint32_t channelCount() const = 0;
+ virtual audio_channel_mask_t inChannelMask(int id) const = 0;
+ virtual uint32_t inChannelCount(int id) const = 0;
+ virtual audio_channel_mask_t outChannelMask() const = 0;
+ virtual uint32_t outChannelCount() const = 0;
virtual audio_channel_mask_t hapticChannelMask() const = 0;
virtual size_t frameCount() const = 0;
@@ -64,6 +66,8 @@
virtual void resetVolume() = 0;
virtual wp<EffectChain> chain() const = 0;
+
+ virtual bool isAudioPolicyReady() const = 0;
};
// EffectBase(EffectModule) and EffectChain classes both have their own mutex to protect
@@ -164,6 +168,16 @@
void dump(int fd, const Vector<String16>& args);
+protected:
+ bool isInternal_l() const {
+ for (auto handle : mHandles) {
+ if (handle->client() != nullptr) {
+ return false;
+ }
+ }
+ return true;
+ }
+
private:
friend class AudioFlinger; // for mHandles
bool mPinned = false;
@@ -259,7 +273,7 @@
bool isHapticGenerator() const;
status_t setHapticIntensity(int id, int intensity);
- status_t setVibratorInfo(const media::AudioVibratorInfo* vibratorInfo);
+ status_t setVibratorInfo(const media::AudioVibratorInfo& vibratorInfo);
void dump(int fd, const Vector<String16>& args);
@@ -327,7 +341,7 @@
EffectHandle(const sp<EffectBase>& effect,
const sp<AudioFlinger::Client>& client,
const sp<media::IEffectClient>& effectClient,
- int32_t priority);
+ int32_t priority, bool notifyFramesProcessed);
virtual ~EffectHandle();
virtual status_t initCheck();
@@ -342,6 +356,8 @@
android::binder::Status disconnect() override;
android::binder::Status getCblk(media::SharedFileRegion* _aidl_return) override;
+ sp<Client> client() const { return mClient; }
+
private:
void disconnect(bool unpinIfLast);
@@ -356,6 +372,8 @@
void setEnabled(bool enabled);
bool enabled() const { return mEnabled; }
+ void framesProcessed(int32_t frames) const;
+
// Getters
wp<EffectBase> effect() const { return mEffect; }
int id() const {
@@ -389,6 +407,8 @@
bool mEnabled; // cached enable state: needed when the effect is
// restored after being suspended
bool mDisconnected; // Set to true by disconnect()
+ const bool mNotifyFramesProcessed; // true if the client callback event
+ // EVENT_FRAMES_PROCESSED must be generated
};
// the EffectChain class represents a group of effects associated to one audio session.
@@ -511,6 +531,8 @@
sp<EffectCallbackInterface> effectCallback() const { return mEffectCallback; }
wp<ThreadBase> thread() const { return mEffectCallback->thread(); }
+ bool isFirstEffect(int id) const { return !mEffects.isEmpty() && id == mEffects[0]->id(); }
+
void dump(int fd, const Vector<String16>& args);
private:
@@ -544,8 +566,10 @@
bool isOffloadOrMmap() const override;
uint32_t sampleRate() const override;
- audio_channel_mask_t channelMask() const override;
- uint32_t channelCount() const override;
+ audio_channel_mask_t inChannelMask(int id) const override;
+ uint32_t inChannelCount(int id) const override;
+ audio_channel_mask_t outChannelMask() const override;
+ uint32_t outChannelCount() const override;
audio_channel_mask_t hapticChannelMask() const override;
size_t frameCount() const override;
uint32_t latency() const override;
@@ -566,6 +590,10 @@
wp<EffectChain> chain() const override { return mChain; }
+ bool isAudioPolicyReady() const override {
+ return mAudioFlinger.isAudioPolicyReady();
+ }
+
wp<ThreadBase> thread() const { return mThread.load(); }
void setThread(const wp<ThreadBase>& thread) {
@@ -643,11 +671,11 @@
public:
DeviceEffectProxy (const AudioDeviceTypeAddr& device,
const sp<DeviceEffectManagerCallback>& callback,
- effect_descriptor_t *desc, int id)
+ effect_descriptor_t *desc, int id, bool notifyFramesProcessed)
: EffectBase(callback, desc, id, AUDIO_SESSION_DEVICE, false),
mDevice(device), mManagerCallback(callback),
- mMyCallback(new ProxyCallback(wp<DeviceEffectProxy>(this),
- callback)) {}
+ mMyCallback(new ProxyCallback(wp<DeviceEffectProxy>(this), callback)),
+ mNotifyFramesProcessed(notifyFramesProcessed) {}
status_t setEnabled(bool enabled, bool fromHandle) override;
sp<DeviceEffectProxy> asDeviceEffectProxy() override { return this; }
@@ -694,8 +722,10 @@
bool isOffloadOrMmap() const override { return false; }
uint32_t sampleRate() const override;
- audio_channel_mask_t channelMask() const override;
- uint32_t channelCount() const override;
+ audio_channel_mask_t inChannelMask(int id) const override;
+ uint32_t inChannelCount(int id) const override;
+ audio_channel_mask_t outChannelMask() const override;
+ uint32_t outChannelCount() const override;
audio_channel_mask_t hapticChannelMask() const override { return AUDIO_CHANNEL_NONE; }
size_t frameCount() const override { return 0; }
uint32_t latency() const override { return 0; }
@@ -716,6 +746,10 @@
wp<EffectChain> chain() const override { return nullptr; }
+ bool isAudioPolicyReady() const override {
+ return mManagerCallback->isAudioPolicyReady();
+ }
+
int newEffectId();
private:
@@ -734,4 +768,5 @@
std::map<audio_patch_handle_t, sp<EffectHandle>> mEffectHandles; // protected by mProxyLock
sp<EffectModule> mHalEffect; // protected by mProxyLock
struct audio_port_config mDevicePort = { .id = AUDIO_PORT_HANDLE_NONE };
+ const bool mNotifyFramesProcessed;
};
diff --git a/services/audioflinger/FastMixer.cpp b/services/audioflinger/FastMixer.cpp
index 88d4eaf..fc34d95 100644
--- a/services/audioflinger/FastMixer.cpp
+++ b/services/audioflinger/FastMixer.cpp
@@ -204,6 +204,8 @@
(void *)(uintptr_t)fastTrack->mHapticPlaybackEnabled);
mMixer->setParameter(index, AudioMixer::TRACK, AudioMixer::HAPTIC_INTENSITY,
(void *)(uintptr_t)fastTrack->mHapticIntensity);
+ mMixer->setParameter(index, AudioMixer::TRACK, AudioMixer::HAPTIC_MAX_AMPLITUDE,
+ (void *)(&(fastTrack->mHapticMaxAmplitude)));
mMixer->enable(index);
break;
diff --git a/services/audioflinger/FastMixerState.h b/services/audioflinger/FastMixerState.h
index 857d3de..ce3cc14 100644
--- a/services/audioflinger/FastMixerState.h
+++ b/services/audioflinger/FastMixerState.h
@@ -17,6 +17,8 @@
#ifndef ANDROID_AUDIO_FAST_MIXER_STATE_H
#define ANDROID_AUDIO_FAST_MIXER_STATE_H
+#include <math.h>
+
#include <audio_utils/minifloat.h>
#include <system/audio.h>
#include <media/AudioMixer.h>
@@ -51,6 +53,7 @@
int mGeneration; // increment when any field is assigned
bool mHapticPlaybackEnabled = false; // haptic playback is enabled or not
os::HapticScale mHapticIntensity = os::HapticScale::MUTE; // intensity of haptic data
+ float mHapticMaxAmplitude = NAN; // max amplitude allowed for haptic data
};
// Represents a single state of the fast mixer
diff --git a/services/audioflinger/PatchPanel.cpp b/services/audioflinger/PatchPanel.cpp
index a381c7d..93118b8 100644
--- a/services/audioflinger/PatchPanel.cpp
+++ b/services/audioflinger/PatchPanel.cpp
@@ -258,6 +258,7 @@
reinterpret_cast<PlaybackThread*>(thread.get()), false /*closeThread*/);
} else {
audio_config_t config = AUDIO_CONFIG_INITIALIZER;
+ audio_config_base_t mixerConfig = AUDIO_CONFIG_BASE_INITIALIZER;
audio_io_handle_t output = AUDIO_IO_HANDLE_NONE;
audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE;
if (patch->sinks[0].config_mask & AUDIO_PORT_CONFIG_SAMPLE_RATE) {
@@ -276,6 +277,7 @@
patch->sinks[0].ext.device.hw_module,
&output,
&config,
+ &mixerConfig,
outputDevice,
outputDeviceAddress,
flags);
diff --git a/services/audioflinger/PlaybackTracks.h b/services/audioflinger/PlaybackTracks.h
index 0929055..b069462 100644
--- a/services/audioflinger/PlaybackTracks.h
+++ b/services/audioflinger/PlaybackTracks.h
@@ -19,6 +19,8 @@
#error This header file should only be included from AudioFlinger.h
#endif
+#include <math.h>
+
// Checks and monitors OP_PLAY_AUDIO
class OpPlayAudioMonitor : public RefBase {
public:
@@ -161,6 +163,8 @@
}
/** Return at what intensity to play haptics, used in mixer. */
os::HapticScale getHapticIntensity() const { return mHapticIntensity; }
+ /** Return the maximum amplitude allowed for haptics data, used in mixer. */
+ float getHapticMaxAmplitude() const { return mHapticMaxAmplitude; }
/** Set intensity of haptic playback, should be set after querying vibrator service. */
void setHapticIntensity(os::HapticScale hapticIntensity) {
if (os::isValidHapticScale(hapticIntensity)) {
@@ -168,6 +172,12 @@
setHapticPlaybackEnabled(mHapticIntensity != os::HapticScale::MUTE);
}
}
+ /** Set maximum amplitude allowed for haptic data, should be set after querying
+ * vibrator service.
+ */
+ void setHapticMaxAmplitude(float maxAmplitude) {
+ mHapticMaxAmplitude = maxAmplitude;
+ }
sp<os::ExternalVibration> getExternalVibration() const { return mExternalVibration; }
void setTeePatches(TeePatches teePatches);
@@ -185,6 +195,10 @@
audio_output_flags_t getOutputFlags() const { return mFlags; }
float getSpeed() const { return mSpeed; }
+
+ bool canBeSpatialized() const { return (mAttr.flags
+ & (AUDIO_FLAG_CONTENT_SPATIALIZED | AUDIO_FLAG_NEVER_SPATIALIZE)) == 0; }
+
protected:
// for numerous
friend class PlaybackThread;
@@ -282,6 +296,8 @@
bool mHapticPlaybackEnabled = false; // indicates haptic playback enabled or not
// intensity to play haptic data
os::HapticScale mHapticIntensity = os::HapticScale::MUTE;
+ // max amplitude allowed for haptic data
+ float mHapticMaxAmplitude = NAN;
class AudioVibrationController : public os::BnExternalVibrationController {
public:
explicit AudioVibrationController(Track* track) : mTrack(track) {}
diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp
index b9cdab8..683c4f7 100644
--- a/services/audioflinger/Threads.cpp
+++ b/services/audioflinger/Threads.cpp
@@ -50,8 +50,10 @@
#include <audio_utils/format.h>
#include <audio_utils/minifloat.h>
#include <audio_utils/safe_math.h>
-#include <system/audio_effects/effect_ns.h>
#include <system/audio_effects/effect_aec.h>
+#include <system/audio_effects/effect_downmix.h>
+#include <system/audio_effects/effect_ns.h>
+#include <system/audio_effects/effect_spatializer.h>
#include <system/audio.h>
// NBAIO implementations
@@ -507,6 +509,8 @@
return "MMAP_PLAYBACK";
case MMAP_CAPTURE:
return "MMAP_CAPTURE";
+ case SPATIALIZER:
+ return "SPATIALIZER";
default:
return "unknown";
}
@@ -722,6 +726,19 @@
sendConfigEvent_l(configEvent);
}
+void AudioFlinger::ThreadBase::sendCheckOutputStageEffectsEvent()
+{
+ Mutex::Autolock _l(mLock);
+ sendCheckOutputStageEffectsEvent_l();
+}
+
+void AudioFlinger::ThreadBase::sendCheckOutputStageEffectsEvent_l()
+{
+ sp<ConfigEvent> configEvent =
+ (ConfigEvent *)new CheckOutputStageEffectsEvent();
+ sendConfigEvent_l(configEvent);
+}
+
// post condition: mConfigEvents.isEmpty()
void AudioFlinger::ThreadBase::processConfigEvents_l()
{
@@ -784,6 +801,11 @@
(ResizeBufferConfigEventData *)event->mData.get();
resizeInputBuffer_l(data->mMaxSharedAudioHistoryMs);
} break;
+
+ case CFG_EVENT_CHECK_OUTPUT_STAGE_EFFECTS: {
+ setCheckOutputStageEffects();
+ } break;
+
default:
ALOG_ASSERT(false, "processConfigEvents_l() unknown event type %d", event->mType);
break;
@@ -1008,6 +1030,8 @@
return String16("MmapPlayback");
case MMAP_CAPTURE:
return String16("MmapCapture");
+ case SPATIALIZER:
+ return String16("AudioSpatial");
default:
ALOG_ASSERT(false);
return String16("AudioUnknown");
@@ -1401,6 +1425,13 @@
return BAD_VALUE;
}
break;
+ case SPATIALIZER:
+ if (!audio_is_global_session(sessionId)) {
+ ALOGW("checkEffectCompatibility_l(): non global effect %s on SPATIALIZER"
+ " thread %s", desc->name, mThreadName);
+ return BAD_VALUE;
+ }
+ break;
default:
LOG_ALWAYS_FATAL("checkEffectCompatibility_l(): wrong thread type %d", mType);
}
@@ -1418,7 +1449,8 @@
int *enabled,
status_t *status,
bool pinned,
- bool probe)
+ bool probe,
+ bool notifyFramesProcessed)
{
sp<EffectModule> effect;
sp<EffectHandle> handle;
@@ -1477,18 +1509,19 @@
if (effect->isHapticGenerator()) {
// TODO(b/184194057): Use the vibrator information from the vibrator that will be used
// for the HapticGenerator.
- const media::AudioVibratorInfo* defaultVibratorInfo =
- mAudioFlinger->getDefaultVibratorInfo_l();
- if (defaultVibratorInfo != nullptr) {
+ const std::optional<media::AudioVibratorInfo> defaultVibratorInfo =
+ std::move(mAudioFlinger->getDefaultVibratorInfo_l());
+ if (defaultVibratorInfo) {
// Only set the vibrator info when it is a valid one.
- effect->setVibratorInfo(defaultVibratorInfo);
+ effect->setVibratorInfo(*defaultVibratorInfo);
}
}
// create effect handle and connect it to effect module
- handle = new EffectHandle(effect, client, effectClient, priority);
+ handle = new EffectHandle(effect, client, effectClient, priority, notifyFramesProcessed);
lStatus = handle->initCheck();
if (lStatus == OK) {
lStatus = effect->addHandle(handle.get());
+ sendCheckOutputStageEffectsEvent_l();
}
if (enabled != NULL) {
*enabled = (int)effect->isEnabled();
@@ -1531,6 +1564,7 @@
if (remove) {
removeEffect_l(effect, true);
}
+ sendCheckOutputStageEffectsEvent_l();
}
if (remove) {
mAudioFlinger->updateOrphanEffectChains(effect);
@@ -1888,6 +1922,14 @@
item->selfrecord();
}
+product_strategy_t AudioFlinger::ThreadBase::getStrategyForStream(audio_stream_type_t stream) const
+{
+ if (!mAudioFlinger->isAudioPolicyReady()) {
+ return PRODUCT_STRATEGY_NONE;
+ }
+ return AudioSystem::getStrategyForStream(stream);
+}
+
// ----------------------------------------------------------------------------
// Playback
// ----------------------------------------------------------------------------
@@ -1896,15 +1938,16 @@
AudioStreamOut* output,
audio_io_handle_t id,
type_t type,
- bool systemReady)
+ bool systemReady,
+ audio_config_base_t *mixerConfig)
: ThreadBase(audioFlinger, id, type, systemReady, true /* isOut */),
mNormalFrameCount(0), mSinkBuffer(NULL),
- mMixerBufferEnabled(AudioFlinger::kEnableExtendedPrecision),
+ mMixerBufferEnabled(AudioFlinger::kEnableExtendedPrecision || type == SPATIALIZER),
mMixerBuffer(NULL),
mMixerBufferSize(0),
mMixerBufferFormat(AUDIO_FORMAT_INVALID),
mMixerBufferValid(false),
- mEffectBufferEnabled(AudioFlinger::kEnableExtendedPrecision),
+ mEffectBufferEnabled(AudioFlinger::kEnableExtendedPrecision || type == SPATIALIZER),
mEffectBuffer(NULL),
mEffectBufferSize(0),
mEffectBufferFormat(AUDIO_FORMAT_INVALID),
@@ -1956,8 +1999,18 @@
mOutput->audioHwDev->moduleName(), AUDIO_HARDWARE_MODULE_ID_MSD) == 0;
}
+ if (mixerConfig != nullptr && mixerConfig->channel_mask != AUDIO_CHANNEL_NONE) {
+ mMixerChannelMask = mixerConfig->channel_mask;
+ }
+
readOutputParameters_l();
+ if (mType != SPATIALIZER
+ && mMixerChannelMask != mChannelMask) {
+ LOG_ALWAYS_FATAL("HAL channel mask %#x does not match mixer channel mask %#x",
+ mChannelMask, mMixerChannelMask);
+ }
+
// TODO: We may also match on address as well as device type for
// AUDIO_DEVICE_OUT_BUS, AUDIO_DEVICE_OUT_ALL_A2DP, AUDIO_DEVICE_OUT_REMOTE_SUBMIX
if (type == MIXER || type == DIRECT || type == OFFLOAD) {
@@ -1986,6 +2039,7 @@
free(mSinkBuffer);
free(mMixerBuffer);
free(mEffectBuffer);
+ free(mEffectToSinkBuffer);
}
// Thread virtuals
@@ -2080,10 +2134,12 @@
write(fd, result.string(), result.size());
}
-void AudioFlinger::PlaybackThread::dumpInternals_l(int fd, const Vector<String16>& args __unused)
+void AudioFlinger::PlaybackThread::dumpInternals_l(int fd, const Vector<String16>& args)
{
dprintf(fd, " Master volume: %f\n", mMasterVolume);
dprintf(fd, " Master mute: %s\n", mMasterMute ? "on" : "off");
+ dprintf(fd, " Mixer channel Mask: %#x (%s)\n",
+ mMixerChannelMask, channelMaskToString(mMixerChannelMask, true /* output */).c_str());
if (mHapticChannelMask != AUDIO_CHANNEL_NONE) {
dprintf(fd, " Haptic channel mask: %#x (%s)\n", mHapticChannelMask,
channelMaskToString(mHapticChannelMask, true /* output */).c_str());
@@ -2109,7 +2165,7 @@
}
if (output != nullptr) {
dprintf(fd, " Hal stream dump:\n");
- (void)output->stream->dump(fd);
+ (void)output->stream->dump(fd, args);
}
}
@@ -2397,11 +2453,11 @@
// all tracks in same audio session must share the same routing strategy otherwise
// conflicts will happen when tracks are moved from one output to another by audio policy
// manager
- product_strategy_t strategy = AudioSystem::getStrategyForStream(streamType);
+ product_strategy_t strategy = getStrategyForStream(streamType);
for (size_t i = 0; i < mTracks.size(); ++i) {
sp<Track> t = mTracks[i];
if (t != 0 && t->isExternalTrack()) {
- product_strategy_t actual = AudioSystem::getStrategyForStream(t->streamType());
+ product_strategy_t actual = getStrategyForStream(t->streamType());
if (sessionId == t->sessionId() && strategy != actual) {
ALOGE("createTrack_l() mismatched strategy; expected %u but found %u",
strategy, actual);
@@ -2445,7 +2501,7 @@
if (chain != 0) {
ALOGV("createTrack_l() setting main buffer %p", chain->inBuffer());
track->setMainBuffer(chain->inBuffer());
- chain->setStrategy(AudioSystem::getStrategyForStream(track->streamType()));
+ chain->setStrategy(getStrategyForStream(track->streamType()));
chain->incTrackCnt();
}
@@ -2613,8 +2669,19 @@
mLock.unlock();
const int intensity = AudioFlinger::onExternalVibrationStart(
track->getExternalVibration());
+ std::optional<media::AudioVibratorInfo> vibratorInfo;
+ {
+ // TODO(b/184194780): Use the vibrator information from the vibrator that will be
+ // used to play this track.
+ Mutex::Autolock _l(mAudioFlinger->mLock);
+ vibratorInfo = std::move(mAudioFlinger->getDefaultVibratorInfo_l());
+ }
mLock.lock();
track->setHapticIntensity(static_cast<os::HapticScale>(intensity));
+ if (vibratorInfo) {
+ track->setHapticMaxAmplitude(vibratorInfo->maxAmplitude);
+ }
+
// Haptic playback should be enabled by vibrator service.
if (track->getHapticPlaybackEnabled()) {
// Disable haptic playback of all active track to ensure only
@@ -2814,14 +2881,20 @@
if (!audio_is_output_channel(mChannelMask)) {
LOG_ALWAYS_FATAL("HAL channel mask %#x not valid for output", mChannelMask);
}
- if ((mType == MIXER || mType == DUPLICATING)
- && !isValidPcmSinkChannelMask(mChannelMask)) {
+ if (hasMixer() && !isValidPcmSinkChannelMask(mChannelMask)) {
LOG_ALWAYS_FATAL("HAL channel mask %#x not supported for mixed output",
mChannelMask);
}
+
+ if (mMixerChannelMask == AUDIO_CHANNEL_NONE) {
+ mMixerChannelMask = mChannelMask;
+ }
+
mChannelCount = audio_channel_count_from_out_mask(mChannelMask);
mBalance.setChannelMask(mChannelMask);
+ uint32_t mixerChannelCount = audio_channel_count_from_out_mask(mMixerChannelMask);
+
// Get actual HAL format.
status_t result = mOutput->stream->getAudioProperties(nullptr, nullptr, &mHALFormat);
LOG_ALWAYS_FATAL_IF(result != OK, "Error when retrieving output stream format: %d", result);
@@ -2831,8 +2904,7 @@
if (!audio_is_valid_format(mFormat)) {
LOG_ALWAYS_FATAL("HAL format %#x not valid for output", mFormat);
}
- if ((mType == MIXER || mType == DUPLICATING)
- && !isValidPcmSinkFormat(mFormat)) {
+ if (hasMixer() && !isValidPcmSinkFormat(mFormat)) {
LOG_FATAL("HAL format %#x not supported for mixed output",
mFormat);
}
@@ -2841,7 +2913,7 @@
LOG_ALWAYS_FATAL_IF(result != OK,
"Error when retrieving output stream buffer size: %d", result);
mFrameCount = mBufferSize / mFrameSize;
- if ((mType == MIXER || mType == DUPLICATING) && (mFrameCount & 15)) {
+ if (hasMixer() && (mFrameCount & 15)) {
ALOGW("HAL output buffer size is %zu frames but AudioMixer requires multiples of 16 frames",
mFrameCount);
}
@@ -2914,7 +2986,7 @@
}
mNormalFrameCount = multiplier * mFrameCount;
// round up to nearest 16 frames to satisfy AudioMixer
- if (mType == MIXER || mType == DUPLICATING) {
+ if (hasMixer()) {
mNormalFrameCount = (mNormalFrameCount + 15) & ~15;
}
ALOGI("HAL output buffer size %zu frames, normal sink buffer size %zu frames", mFrameCount,
@@ -2930,18 +3002,25 @@
// Originally this was int16_t[] array, need to remove legacy implications.
free(mSinkBuffer);
mSinkBuffer = NULL;
+ free(mEffectToSinkBuffer);
+ mEffectToSinkBuffer = nullptr;
+
// For sink buffer size, we use the frame size from the downstream sink to avoid problems
// with non PCM formats for compressed music, e.g. AAC, and Offload threads.
const size_t sinkBufferSize = mNormalFrameCount * mFrameSize;
(void)posix_memalign(&mSinkBuffer, 32, sinkBufferSize);
+ if (mType == SPATIALIZER) {
+ (void)posix_memalign(&mEffectToSinkBuffer, 32, sinkBufferSize);
+ }
+
// We resize the mMixerBuffer according to the requirements of the sink buffer which
// drives the output.
free(mMixerBuffer);
mMixerBuffer = NULL;
if (mMixerBufferEnabled) {
mMixerBufferFormat = AUDIO_FORMAT_PCM_FLOAT; // no longer valid: AUDIO_FORMAT_PCM_16_BIT.
- mMixerBufferSize = mNormalFrameCount * mChannelCount
+ mMixerBufferSize = mNormalFrameCount * mixerChannelCount
* audio_bytes_per_sample(mMixerBufferFormat);
(void)posix_memalign(&mMixerBuffer, 32, mMixerBufferSize);
}
@@ -2949,7 +3028,7 @@
mEffectBuffer = NULL;
if (mEffectBufferEnabled) {
mEffectBufferFormat = EFFECT_BUFFER_FORMAT;
- mEffectBufferSize = mNormalFrameCount * mChannelCount
+ mEffectBufferSize = mNormalFrameCount * mixerChannelCount
* audio_bytes_per_sample(mEffectBufferFormat);
(void)posix_memalign(&mEffectBuffer, 32, mEffectBufferSize);
}
@@ -2958,6 +3037,7 @@
mChannelMask = static_cast<audio_channel_mask_t>(mChannelMask & ~mHapticChannelMask);
mHapticChannelCount = audio_channel_count_from_out_mask(mHapticChannelMask);
mChannelCount -= mHapticChannelCount;
+ mMixerChannelMask = static_cast<audio_channel_mask_t>(mMixerChannelMask & ~mHapticChannelMask);
// force reconfiguration of effect chains and engines to take new buffer size and audio
// parameters into account
@@ -3051,15 +3131,15 @@
// session AUDIO_SESSION_OUTPUT_MIX is placed in same strategy as MUSIC stream so that
// it is moved to correct output by audio policy manager when A2DP is connected or disconnected
if (sessionId == AUDIO_SESSION_OUTPUT_MIX) {
- return AudioSystem::getStrategyForStream(AUDIO_STREAM_MUSIC);
+ return getStrategyForStream(AUDIO_STREAM_MUSIC);
}
for (size_t i = 0; i < mTracks.size(); i++) {
sp<Track> track = mTracks[i];
if (sessionId == track->sessionId() && !track->isInvalid()) {
- return AudioSystem::getStrategyForStream(track->streamType());
+ return getStrategyForStream(track->streamType());
}
}
- return AudioSystem::getStrategyForStream(AUDIO_STREAM_MUSIC);
+ return getStrategyForStream(AUDIO_STREAM_MUSIC);
}
@@ -3348,7 +3428,8 @@
// Only one effect chain can be present in direct output thread and it uses
// the sink buffer as input
if (mType != DIRECT) {
- size_t numSamples = mNormalFrameCount * (mChannelCount + mHapticChannelCount);
+ size_t numSamples = mNormalFrameCount
+ * (audio_channel_count_from_out_mask(mMixerChannelMask) + mHapticChannelCount);
status_t result = mAudioFlinger->mEffectsFactoryHal->allocateBuffer(
numSamples * sizeof(effect_buffer_t),
&halInBuffer);
@@ -3531,6 +3612,8 @@
audio_patch_handle_t lastDownstreamPatchHandle = AUDIO_PATCH_HANDLE_NONE;
+ sendCheckOutputStageEffectsEvent();
+
// loopCount is used for statistics and diagnostics.
for (int64_t loopCount = 0; !exitPending(); ++loopCount)
{
@@ -3587,11 +3670,18 @@
}
}
+ if (mCheckOutputStageEffects.exchange(false)) {
+ checkOutputStageEffects();
+ }
+
{ // scope for mLock
Mutex::Autolock _l(mLock);
processConfigEvents_l();
+ if (mCheckOutputStageEffects.load()) {
+ continue;
+ }
// See comment at declaration of logString for why this is done under mLock
if (logString != NULL) {
@@ -3754,6 +3844,8 @@
//
// mMixerBufferValid is only set true by MixerThread::prepareTracks_l().
// TODO use mSleepTimeUs == 0 as an additional condition.
+ uint32_t mixerChannelCount = mEffectBufferValid ?
+ audio_channel_count_from_out_mask(mMixerChannelMask) : mChannelCount;
if (mMixerBufferValid) {
void *buffer = mEffectBufferValid ? mEffectBuffer : mSinkBuffer;
audio_format_t format = mEffectBufferValid ? mEffectBufferFormat : mFormat;
@@ -3774,7 +3866,7 @@
}
memcpy_by_audio_format(buffer, format, mMixerBuffer, mMixerBufferFormat,
- mNormalFrameCount * (mChannelCount + mHapticChannelCount));
+ mNormalFrameCount * (mixerChannelCount + mHapticChannelCount));
// If we're going directly to the sink and there are haptic channels,
// we should adjust channels as the sample data is partially interleaved
@@ -3807,8 +3899,11 @@
&& activeHapticSessionId == effectChains[i]->sessionId()) {
// Haptic data is active in this case, copy it directly from
// in buffer to out buffer.
+ uint32_t channelCount =
+ effectChains[i]->sessionId() == AUDIO_SESSION_OUTPUT_STAGE ?
+ mixerChannelCount : mChannelCount;
const size_t audioBufferSize = mNormalFrameCount
- * audio_bytes_per_frame(mChannelCount, EFFECT_BUFFER_FORMAT);
+ * audio_bytes_per_frame(channelCount, EFFECT_BUFFER_FORMAT);
memcpy_by_audio_format(
(uint8_t*)effectChains[i]->outBuffer() + audioBufferSize,
EFFECT_BUFFER_FORMAT,
@@ -3848,8 +3943,23 @@
mBalance.process((float *)mEffectBuffer, mNormalFrameCount);
}
- memcpy_by_audio_format(mSinkBuffer, mFormat, mEffectBuffer, mEffectBufferFormat,
- mNormalFrameCount * (mChannelCount + mHapticChannelCount));
+ if (mType == SPATIALIZER) {
+ memcpy_by_audio_format(mEffectToSinkBuffer, mFormat, mEffectBuffer,
+ mEffectBufferFormat,
+ mNormalFrameCount * (mChannelCount + mHapticChannelCount));
+ accumulate_by_audio_format(mSinkBuffer, mEffectToSinkBuffer, mFormat,
+ mNormalFrameCount * mChannelCount);
+ const size_t audioBufferSize = mNormalFrameCount
+ * audio_bytes_per_frame(mChannelCount, mFormat);
+ memcpy_by_audio_format(
+ (uint8_t*)mSinkBuffer + audioBufferSize,
+ mFormat,
+ (uint8_t*)mEffectToSinkBuffer + audioBufferSize,
+ mFormat, mNormalFrameCount * mHapticChannelCount);
+ } else {
+ memcpy_by_audio_format(mSinkBuffer, mFormat, mEffectBuffer, mEffectBufferFormat,
+ mNormalFrameCount * (mChannelCount + mHapticChannelCount));
+ }
// The sample data is partially interleaved when haptic channels exist,
// we need to adjust channels here.
if (mHapticChannelCount > 0) {
@@ -4448,8 +4558,8 @@
// ----------------------------------------------------------------------------
AudioFlinger::MixerThread::MixerThread(const sp<AudioFlinger>& audioFlinger, AudioStreamOut* output,
- audio_io_handle_t id, bool systemReady, type_t type)
- : PlaybackThread(audioFlinger, output, id, type, systemReady),
+ audio_io_handle_t id, bool systemReady, type_t type, audio_config_base_t *mixerConfig)
+ : PlaybackThread(audioFlinger, output, id, type, systemReady, mixerConfig),
// mAudioMixer below
// mFastMixer below
mFastMixerFutex(0),
@@ -4504,6 +4614,7 @@
&& Intersection(outDeviceTypes(), getAudioDeviceOutAllA2dpSet()).empty();
break;
}
+ ALOG_ASSERT(initFastMixer && mType == SPATIALIZER);
ALOGW_IF(initFastMixer == false && mFrameCount < mNormalFrameCount,
"FastMixer is preferred for this sink as frameCount %zu is less than threshold %zu",
mFrameCount, mNormalFrameCount);
@@ -4566,6 +4677,7 @@
fastTrack->mFormat = mFormat; // mPipeSink format for audio to FastMixer
fastTrack->mHapticPlaybackEnabled = mHapticChannelMask != AUDIO_CHANNEL_NONE;
fastTrack->mHapticIntensity = os::HapticScale::NONE;
+ fastTrack->mHapticMaxAmplitude = NAN;
fastTrack->mGeneration++;
state->mFastTracksGen++;
state->mTrackMask = 1;
@@ -4861,6 +4973,9 @@
// before effects processing or output.
if (mMixerBufferValid) {
memset(mMixerBuffer, 0, mMixerBufferSize);
+ if (mType == SPATIALIZER) {
+ memset(mSinkBuffer, 0, mSinkBufferSize);
+ }
} else {
memset(mSinkBuffer, 0, mSinkBufferSize);
}
@@ -5103,6 +5218,7 @@
fastTrack->mFormat = track->mFormat;
fastTrack->mHapticPlaybackEnabled = track->getHapticPlaybackEnabled();
fastTrack->mHapticIntensity = track->getHapticIntensity();
+ fastTrack->mHapticMaxAmplitude = track->getHapticMaxAmplitude();
fastTrack->mGeneration++;
state->mTrackMask |= 1 << j;
didModify = true;
@@ -5352,11 +5468,21 @@
trackId,
AudioMixer::TRACK,
AudioMixer::CHANNEL_MASK, (void *)(uintptr_t)track->channelMask());
- mAudioMixer->setParameter(
- trackId,
- AudioMixer::TRACK,
- AudioMixer::MIXER_CHANNEL_MASK,
- (void *)(uintptr_t)(mChannelMask | mHapticChannelMask));
+
+ if (mType == SPATIALIZER && !track->canBeSpatialized()) {
+ mAudioMixer->setParameter(
+ trackId,
+ AudioMixer::TRACK,
+ AudioMixer::MIXER_CHANNEL_MASK,
+ (void *)(uintptr_t)(mChannelMask | mHapticChannelMask));
+ } else {
+ mAudioMixer->setParameter(
+ trackId,
+ AudioMixer::TRACK,
+ AudioMixer::MIXER_CHANNEL_MASK,
+ (void *)(uintptr_t)(mMixerChannelMask | mHapticChannelMask));
+ }
+
// limit track sample rate to 2 x output sample rate, which changes at re-configuration
uint32_t maxSampleRate = mSampleRate * AUDIO_RESAMPLER_DOWN_RATIO_MAX;
uint32_t reqSampleRate = proxy->getSampleRate();
@@ -5393,16 +5519,27 @@
if (mMixerBufferEnabled
&& (track->mainBuffer() == mSinkBuffer
|| track->mainBuffer() == mMixerBuffer)) {
- mAudioMixer->setParameter(
- trackId,
- AudioMixer::TRACK,
- AudioMixer::MIXER_FORMAT, (void *)mMixerBufferFormat);
- mAudioMixer->setParameter(
- trackId,
- AudioMixer::TRACK,
- AudioMixer::MAIN_BUFFER, (void *)mMixerBuffer);
- // TODO: override track->mainBuffer()?
- mMixerBufferValid = true;
+ if (mType == SPATIALIZER && !track->canBeSpatialized()) {
+ mAudioMixer->setParameter(
+ trackId,
+ AudioMixer::TRACK,
+ AudioMixer::MIXER_FORMAT, (void *)mFormat);
+ mAudioMixer->setParameter(
+ trackId,
+ AudioMixer::TRACK,
+ AudioMixer::MAIN_BUFFER, (void *)mSinkBuffer);
+ } else {
+ mAudioMixer->setParameter(
+ trackId,
+ AudioMixer::TRACK,
+ AudioMixer::MIXER_FORMAT, (void *)mMixerBufferFormat);
+ mAudioMixer->setParameter(
+ trackId,
+ AudioMixer::TRACK,
+ AudioMixer::MAIN_BUFFER, (void *)mMixerBuffer);
+ // TODO: override track->mainBuffer()?
+ mMixerBufferValid = true;
+ }
} else {
mAudioMixer->setParameter(
trackId,
@@ -5425,6 +5562,10 @@
trackId,
AudioMixer::TRACK,
AudioMixer::HAPTIC_INTENSITY, (void *)(uintptr_t)track->getHapticIntensity());
+ mAudioMixer->setParameter(
+ trackId,
+ AudioMixer::TRACK,
+ AudioMixer::HAPTIC_MAX_AMPLITUDE, (void *)(&(track->mHapticMaxAmplitude)));
// reset retry count
track->mRetryCount = kMaxTrackRetries;
@@ -5575,7 +5716,8 @@
// remove all the tracks that need to be...
removeTracks_l(*tracksToRemove);
- if (getEffectChain_l(AUDIO_SESSION_OUTPUT_MIX) != 0) {
+ if (getEffectChain_l(AUDIO_SESSION_OUTPUT_MIX) != 0 ||
+ getEffectChain_l(AUDIO_SESSION_OUTPUT_STAGE) != 0) {
mEffectBufferValid = true;
}
@@ -5587,8 +5729,10 @@
// sink or mix buffer must be cleared if all tracks are connected to an
// effect chain as in this case the mixer will not write to the sink or mix buffer
// and track effects will accumulate into it
- if ((mBytesRemaining == 0) && ((mixedTracks != 0 && mixedTracks == tracksWithEffect) ||
- (mixedTracks == 0 && fastTracks > 0))) {
+ // always clear sink buffer for spatializer output as the output of the spatializer
+ // effect will be accumulated into it
+ if ((mBytesRemaining == 0) && (((mixedTracks != 0 && mixedTracks == tracksWithEffect) ||
+ (mixedTracks == 0 && fastTracks > 0)) || (mType == SPATIALIZER))) {
// FIXME as a performance optimization, should remember previous zero status
if (mMixerBufferValid) {
memset(mMixerBuffer, 0, mMixerBufferSize);
@@ -6971,6 +7115,69 @@
MixerThread::cacheParameters_l();
}
+// ----------------------------------------------------------------------------
+
+AudioFlinger::SpatializerThread::SpatializerThread(const sp<AudioFlinger>& audioFlinger,
+ AudioStreamOut* output,
+ audio_io_handle_t id,
+ bool systemReady,
+ audio_config_base_t *mixerConfig)
+ : MixerThread(audioFlinger, output, id, systemReady, SPATIALIZER, mixerConfig)
+{
+}
+
+void AudioFlinger::SpatializerThread::checkOutputStageEffects()
+{
+ bool hasVirtualizer = false;
+ bool hasDownMixer = false;
+ sp<EffectHandle> finalDownMixer;
+ {
+ Mutex::Autolock _l(mLock);
+ sp<EffectChain> chain = getEffectChain_l(AUDIO_SESSION_OUTPUT_STAGE);
+ if (chain != 0) {
+ hasVirtualizer = chain->getEffectFromType_l(FX_IID_SPATIALIZER) != nullptr;
+ hasDownMixer = chain->getEffectFromType_l(EFFECT_UIID_DOWNMIX) != nullptr;
+ }
+
+ finalDownMixer = mFinalDownMixer;
+ mFinalDownMixer.clear();
+ }
+
+ if (hasVirtualizer) {
+ if (finalDownMixer != nullptr) {
+ int32_t ret;
+ finalDownMixer->disable(&ret);
+ }
+ finalDownMixer.clear();
+ } else if (!hasDownMixer) {
+ std::vector<effect_descriptor_t> descriptors;
+ status_t status = mAudioFlinger->mEffectsFactoryHal->getDescriptors(
+ EFFECT_UIID_DOWNMIX, &descriptors);
+ if (status != NO_ERROR) {
+ return;
+ }
+ ALOG_ASSERT(!descriptors.empty(),
+ "%s getDescriptors() returned no error but empty list", __func__);
+
+ finalDownMixer = createEffect_l(nullptr /*client*/, nullptr /*effectClient*/,
+ 0 /*priority*/, AUDIO_SESSION_OUTPUT_STAGE, &descriptors[0], nullptr /*enabled*/,
+ &status, false /*pinned*/, false /*probe*/, false /*notifyFramesProcessed*/);
+
+ if (finalDownMixer == nullptr || (status != NO_ERROR && status != ALREADY_EXISTS)) {
+ ALOGW("%s error creating downmixer %d", __func__, status);
+ finalDownMixer.clear();
+ } else {
+ int32_t ret;
+ finalDownMixer->enable(&ret);
+ }
+ }
+
+ {
+ Mutex::Autolock _l(mLock);
+ mFinalDownMixer = finalDownMixer;
+ }
+}
+
// ----------------------------------------------------------------------------
// Record
@@ -9278,7 +9485,7 @@
mActiveTracks.add(track);
sp<EffectChain> chain = getEffectChain_l(mSessionId);
if (chain != 0) {
- chain->setStrategy(AudioSystem::getStrategyForStream(streamType()));
+ chain->setStrategy(getStrategyForStream(streamType()));
chain->incTrackCnt();
chain->incActiveTrackCnt();
}
diff --git a/services/audioflinger/Threads.h b/services/audioflinger/Threads.h
index 16082a9..0e86391 100644
--- a/services/audioflinger/Threads.h
+++ b/services/audioflinger/Threads.h
@@ -32,6 +32,7 @@
OFFLOAD, // Thread class is OffloadThread
MMAP_PLAYBACK, // Thread class for MMAP playback stream
MMAP_CAPTURE, // Thread class for MMAP capture stream
+ SPATIALIZER, //
// If you add any values here, also update ThreadBase::threadTypeToString()
};
@@ -53,7 +54,8 @@
CFG_EVENT_CREATE_AUDIO_PATCH,
CFG_EVENT_RELEASE_AUDIO_PATCH,
CFG_EVENT_UPDATE_OUT_DEVICE,
- CFG_EVENT_RESIZE_BUFFER
+ CFG_EVENT_RESIZE_BUFFER,
+ CFG_EVENT_CHECK_OUTPUT_STAGE_EFFECTS
};
class ConfigEventData: public RefBase {
@@ -87,7 +89,13 @@
public:
virtual ~ConfigEvent() {}
- void dump(char *buffer, size_t size) { mData->dump(buffer, size); }
+ void dump(char *buffer, size_t size) {
+ snprintf(buffer, size, "Event type: %d\n", mType);
+ if (mData != nullptr) {
+ snprintf(buffer, size, "Data:\n");
+ mData->dump(buffer, size);
+ }
+ }
const int mType; // event type e.g. CFG_EVENT_IO
Mutex mLock; // mutex associated with mCond
@@ -110,7 +118,7 @@
mEvent(event), mPid(pid), mPortId(portId) {}
virtual void dump(char *buffer, size_t size) {
- snprintf(buffer, size, "IO event: event %d\n", mEvent);
+ snprintf(buffer, size, "- IO event: event %d\n", mEvent);
}
const audio_io_config_event mEvent;
@@ -133,7 +141,7 @@
mPid(pid), mTid(tid), mPrio(prio), mForApp(forApp) {}
virtual void dump(char *buffer, size_t size) {
- snprintf(buffer, size, "Prio event: pid %d, tid %d, prio %d, for app? %d\n",
+ snprintf(buffer, size, "- Prio event: pid %d, tid %d, prio %d, for app? %d\n",
mPid, mTid, mPrio, mForApp);
}
@@ -158,7 +166,7 @@
mKeyValuePairs(keyValuePairs) {}
virtual void dump(char *buffer, size_t size) {
- snprintf(buffer, size, "KeyValue: %s\n", mKeyValuePairs.string());
+ snprintf(buffer, size, "- KeyValue: %s\n", mKeyValuePairs.string());
}
const String8 mKeyValuePairs;
@@ -181,7 +189,7 @@
mPatch(patch), mHandle(handle) {}
virtual void dump(char *buffer, size_t size) {
- snprintf(buffer, size, "Patch handle: %u\n", mHandle);
+ snprintf(buffer, size, "- Patch handle: %u\n", mHandle);
}
const struct audio_patch mPatch;
@@ -205,7 +213,7 @@
mHandle(handle) {}
virtual void dump(char *buffer, size_t size) {
- snprintf(buffer, size, "Patch handle: %u\n", mHandle);
+ snprintf(buffer, size, "- Patch handle: %u\n", mHandle);
}
audio_patch_handle_t mHandle;
@@ -227,7 +235,7 @@
mOutDevices(outDevices) {}
virtual void dump(char *buffer, size_t size) {
- snprintf(buffer, size, "Devices: %s", android::toString(mOutDevices).c_str());
+ snprintf(buffer, size, "- Devices: %s", android::toString(mOutDevices).c_str());
}
DeviceDescriptorBaseVector mOutDevices;
@@ -249,7 +257,7 @@
mMaxSharedAudioHistoryMs(maxSharedAudioHistoryMs) {}
virtual void dump(char *buffer, size_t size) {
- snprintf(buffer, size, "mMaxSharedAudioHistoryMs: %d", mMaxSharedAudioHistoryMs);
+ snprintf(buffer, size, "- mMaxSharedAudioHistoryMs: %d", mMaxSharedAudioHistoryMs);
}
int32_t mMaxSharedAudioHistoryMs;
@@ -265,6 +273,16 @@
virtual ~ResizeBufferConfigEvent() {}
};
+ class CheckOutputStageEffectsEvent : public ConfigEvent {
+ public:
+ CheckOutputStageEffectsEvent() :
+ ConfigEvent(CFG_EVENT_CHECK_OUTPUT_STAGE_EFFECTS) {
+ }
+
+ virtual ~CheckOutputStageEffectsEvent() {}
+ };
+
+
class PMDeathRecipient : public IBinder::DeathRecipient {
public:
explicit PMDeathRecipient(const wp<ThreadBase>& thread) : mThread(thread) {}
@@ -290,8 +308,11 @@
// dynamic externally-visible
uint32_t sampleRate() const { return mSampleRate; }
audio_channel_mask_t channelMask() const { return mChannelMask; }
+ virtual audio_channel_mask_t mixerChannelMask() const { return mChannelMask; }
+
audio_format_t format() const { return mHALFormat; }
uint32_t channelCount() const { return mChannelCount; }
+
// Called by AudioFlinger::frameCount(audio_io_handle_t output) and effects,
// and returns the [normal mix] buffer's frame count.
virtual size_t frameCount() const = 0;
@@ -330,7 +351,11 @@
status_t sendUpdateOutDeviceConfigEvent(
const DeviceDescriptorBaseVector& outDevices);
void sendResizeBufferConfigEvent_l(int32_t maxSharedAudioHistoryMs);
+ void sendCheckOutputStageEffectsEvent();
+ void sendCheckOutputStageEffectsEvent_l();
+
void processConfigEvents_l();
+ virtual void setCheckOutputStageEffects() {}
virtual void cacheParameters_l() = 0;
virtual status_t createAudioPatch_l(const struct audio_patch *patch,
audio_patch_handle_t *handle) = 0;
@@ -383,7 +408,8 @@
int *enabled,
status_t *status /*non-NULL*/,
bool pinned,
- bool probe);
+ bool probe,
+ bool notifyFramesProcessed);
// return values for hasAudioSession (bit field)
enum effect_state {
@@ -574,6 +600,8 @@
return INVALID_OPERATION;
}
+ product_strategy_t getStrategyForStream(audio_stream_type_t stream) const;
+
virtual void dumpInternals_l(int fd __unused, const Vector<String16>& args __unused)
{ }
virtual void dumpTracks_l(int fd __unused, const Vector<String16>& args __unused) { }
@@ -824,7 +852,8 @@
static const nsecs_t kMaxNextBufferDelayNs = 100000000;
PlaybackThread(const sp<AudioFlinger>& audioFlinger, AudioStreamOut* output,
- audio_io_handle_t id, type_t type, bool systemReady);
+ audio_io_handle_t id, type_t type, bool systemReady,
+ audio_config_base_t *mixerConfig = nullptr);
virtual ~PlaybackThread();
// Thread virtuals
@@ -881,6 +910,8 @@
mActiveTracks.updatePowerState(this, true /* force */);
}
+ virtual void checkOutputStageEffects() {}
+
void dumpInternals_l(int fd, const Vector<String16>& args) override;
void dumpTracks_l(int fd, const Vector<String16>& args) override;
@@ -973,6 +1004,10 @@
virtual size_t frameCount() const { return mNormalFrameCount; }
+ audio_channel_mask_t mixerChannelMask() const override {
+ return mMixerChannelMask;
+ }
+
status_t getTimestamp_l(AudioTimestamp& timestamp);
void addPatchTrack(const sp<PatchTrack>& track);
@@ -1015,6 +1050,9 @@
PlaybackThread::Track* getTrackById_l(audio_port_handle_t trackId);
+ bool hasMixer() const {
+ return mType == MIXER || mType == DUPLICATING || mType == SPATIALIZER;
+ }
protected:
// updated by readOutputParameters_l()
size_t mNormalFrameCount; // normal mixer and effects
@@ -1084,6 +1122,11 @@
// for any processing (including output processing).
bool mEffectBufferValid;
+ // Frame size aligned buffer used to convert mEffectBuffer samples to mSinkBuffer format prior
+ // to accumulate into mSinkBuffer on SPATIALIZER threads
+ void* mEffectToSinkBuffer = nullptr;
+
+
// suspend count, > 0 means suspended. While suspended, the thread continues to pull from
// tracks and mix, but doesn't write to HAL. A2DP and SCO HAL implementations can't handle
// concurrent use of both of them, so Audio Policy Service suspends one of the threads to
@@ -1101,6 +1144,9 @@
// haptic playback.
audio_channel_mask_t mHapticChannelMask = AUDIO_CHANNEL_NONE;
uint32_t mHapticChannelCount = 0;
+
+ audio_channel_mask_t mMixerChannelMask = AUDIO_CHANNEL_NONE;
+
private:
// mMasterMute is in both PlaybackThread and in AudioFlinger. When a
// PlaybackThread needs to find out if master-muted, it checks it's local
@@ -1134,6 +1180,9 @@
// Cache various calculated values, at threadLoop() entry and after a parameter change
virtual void cacheParameters_l();
+ void setCheckOutputStageEffects() override {
+ mCheckOutputStageEffects.store(true);
+ }
virtual uint32_t correctLatency_l(uint32_t latency) const;
@@ -1314,6 +1363,8 @@
// audio patch used by the downstream software patch.
// Only used if ThreadBase::mIsMsdDevice is true.
struct audio_patch mDownStreamPatch;
+
+ std::atomic_bool mCheckOutputStageEffects{};
};
class MixerThread : public PlaybackThread {
@@ -1322,7 +1373,8 @@
AudioStreamOut* output,
audio_io_handle_t id,
bool systemReady,
- type_t type = MIXER);
+ type_t type = MIXER,
+ audio_config_base_t *mixerConfig = nullptr);
virtual ~MixerThread();
// Thread virtuals
@@ -1611,6 +1663,24 @@
}
};
+class SpatializerThread : public MixerThread {
+public:
+ SpatializerThread(const sp<AudioFlinger>& audioFlinger,
+ AudioStreamOut* output,
+ audio_io_handle_t id,
+ bool systemReady,
+ audio_config_base_t *mixerConfig);
+ ~SpatializerThread() override {}
+
+ bool hasFastMixer() const override { return false; }
+
+protected:
+ void checkOutputStageEffects() override;
+
+private:
+ sp<EffectHandle> mFinalDownMixer;
+};
+
// record thread
class RecordThread : public ThreadBase
{
diff --git a/services/audiopolicy/AudioPolicyInterface.h b/services/audiopolicy/AudioPolicyInterface.h
index 2e49e71..4078278 100644
--- a/services/audiopolicy/AudioPolicyInterface.h
+++ b/services/audiopolicy/AudioPolicyInterface.h
@@ -333,6 +333,50 @@
virtual status_t getDevicesForRoleAndCapturePreset(audio_source_t audioSource,
device_role_t role,
AudioDeviceTypeAddrVector &devices) = 0;
+
+ /**
+ * Queries if some kind of spatialization will be performed if the audio playback context
+ * described by the provided arguments is present.
+ * The context is made of:
+ * - The audio attributes describing the playback use case.
+ * - The audio configuration describing the audio format, channels, sampling rate ...
+ * - The devices describing the sink audio device selected for playback.
+ * All arguments are optional and only the specified arguments are used to match against
+ * supported criteria. For instance, supplying no argument will tell if spatialization is
+ * supported or not in general.
+ * @param attr audio attributes describing the playback use case
+ * @param config audio configuration describing the audio format, channels, sampling rate...
+ * @param devices the sink audio device selected for playback
+ * @return true if spatialization is enabled for this context,
+ * false otherwise
+ */
+ virtual bool canBeSpatialized(const audio_attributes_t *attr,
+ const audio_config_t *config,
+ const AudioDeviceTypeAddrVector &devices) const = 0;
+
+ /**
+ * Opens a specialized spatializer output if supported by the platform.
+ * If several spatializer output profiles exist, the one supporting the sink device
+ * corresponding to the provided audio attributes will be selected.
+ * Only one spatializer output stream can be opened at a time and an error is returned
+ * if one already exists.
+ * @param config audio format, channel mask and sampling rate to be used as the mixer
+ * configuration for the spatializer mixer created.
+ * @param attr audio attributes describing the playback use case that will drive the
+ * sink device selection
+ * @param output the IO handle of the output opened
+ * @return NO_ERROR if an output was opened, INVALID_OPERATION or BAD_VALUE otherwise
+ */
+ virtual status_t getSpatializerOutput(const audio_config_base_t *config,
+ const audio_attributes_t *attr,
+ audio_io_handle_t *output) = 0;
+
+ /**
+ * Closes a previously opened specialized spatializer output.
+ * @param output the IO handle of the output to close.
+ * @return NO_ERROR if an output was closed, INVALID_OPERATION or BAD_VALUE otherwise
+ */
+ virtual status_t releaseSpatializerOutput(audio_io_handle_t output) = 0;
};
@@ -359,7 +403,8 @@
// The audio policy manager can check if the proposed parameters are suitable or not and act accordingly.
virtual status_t openOutput(audio_module_handle_t module,
audio_io_handle_t *output,
- audio_config_t *config,
+ audio_config_t *halConfig,
+ audio_config_base_t *mixerConfig,
const sp<DeviceDescriptorBase>& device,
uint32_t *latencyMs,
audio_output_flags_t flags) = 0;
diff --git a/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h b/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h
index 1f9b535..7c7f02d 100644
--- a/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h
+++ b/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h
@@ -362,7 +362,8 @@
const struct audio_port_config *srcConfig = NULL) const;
virtual void toAudioPort(struct audio_port_v7 *port) const;
- status_t open(const audio_config_t *config,
+ status_t open(const audio_config_t *halConfig,
+ const audio_config_base_t *mixerConfig,
const DeviceVector &devices,
audio_stream_type_t stream,
audio_output_flags_t flags,
@@ -423,6 +424,7 @@
uint32_t mDirectOpenCount; // number of clients using this output (direct outputs only)
audio_session_t mDirectClientSession; // session id of the direct output client
bool mPendingReopenToQueryProfiles = false;
+ audio_channel_mask_t mMixerChannelMask = AUDIO_CHANNEL_NONE;
};
// Audio output driven by an input device directly.
diff --git a/services/audiopolicy/common/managerdefinitions/include/AudioPolicyConfig.h b/services/audiopolicy/common/managerdefinitions/include/AudioPolicyConfig.h
index cf1f64c..a8fd856 100644
--- a/services/audiopolicy/common/managerdefinitions/include/AudioPolicyConfig.h
+++ b/services/audiopolicy/common/managerdefinitions/include/AudioPolicyConfig.h
@@ -202,6 +202,20 @@
{AUDIO_FORMAT_AC4, {}}};
}
+ //TODO: b/193496180 use spatializer flag at audio HAL when available
+ // until then, use DEEP_BUFFER+FAST flag combo to indicate the spatializer output profile
+ void convertSpatializerFlag()
+ {
+ for (const auto& hwModule : mHwModules) {
+ for (const auto& curProfile : hwModule->getOutputProfiles()) {
+ if (curProfile->getFlags()
+ == (AUDIO_OUTPUT_FLAG_FAST | AUDIO_OUTPUT_FLAG_DEEP_BUFFER)) {
+ curProfile->setFlags(AUDIO_OUTPUT_FLAG_SPATIALIZER);
+ }
+ }
+ }
+ }
+
private:
static const constexpr char* const kDefaultEngineLibraryNameSuffix = "default";
diff --git a/services/audiopolicy/common/managerdefinitions/include/DeviceDescriptor.h b/services/audiopolicy/common/managerdefinitions/include/DeviceDescriptor.h
index 20b4044..58d05c6 100644
--- a/services/audiopolicy/common/managerdefinitions/include/DeviceDescriptor.h
+++ b/services/audiopolicy/common/managerdefinitions/include/DeviceDescriptor.h
@@ -168,6 +168,10 @@
DeviceVector getDevicesFromDeviceTypeAddrVec(
const AudioDeviceTypeAddrVector& deviceTypeAddrVector) const;
+ // Return the device vector that contains device descriptor whose AudioDeviceTypeAddr appears
+ // in the given AudioDeviceTypeAddrVector
+ AudioDeviceTypeAddrVector toTypeAddrVector() const;
+
// If there are devices with the given type and the devices to add is not empty,
// remove all the devices with the given type and add all the devices to add.
void replaceDevicesByType(audio_devices_t typeToRemove, const DeviceVector &devicesToAdd);
diff --git a/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp b/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp
index 6b08f7c..f3d2326 100644
--- a/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp
+++ b/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp
@@ -491,7 +491,8 @@
return true;
}
-status_t SwAudioOutputDescriptor::open(const audio_config_t *config,
+status_t SwAudioOutputDescriptor::open(const audio_config_t *halConfig,
+ const audio_config_base_t *mixerConfig,
const DeviceVector &devices,
audio_stream_type_t stream,
audio_output_flags_t flags,
@@ -504,45 +505,62 @@
"with the requested devices, all device types: %s",
__func__, dumpDeviceTypes(devices.types()).c_str());
- audio_config_t lConfig;
- if (config == nullptr) {
- lConfig = AUDIO_CONFIG_INITIALIZER;
- lConfig.sample_rate = mSamplingRate;
- lConfig.channel_mask = mChannelMask;
- lConfig.format = mFormat;
+ audio_config_t lHalConfig;
+ if (halConfig == nullptr) {
+ lHalConfig = AUDIO_CONFIG_INITIALIZER;
+ lHalConfig.sample_rate = mSamplingRate;
+ lHalConfig.channel_mask = mChannelMask;
+ lHalConfig.format = mFormat;
} else {
- lConfig = *config;
+ lHalConfig = *halConfig;
}
// if the selected profile is offloaded and no offload info was specified,
// create a default one
if ((mProfile->getFlags() & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) &&
- lConfig.offload_info.format == AUDIO_FORMAT_DEFAULT) {
+ lHalConfig.offload_info.format == AUDIO_FORMAT_DEFAULT) {
flags = (audio_output_flags_t)(flags | AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD);
- lConfig.offload_info = AUDIO_INFO_INITIALIZER;
- lConfig.offload_info.sample_rate = lConfig.sample_rate;
- lConfig.offload_info.channel_mask = lConfig.channel_mask;
- lConfig.offload_info.format = lConfig.format;
- lConfig.offload_info.stream_type = stream;
- lConfig.offload_info.duration_us = -1;
- lConfig.offload_info.has_video = true; // conservative
- lConfig.offload_info.is_streaming = true; // likely
- lConfig.offload_info.encapsulation_mode = lConfig.offload_info.encapsulation_mode;
- lConfig.offload_info.content_id = lConfig.offload_info.content_id;
- lConfig.offload_info.sync_id = lConfig.offload_info.sync_id;
+ lHalConfig.offload_info = AUDIO_INFO_INITIALIZER;
+ lHalConfig.offload_info.sample_rate = lHalConfig.sample_rate;
+ lHalConfig.offload_info.channel_mask = lHalConfig.channel_mask;
+ lHalConfig.offload_info.format = lHalConfig.format;
+ lHalConfig.offload_info.stream_type = stream;
+ lHalConfig.offload_info.duration_us = -1;
+ lHalConfig.offload_info.has_video = true; // conservative
+ lHalConfig.offload_info.is_streaming = true; // likely
+ lHalConfig.offload_info.encapsulation_mode = lHalConfig.offload_info.encapsulation_mode;
+ lHalConfig.offload_info.content_id = lHalConfig.offload_info.content_id;
+ lHalConfig.offload_info.sync_id = lHalConfig.offload_info.sync_id;
+ }
+
+ audio_config_base_t lMixerConfig;
+ if (mixerConfig == nullptr) {
+ lMixerConfig = AUDIO_CONFIG_BASE_INITIALIZER;
+ lMixerConfig.sample_rate = lHalConfig.sample_rate;
+ lMixerConfig.channel_mask = lHalConfig.channel_mask;
+ lMixerConfig.format = lHalConfig.format;
+ } else {
+ lMixerConfig = *mixerConfig;
}
mFlags = (audio_output_flags_t)(mFlags | flags);
+ //TODO: b/193496180 use spatializer flag at audio HAL when available
+ audio_output_flags_t halFlags = mFlags;
+ if ((mFlags & AUDIO_OUTPUT_FLAG_SPATIALIZER) != 0) {
+ halFlags = (audio_output_flags_t)(AUDIO_OUTPUT_FLAG_FAST | AUDIO_OUTPUT_FLAG_DEEP_BUFFER);
+ }
+
ALOGV("opening output for device %s profile %p name %s",
mDevices.toString().c_str(), mProfile.get(), mProfile->getName().c_str());
status_t status = mClientInterface->openOutput(mProfile->getModuleHandle(),
output,
- &lConfig,
+ &lHalConfig,
+ &lMixerConfig,
device,
&mLatency,
- mFlags);
+ halFlags);
if (status == NO_ERROR) {
LOG_ALWAYS_FATAL_IF(*output == AUDIO_IO_HANDLE_NONE,
@@ -550,9 +568,10 @@
"selected device %s for opening",
__FUNCTION__, *output, devices.toString().c_str(),
device->toString().c_str());
- mSamplingRate = lConfig.sample_rate;
- mChannelMask = lConfig.channel_mask;
- mFormat = lConfig.format;
+ mSamplingRate = lHalConfig.sample_rate;
+ mChannelMask = lHalConfig.channel_mask;
+ mFormat = lHalConfig.format;
+ mMixerChannelMask = lMixerConfig.channel_mask;
mId = PolicyAudioPort::getNextUniqueId();
mIoHandle = *output;
mProfile->curOpenCount++;
diff --git a/services/audiopolicy/common/managerdefinitions/src/DeviceDescriptor.cpp b/services/audiopolicy/common/managerdefinitions/src/DeviceDescriptor.cpp
index a92d31e..1722032 100644
--- a/services/audiopolicy/common/managerdefinitions/src/DeviceDescriptor.cpp
+++ b/services/audiopolicy/common/managerdefinitions/src/DeviceDescriptor.cpp
@@ -451,6 +451,14 @@
return devices;
}
+AudioDeviceTypeAddrVector DeviceVector::toTypeAddrVector() const {
+ AudioDeviceTypeAddrVector result;
+ for (const auto& device : *this) {
+ result.push_back(AudioDeviceTypeAddr(device->type(), device->address()));
+ }
+ return result;
+}
+
void DeviceVector::replaceDevicesByType(
audio_devices_t typeToRemove, const DeviceVector &devicesToAdd) {
DeviceVector devicesToRemove = getDevicesFromType(typeToRemove);
diff --git a/services/audiopolicy/fuzzer/audiopolicy_fuzzer.cpp b/services/audiopolicy/fuzzer/audiopolicy_fuzzer.cpp
index 7000cd9..8584702 100644
--- a/services/audiopolicy/fuzzer/audiopolicy_fuzzer.cpp
+++ b/services/audiopolicy/fuzzer/audiopolicy_fuzzer.cpp
@@ -163,7 +163,9 @@
AUDIO_FLAG_BYPASS_MUTE, AUDIO_FLAG_LOW_LATENCY,
AUDIO_FLAG_DEEP_BUFFER, AUDIO_FLAG_NO_MEDIA_PROJECTION,
AUDIO_FLAG_MUTE_HAPTIC, AUDIO_FLAG_NO_SYSTEM_CAPTURE,
- AUDIO_FLAG_CAPTURE_PRIVATE};
+ AUDIO_FLAG_CAPTURE_PRIVATE, AUDIO_FLAG_CONTENT_SPATIALIZED,
+ AUDIO_FLAG_NEVER_SPATIALIZE,
+ };
std::vector<audio_policy_dev_state_t> kAudioPolicyDeviceStates = {
AUDIO_POLICY_DEVICE_STATE_UNAVAILABLE,
diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
index cb9d700..e334532 100644
--- a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
+++ b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp
@@ -246,8 +246,8 @@
sp<SwAudioOutputDescriptor> desc = mOutputs.valueFor(output);
// close unused outputs after device disconnection or direct outputs that have
// been opened by checkOutputsForDevice() to query dynamic parameters
- if ((state == AUDIO_POLICY_DEVICE_STATE_UNAVAILABLE) ||
- (((desc->mFlags & AUDIO_OUTPUT_FLAG_DIRECT) != 0) &&
+ if ((state == AUDIO_POLICY_DEVICE_STATE_UNAVAILABLE)
+ || (((desc->mFlags & AUDIO_OUTPUT_FLAG_DIRECT) != 0) &&
(desc->mDirectOpenCount == 0))) {
clearAudioSourcesForOutput(output);
closeOutput(output);
@@ -925,6 +925,32 @@
return profile;
}
+sp<IOProfile> AudioPolicyManager::getSpatializerOutputProfile(
+ const audio_config_t *config __unused, const AudioDeviceTypeAddrVector &devices) const
+{
+ for (const auto& hwModule : mHwModules) {
+ for (const auto& curProfile : hwModule->getOutputProfiles()) {
+ if (curProfile->getFlags() != AUDIO_OUTPUT_FLAG_SPATIALIZER) {
+ continue;
+ }
+ // reject profiles not corresponding to a device currently available
+ DeviceVector supportedDevices = curProfile->getSupportedDevices();
+ if (!mAvailableOutputDevices.containsAtLeastOne(supportedDevices)) {
+ continue;
+ }
+ if (!devices.empty()) {
+ if (supportedDevices.getDevicesFromDeviceTypeAddrVec(devices).size()
+ != devices.size()) {
+ continue;
+ }
+ }
+ ALOGV("%s found profile %s", __func__, curProfile->getName().c_str());
+ return curProfile;
+ }
+ }
+ return nullptr;
+}
+
audio_io_handle_t AudioPolicyManager::getOutput(audio_stream_type_t stream)
{
DeviceVector devices = mEngine->getOutputDevicesForStream(stream, false /*fromCache*/);
@@ -1094,7 +1120,7 @@
*output = AUDIO_IO_HANDLE_NONE;
if (!msdDevices.isEmpty()) {
- *output = getOutputForDevices(msdDevices, session, *stream, config, flags);
+ *output = getOutputForDevices(msdDevices, session, resultAttr, config, flags);
if (*output != AUDIO_IO_HANDLE_NONE && setMsdOutputPatches(&outputDevices) == NO_ERROR) {
ALOGV("%s() Using MSD devices %s instead of devices %s",
__func__, msdDevices.toString().c_str(), outputDevices.toString().c_str());
@@ -1103,7 +1129,7 @@
}
}
if (*output == AUDIO_IO_HANDLE_NONE) {
- *output = getOutputForDevices(outputDevices, session, *stream, config,
+ *output = getOutputForDevices(outputDevices, session, resultAttr, config,
flags, resultAttr->flags & AUDIO_FLAG_MUTE_HAPTIC);
}
if (*output == AUDIO_IO_HANDLE_NONE) {
@@ -1265,7 +1291,8 @@
// all MSD patches to prioritize this request over any active output on MSD.
releaseMsdOutputPatches(devices);
- status_t status = outputDesc->open(config, devices, stream, flags, output);
+ status_t status =
+ outputDesc->open(config, nullptr /* mixerConfig */, devices, stream, flags, output);
// only accept an output with the requested parameters
if (status != NO_ERROR ||
@@ -1300,7 +1327,7 @@
audio_io_handle_t AudioPolicyManager::getOutputForDevices(
const DeviceVector &devices,
audio_session_t session,
- audio_stream_type_t stream,
+ const audio_attributes_t *attr,
const audio_config_t *config,
audio_output_flags_t *flags,
bool forceMutingHaptic)
@@ -1322,6 +1349,9 @@
if ((*flags & AUDIO_OUTPUT_FLAG_HW_AV_SYNC) != 0) {
*flags = (audio_output_flags_t)(*flags | AUDIO_OUTPUT_FLAG_DIRECT);
}
+
+ audio_stream_type_t stream = mEngine->getStreamTypeForAttributes(*attr);
+
// only allow deep buffering for music stream type
if (stream != AUDIO_STREAM_MUSIC) {
*flags = (audio_output_flags_t)(*flags &~AUDIO_OUTPUT_FLAG_DEEP_BUFFER);
@@ -1341,6 +1371,11 @@
ALOGV("Set VoIP and Direct output flags for PCM format");
}
+ if (mSpatializerOutput != nullptr
+ && canBeSpatialized(attr, config, devices.toTypeAddrVector())) {
+ return mSpatializerOutput->mIoHandle;
+ }
+
audio_config_t directConfig = *config;
directConfig.channel_mask = channelMask;
status_t status = openDirectOutput(stream, session, &directConfig, *flags, devices, &output);
@@ -4802,6 +4837,205 @@
return source;
}
+/* static */
+bool AudioPolicyManager::isChannelMaskSpatialized(audio_channel_mask_t channels) {
+ switch (channels) {
+ case AUDIO_CHANNEL_OUT_5POINT1:
+ case AUDIO_CHANNEL_OUT_5POINT1POINT2:
+ case AUDIO_CHANNEL_OUT_5POINT1POINT4:
+ case AUDIO_CHANNEL_OUT_7POINT1:
+ case AUDIO_CHANNEL_OUT_7POINT1POINT2:
+ case AUDIO_CHANNEL_OUT_7POINT1POINT4:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool AudioPolicyManager::canBeSpatialized(const audio_attributes_t *attr,
+ const audio_config_t *config,
+ const AudioDeviceTypeAddrVector &devices) const
+{
+ // The caller can have the audio attributes criteria ignored by either passing a null ptr or
+ // the AUDIO_ATTRIBUTES_INITIALIZER value.
+ // If attributes are specified, current policy is to only allow spatialization for media
+ // and game usages.
+ if (attr != nullptr && *attr != AUDIO_ATTRIBUTES_INITIALIZER) {
+ if (attr->usage != AUDIO_USAGE_MEDIA && attr->usage != AUDIO_USAGE_GAME) {
+ return false;
+ }
+ if ((attr->flags & (AUDIO_FLAG_CONTENT_SPATIALIZED | AUDIO_FLAG_NEVER_SPATIALIZE)) != 0) {
+ return false;
+ }
+ }
+
+ // The caller can have the devices criteria ignored by passing and empty vector, and
+ // getSpatializerOutputProfile() will ignore the devices when looking for a match.
+ // Otherwise an output profile supporting a spatializer effect that can be routed
+ // to the specified devices must exist.
+ sp<IOProfile> profile =
+ getSpatializerOutputProfile(config, devices);
+ if (profile == nullptr) {
+ return false;
+ }
+
+ // The caller can have the audio config criteria ignored by either passing a null ptr or
+ // the AUDIO_CONFIG_INITIALIZER value.
+ // If an audio config is specified, current policy is to only allow spatialization for
+ // some positional channel masks.
+ // If the spatializer output is already opened, only channel masks included in the
+ // spatializer output mixer channel mask are allowed.
+
+ if (config != nullptr && *config != AUDIO_CONFIG_INITIALIZER) {
+ if (!isChannelMaskSpatialized(config->channel_mask)) {
+ return false;
+ }
+ if (mSpatializerOutput != nullptr && mSpatializerOutput->mProfile == profile) {
+ if ((config->channel_mask & mSpatializerOutput->mMixerChannelMask)
+ != config->channel_mask) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+void AudioPolicyManager::checkVirtualizerClientRoutes() {
+ std::set<audio_stream_type_t> streamsToInvalidate;
+ for (size_t i = 0; i < mOutputs.size(); i++) {
+ const sp<SwAudioOutputDescriptor>& desc = mOutputs[i];
+ for (const sp<TrackClientDescriptor>& client : desc->getClientIterable()) {
+ audio_attributes_t attr = client->attributes();
+ DeviceVector devices = mEngine->getOutputDevicesForAttributes(attr, nullptr, false);
+ AudioDeviceTypeAddrVector devicesTypeAddress = devices.toTypeAddrVector();
+ audio_config_base_t clientConfig = client->config();
+ audio_config_t config = audio_config_initializer(&clientConfig);
+ if (desc != mSpatializerOutput
+ && canBeSpatialized(&attr, &config, devicesTypeAddress)) {
+ streamsToInvalidate.insert(client->stream());
+ }
+ }
+ }
+
+ for (audio_stream_type_t stream : streamsToInvalidate) {
+ mpClientInterface->invalidateStream(stream);
+ }
+}
+
+status_t AudioPolicyManager::getSpatializerOutput(const audio_config_base_t *mixerConfig,
+ const audio_attributes_t *attr,
+ audio_io_handle_t *output) {
+ *output = AUDIO_IO_HANDLE_NONE;
+
+ DeviceVector devices = mEngine->getOutputDevicesForAttributes(*attr, nullptr, false);
+ AudioDeviceTypeAddrVector devicesTypeAddress = devices.toTypeAddrVector();
+ audio_config_t *configPtr = nullptr;
+ audio_config_t config;
+ if (mixerConfig != nullptr) {
+ config = audio_config_initializer(mixerConfig);
+ configPtr = &config;
+ }
+ if (!canBeSpatialized(attr, configPtr, devicesTypeAddress)) {
+ ALOGW("%s provided attributes or mixer config cannot be spatialized", __func__);
+ return BAD_VALUE;
+ }
+
+ sp<IOProfile> profile =
+ getSpatializerOutputProfile(configPtr, devicesTypeAddress);
+ if (profile == nullptr) {
+ ALOGW("%s no suitable output profile for provided attributes or mixer config", __func__);
+ return BAD_VALUE;
+ }
+
+ if (mSpatializerOutput != nullptr && mSpatializerOutput->mProfile == profile
+ && configPtr != nullptr
+ && configPtr->channel_mask == mSpatializerOutput->mMixerChannelMask) {
+ *output = mSpatializerOutput->mIoHandle;
+ ALOGV("%s returns current spatializer output %d", __func__, *output);
+ return NO_ERROR;
+ }
+ mSpatializerOutput.clear();
+ for (size_t i = 0; i < mOutputs.size(); i++) {
+ sp<SwAudioOutputDescriptor> desc = mOutputs.valueAt(i);
+ if (!desc->isDuplicated() && desc->mProfile == profile) {
+ mSpatializerOutput = desc;
+ break;
+ }
+ }
+ if (mSpatializerOutput == nullptr) {
+ ALOGW("%s no opened spatializer output for profile %s",
+ __func__, profile->getName().c_str());
+ return BAD_VALUE;
+ }
+
+ if (configPtr != nullptr
+ && configPtr->channel_mask != mSpatializerOutput->mMixerChannelMask) {
+ audio_config_base_t savedMixerConfig = {
+ .sample_rate = mSpatializerOutput->getSamplingRate(),
+ .format = mSpatializerOutput->getFormat(),
+ .channel_mask = mSpatializerOutput->mMixerChannelMask,
+ };
+ DeviceVector savedDevices = mSpatializerOutput->devices();
+
+ closeOutput(mSpatializerOutput->mIoHandle);
+ mSpatializerOutput.clear();
+
+ const sp<SwAudioOutputDescriptor> desc =
+ new SwAudioOutputDescriptor(profile, mpClientInterface);
+ status_t status = desc->open(nullptr, mixerConfig, devices,
+ mEngine->getStreamTypeForAttributes(*attr),
+ AUDIO_OUTPUT_FLAG_SPATIALIZER, output);
+ if (status != NO_ERROR) {
+ ALOGW("%s failed opening output: status %d, output %d", __func__, status, *output);
+ if (*output != AUDIO_IO_HANDLE_NONE) {
+ desc->close();
+ }
+ // re open the spatializer output with previous channel mask
+ status_t newStatus = desc->open(nullptr, &savedMixerConfig, savedDevices,
+ mEngine->getStreamTypeForAttributes(*attr),
+ AUDIO_OUTPUT_FLAG_SPATIALIZER, output);
+ if (newStatus != NO_ERROR) {
+ if (*output != AUDIO_IO_HANDLE_NONE) {
+ desc->close();
+ }
+ ALOGE("%s failed to re-open mSpatializerOutput, status %d", __func__, newStatus);
+ } else {
+ mSpatializerOutput = desc;
+ addOutput(*output, desc);
+ }
+ mPreviousOutputs = mOutputs;
+ mpClientInterface->onAudioPortListUpdate();
+ *output = AUDIO_IO_HANDLE_NONE;
+ return status;
+ }
+ mSpatializerOutput = desc;
+ addOutput(*output, desc);
+ mPreviousOutputs = mOutputs;
+ mpClientInterface->onAudioPortListUpdate();
+ }
+
+ checkVirtualizerClientRoutes();
+
+ *output = mSpatializerOutput->mIoHandle;
+ ALOGV("%s returns new spatializer output %d", __func__, *output);
+ return NO_ERROR;
+}
+
+status_t AudioPolicyManager::releaseSpatializerOutput(audio_io_handle_t output) {
+ if (mSpatializerOutput == nullptr) {
+ return INVALID_OPERATION;
+ }
+ if (mSpatializerOutput->mIoHandle != output) {
+ return BAD_VALUE;
+ }
+
+ mSpatializerOutput.clear();
+
+ checkVirtualizerClientRoutes();
+
+ return NO_ERROR;
+}
+
// ----------------------------------------------------------------------------
// AudioPolicyManager
// ----------------------------------------------------------------------------
@@ -4851,6 +5085,8 @@
ALOGE("could not load audio policy configuration file, setting defaults");
getConfig().setDefault();
}
+ //TODO: b/193496180 use spatializer flag at audio HAL when available
+ getConfig().convertSpatializerFlag();
}
status_t AudioPolicyManager::initialize() {
@@ -4990,7 +5226,8 @@
sp<SwAudioOutputDescriptor> outputDesc = new SwAudioOutputDescriptor(outProfile,
mpClientInterface);
audio_io_handle_t output = AUDIO_IO_HANDLE_NONE;
- status_t status = outputDesc->open(nullptr, DeviceVector(supportedDevice),
+ status_t status = outputDesc->open(nullptr /* halConfig */, nullptr /* mixerConfig */,
+ DeviceVector(supportedDevice),
AUDIO_STREAM_DEFAULT,
AUDIO_OUTPUT_FLAG_NONE, &output);
if (status != NO_ERROR) {
@@ -6995,7 +7232,7 @@
}
sp<SwAudioOutputDescriptor> desc = new SwAudioOutputDescriptor(profile, mpClientInterface);
audio_io_handle_t output = AUDIO_IO_HANDLE_NONE;
- status_t status = desc->open(nullptr, devices,
+ status_t status = desc->open(nullptr /* halConfig */, nullptr /* mixerConfig */, devices,
AUDIO_STREAM_DEFAULT, AUDIO_OUTPUT_FLAG_NONE, &output);
if (status != NO_ERROR) {
return nullptr;
@@ -7025,7 +7262,7 @@
config.offload_info.channel_mask = config.channel_mask;
config.offload_info.format = config.format;
- status = desc->open(&config, devices,
+ status = desc->open(&config, nullptr /* mixerConfig */, devices,
AUDIO_STREAM_DEFAULT, AUDIO_OUTPUT_FLAG_NONE, &output);
if (status != NO_ERROR) {
return nullptr;
diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.h b/services/audiopolicy/managerdefault/AudioPolicyManager.h
index 98f96d1..967aa10 100644
--- a/services/audiopolicy/managerdefault/AudioPolicyManager.h
+++ b/services/audiopolicy/managerdefault/AudioPolicyManager.h
@@ -356,6 +356,16 @@
BAD_VALUE : NO_ERROR;
}
+ virtual bool canBeSpatialized(const audio_attributes_t *attr,
+ const audio_config_t *config,
+ const AudioDeviceTypeAddrVector &devices) const;
+
+ virtual status_t getSpatializerOutput(const audio_config_base_t *config,
+ const audio_attributes_t *attr,
+ audio_io_handle_t *output);
+
+ virtual status_t releaseSpatializerOutput(audio_io_handle_t output);
+
bool isCallScreenModeSupported() override;
void onNewAudioModulesAvailable() override;
@@ -797,6 +807,8 @@
sp<SwAudioOutputDescriptor> mPrimaryOutput; // primary output descriptor
// list of descriptors for outputs currently opened
+ sp<SwAudioOutputDescriptor> mSpatializerOutput;
+
SwAudioOutputCollection mOutputs;
// copy of mOutputs before setDeviceConnectionState() opens new outputs
// reset to mOutputs when updateDevicesAndOutputs() is called.
@@ -933,7 +945,7 @@
audio_io_handle_t getOutputForDevices(
const DeviceVector &devices,
audio_session_t session,
- audio_stream_type_t stream,
+ const audio_attributes_t *attr,
const audio_config_t *config,
audio_output_flags_t *flags,
bool forceMutingHaptic = false);
@@ -948,6 +960,14 @@
audio_output_flags_t flags,
const DeviceVector &devices,
audio_io_handle_t *output);
+
+ sp<IOProfile> getSpatializerOutputProfile(const audio_config_t *config,
+ const AudioDeviceTypeAddrVector &devices) const;
+
+ static bool isChannelMaskSpatialized(audio_channel_mask_t channels);
+
+ void checkVirtualizerClientRoutes();
+
/**
* @brief getInputForDevice selects an input handle for a given input device and
* requester context
diff --git a/services/audiopolicy/service/Android.bp b/services/audiopolicy/service/Android.bp
index 454c020..a73e270 100644
--- a/services/audiopolicy/service/Android.bp
+++ b/services/audiopolicy/service/Android.bp
@@ -16,6 +16,8 @@
"AudioPolicyInterfaceImpl.cpp",
"AudioPolicyService.cpp",
"CaptureStateNotifier.cpp",
+ "Spatializer.cpp",
+ "SpatializerPoseController.cpp",
],
include_dirs: [
@@ -24,9 +26,11 @@
shared_libs: [
"libactivitymanager_aidl",
+ "libandroid",
"libaudioclient",
"libaudioclient_aidl_conversion",
"libaudiofoundation",
+ "libaudiohal",
"libaudiopolicy",
"libaudiopolicymanagerdefault",
"libaudioutils",
@@ -34,12 +38,16 @@
"libcutils",
"libeffectsconfig",
"libhardware_legacy",
+ "libheadtracking",
+ "libheadtracking-binding",
"liblog",
"libmedia_helper",
"libmediametrics",
"libmediautils",
"libpermission",
+ "libsensor",
"libsensorprivacy",
+ "libshmemcompat",
"libutils",
"audioclient-types-aidl-cpp",
"audioflinger-aidl-cpp",
@@ -47,6 +55,7 @@
"audiopolicy-types-aidl-cpp",
"capture_state_listener-aidl-cpp",
"framework-permission-aidl-cpp",
+ "spatializer-aidl-cpp",
],
static_libs: [
@@ -55,6 +64,7 @@
],
header_libs: [
+ "libaudiohal_headers",
"libaudiopolicycommon",
"libaudiopolicyengine_interface_headers",
"libaudiopolicymanager_interface_headers",
@@ -70,6 +80,8 @@
export_shared_lib_headers: [
"libactivitymanager_aidl",
+ "libheadtracking",
+ "libheadtracking-binding",
"libsensorprivacy",
"framework-permission-aidl-cpp",
],
diff --git a/services/audiopolicy/service/AudioPolicyClientImpl.cpp b/services/audiopolicy/service/AudioPolicyClientImpl.cpp
index cd53073..79252d4 100644
--- a/services/audiopolicy/service/AudioPolicyClientImpl.cpp
+++ b/services/audiopolicy/service/AudioPolicyClientImpl.cpp
@@ -40,7 +40,8 @@
status_t AudioPolicyService::AudioPolicyClient::openOutput(audio_module_handle_t module,
audio_io_handle_t *output,
- audio_config_t *config,
+ audio_config_t *halConfig,
+ audio_config_base_t *mixerConfig,
const sp<DeviceDescriptorBase>& device,
uint32_t *latencyMs,
audio_output_flags_t flags)
@@ -55,14 +56,17 @@
media::OpenOutputResponse response;
request.module = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_module_handle_t_int32_t(module));
- request.config = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_config_t_AudioConfig(*config));
+ request.halConfig = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_config_t_AudioConfig(*halConfig));
+ request.mixerConfig =
+ VALUE_OR_RETURN_STATUS(legacy2aidl_audio_config_base_t_AudioConfigBase(*mixerConfig));
request.device = VALUE_OR_RETURN_STATUS(legacy2aidl_DeviceDescriptorBase(device));
request.flags = VALUE_OR_RETURN_STATUS(legacy2aidl_audio_output_flags_t_int32_t_mask(flags));
status_t status = af->openOutput(request, &response);
if (status == OK) {
*output = VALUE_OR_RETURN_STATUS(aidl2legacy_int32_t_audio_io_handle_t(response.output));
- *config = VALUE_OR_RETURN_STATUS(aidl2legacy_AudioConfig_audio_config_t(response.config));
+ *halConfig =
+ VALUE_OR_RETURN_STATUS(aidl2legacy_AudioConfig_audio_config_t(response.config));
*latencyMs = VALUE_OR_RETURN_STATUS(convertIntegral<uint32_t>(response.latencyMs));
}
return status;
diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
index 77223b6..8504489 100644
--- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
+++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
@@ -121,11 +121,14 @@
ALOGV("setDeviceConnectionState()");
Mutex::Autolock _l(mLock);
AutoCallerClear acc;
- return binderStatusFromStatusT(
- mAudioPolicyManager->setDeviceConnectionState(device, state,
+ status_t status = mAudioPolicyManager->setDeviceConnectionState(device, state,
deviceAidl.address.c_str(),
deviceNameAidl.c_str(),
- encodedFormat));
+ encodedFormat);
+ if (status == NO_ERROR) {
+ onCheckSpatializer_l();
+ }
+ return binderStatusFromStatusT(status);
}
Status AudioPolicyService::getDeviceConnectionState(const media::AudioDevice& deviceAidl,
@@ -165,9 +168,13 @@
ALOGV("handleDeviceConfigChange()");
Mutex::Autolock _l(mLock);
AutoCallerClear acc;
- return binderStatusFromStatusT(
- mAudioPolicyManager->handleDeviceConfigChange(device, deviceAidl.address.c_str(),
- deviceNameAidl.c_str(), encodedFormat));
+ status_t status = mAudioPolicyManager->handleDeviceConfigChange(
+ device, deviceAidl.address.c_str(), deviceNameAidl.c_str(), encodedFormat);
+
+ if (status == NO_ERROR) {
+ onCheckSpatializer_l();
+ }
+ return binderStatusFromStatusT(status);
}
Status AudioPolicyService::setPhoneState(media::AudioMode stateAidl, int32_t uidAidl)
@@ -234,6 +241,7 @@
Mutex::Autolock _l(mLock);
AutoCallerClear acc;
mAudioPolicyManager->setForceUse(usage, config);
+ onCheckSpatializer_l();
return Status::ok();
}
@@ -2062,8 +2070,11 @@
return binderStatusFromStatusT(NO_INIT);
}
Mutex::Autolock _l(mLock);
- return binderStatusFromStatusT(
- mAudioPolicyManager->setDevicesRoleForStrategy(strategy, role, devices));
+ status_t status = mAudioPolicyManager->setDevicesRoleForStrategy(strategy, role, devices);
+ if (status == NO_ERROR) {
+ onCheckSpatializer_l();
+ }
+ return binderStatusFromStatusT(status);
}
Status AudioPolicyService::removeDevicesRoleForStrategy(int32_t strategyAidl,
@@ -2076,8 +2087,11 @@
return binderStatusFromStatusT(NO_INIT);
}
Mutex::Autolock _l(mLock);
- return binderStatusFromStatusT(
- mAudioPolicyManager->removeDevicesRoleForStrategy(strategy, role));
+ status_t status = mAudioPolicyManager->removeDevicesRoleForStrategy(strategy, role);
+ if (status == NO_ERROR) {
+ onCheckSpatializer_l();
+ }
+ return binderStatusFromStatusT(status);
}
Status AudioPolicyService::getDevicesForRoleAndStrategy(
@@ -2205,4 +2219,41 @@
return Status::ok();
}
+Status AudioPolicyService::getSpatializer(
+ const sp<media::INativeSpatializerCallback>& callback,
+ media::GetSpatializerResponse* _aidl_return) {
+ _aidl_return->spatializer = nullptr;
+ LOG_ALWAYS_FATAL_IF(callback == nullptr);
+ RETURN_IF_BINDER_ERROR(binderStatusFromStatusT(mSpatializer->registerCallback(callback)));
+ _aidl_return->spatializer = mSpatializer;
+ return Status::ok();
+}
+
+Status AudioPolicyService::canBeSpatialized(
+ const std::optional<media::AudioAttributesInternal>& attrAidl,
+ const std::optional<media::AudioConfig>& configAidl,
+ const std::vector<media::AudioDevice>& devicesAidl,
+ bool* _aidl_return) {
+ if (mAudioPolicyManager == nullptr) {
+ return binderStatusFromStatusT(NO_INIT);
+ }
+ audio_attributes_t attr = AUDIO_ATTRIBUTES_INITIALIZER;
+ if (attrAidl.has_value()) {
+ attr = VALUE_OR_RETURN_BINDER_STATUS(
+ aidl2legacy_AudioAttributesInternal_audio_attributes_t(attrAidl.value()));
+ }
+ audio_config_t config = AUDIO_CONFIG_INITIALIZER;
+ if (configAidl.has_value()) {
+ config = VALUE_OR_RETURN_BINDER_STATUS(
+ aidl2legacy_AudioConfig_audio_config_t(configAidl.value()));
+ }
+ AudioDeviceTypeAddrVector devices = VALUE_OR_RETURN_BINDER_STATUS(
+ convertContainer<AudioDeviceTypeAddrVector>(devicesAidl,
+ aidl2legacy_AudioDeviceTypeAddress));
+
+ Mutex::Autolock _l(mLock);
+ *_aidl_return = mAudioPolicyManager->canBeSpatialized(&attr, &config, devices);
+ return Status::ok();
+}
+
} // namespace android
diff --git a/services/audiopolicy/service/AudioPolicyService.cpp b/services/audiopolicy/service/AudioPolicyService.cpp
index 4d0e1f1..de71a00 100644
--- a/services/audiopolicy/service/AudioPolicyService.cpp
+++ b/services/audiopolicy/service/AudioPolicyService.cpp
@@ -127,6 +127,7 @@
loadAudioPolicyManager();
mAudioPolicyManager = mCreateAudioPolicyManager(mAudioPolicyClient);
}
+
// load audio processing modules
sp<AudioPolicyEffects> audioPolicyEffects = new AudioPolicyEffects();
sp<UidPolicy> uidPolicy = new UidPolicy(this);
@@ -139,6 +140,15 @@
}
uidPolicy->registerSelf();
sensorPrivacyPolicy->registerSelf();
+
+ // Create spatializer if supported
+ const audio_attributes_t attr = attributes_initializer(AUDIO_USAGE_MEDIA);
+ AudioDeviceTypeAddrVector devices;
+ bool hasSpatializer = mAudioPolicyManager->canBeSpatialized(&attr, nullptr, devices);
+ if (hasSpatializer) {
+ mSpatializer = Spatializer::create(this);
+ }
+ AudioSystem::audioPolicyReady();
}
void AudioPolicyService::unloadAudioPolicyManager()
@@ -353,6 +363,59 @@
}
}
+void AudioPolicyService::onCheckSpatializer()
+{
+ Mutex::Autolock _l(mLock);
+ onCheckSpatializer_l();
+}
+
+void AudioPolicyService::onCheckSpatializer_l()
+{
+ if (mSpatializer != nullptr) {
+ mOutputCommandThread->checkSpatializerCommand();
+ }
+}
+
+void AudioPolicyService::doOnCheckSpatializer()
+{
+ Mutex::Autolock _l(mLock);
+
+ if (mSpatializer != nullptr) {
+ if (mSpatializer->getLevel() != media::SpatializationLevel::NONE) {
+ audio_io_handle_t currentOutput = mSpatializer->getOutput();
+ audio_io_handle_t newOutput;
+ const audio_attributes_t attr = attributes_initializer(AUDIO_USAGE_MEDIA);
+ audio_config_base_t config = mSpatializer->getAudioInConfig();
+ status_t status =
+ mAudioPolicyManager->getSpatializerOutput(&config, &attr, &newOutput);
+
+ if (status == NO_ERROR && currentOutput == newOutput) {
+ return;
+ }
+ mLock.unlock();
+ // It is OK to call detachOutput() is none is already attached.
+ mSpatializer->detachOutput();
+ if (status != NO_ERROR || newOutput == AUDIO_IO_HANDLE_NONE) {
+ mLock.lock();
+ return;
+ }
+ status = mSpatializer->attachOutput(newOutput);
+ mLock.lock();
+ if (status != NO_ERROR) {
+ mAudioPolicyManager->releaseSpatializerOutput(newOutput);
+ }
+ } else if (mSpatializer->getLevel() == media::SpatializationLevel::NONE
+ && mSpatializer->getOutput() != AUDIO_IO_HANDLE_NONE) {
+ mLock.unlock();
+ audio_io_handle_t output = mSpatializer->detachOutput();
+ mLock.lock();
+ if (output != AUDIO_IO_HANDLE_NONE) {
+ mAudioPolicyManager->releaseSpatializerOutput(output);
+ }
+ }
+ }
+}
+
status_t AudioPolicyService::clientCreateAudioPatch(const struct audio_patch *patch,
audio_patch_handle_t *handle,
int delayMs)
@@ -990,7 +1053,8 @@
case TRANSACTION_addDevicesRoleForCapturePreset:
case TRANSACTION_removeDevicesRoleForCapturePreset:
case TRANSACTION_clearDevicesRoleForCapturePreset:
- case TRANSACTION_getDevicesForRoleAndCapturePreset: {
+ case TRANSACTION_getDevicesForRoleAndCapturePreset:
+ case TRANSACTION_getSpatializer: {
if (!isServiceUid(IPCThreadState::self()->getCallingUid())) {
ALOGW("%s: transaction %d received from PID %d unauthorized UID %d",
__func__, code, IPCThreadState::self()->getCallingPid(),
@@ -1764,6 +1828,17 @@
mLock.lock();
} break;
+ case CHECK_SPATIALIZER: {
+ ALOGV("AudioCommandThread() processing updateUID states");
+ svc = mService.promote();
+ if (svc == 0) {
+ break;
+ }
+ mLock.unlock();
+ svc->doOnCheckSpatializer();
+ mLock.lock();
+ } break;
+
default:
ALOGW("AudioCommandThread() unknown command %d", command->mCommand);
}
@@ -2075,6 +2150,14 @@
sendCommand(command);
}
+void AudioPolicyService::AudioCommandThread::checkSpatializerCommand()
+{
+ sp<AudioCommand>command = new AudioCommand();
+ command->mCommand = CHECK_SPATIALIZER;
+ ALOGV("AudioCommandThread() adding check spatializer");
+ sendCommand(command);
+}
+
status_t AudioPolicyService::AudioCommandThread::sendCommand(sp<AudioCommand>& command, int delayMs)
{
{
diff --git a/services/audiopolicy/service/AudioPolicyService.h b/services/audiopolicy/service/AudioPolicyService.h
index 7ed829c..27c4e1c 100644
--- a/services/audiopolicy/service/AudioPolicyService.h
+++ b/services/audiopolicy/service/AudioPolicyService.h
@@ -19,6 +19,7 @@
#define ANDROID_AUDIOPOLICYSERVICE_H
#include <android/media/BnAudioPolicyService.h>
+#include <android/media/GetSpatializerResponse.h>
#include <android-base/thread_annotations.h>
#include <cutils/misc.h>
#include <cutils/config_utils.h>
@@ -38,6 +39,7 @@
#include <mediautils/ServiceUtilities.h>
#include "AudioPolicyEffects.h"
#include "CaptureStateNotifier.h"
+#include "Spatializer.h"
#include <AudioPolicyInterface.h>
#include <android/hardware/BnSensorPrivacyListener.h>
#include <android/content/AttributionSourceState.h>
@@ -53,7 +55,8 @@
class AudioPolicyService :
public BinderService<AudioPolicyService>,
public media::BnAudioPolicyService,
- public IBinder::DeathRecipient
+ public IBinder::DeathRecipient,
+ public SpatializerPolicyCallback
{
friend class BinderService<AudioPolicyService>;
@@ -243,11 +246,15 @@
binder::Status registerSoundTriggerCaptureStateListener(
const sp<media::ICaptureStateListener>& listener, bool* _aidl_return) override;
- virtual status_t onTransact(
- uint32_t code,
- const Parcel& data,
- Parcel* reply,
- uint32_t flags);
+ binder::Status getSpatializer(const sp<media::INativeSpatializerCallback>& callback,
+ media::GetSpatializerResponse* _aidl_return) override;
+ binder::Status canBeSpatialized(
+ const std::optional<media::AudioAttributesInternal>& attr,
+ const std::optional<media::AudioConfig>& config,
+ const std::vector<media::AudioDevice>& devices,
+ bool* _aidl_return) override;
+
+ status_t onTransact(uint32_t code, const Parcel& data, Parcel* reply, uint32_t flags) override;
// IBinder::DeathRecipient
virtual void binderDied(const wp<IBinder>& who);
@@ -313,6 +320,16 @@
void onRoutingUpdated();
void doOnRoutingUpdated();
+ /**
+ * Spatializer SpatializerPolicyCallback implementation.
+ * onCheckSpatializer() sends an event on mOutputCommandThread which executes
+ * doOnCheckSpatializer() to check if a Spatializer output must be opened or closed
+ * by audio policy manager and attach/detach the spatializer effect accordingly.
+ */
+ void onCheckSpatializer() override;
+ void onCheckSpatializer_l();
+ void doOnCheckSpatializer();
+
void setEffectSuspended(int effectId,
audio_session_t sessionId,
bool suspended);
@@ -483,7 +500,8 @@
SET_EFFECT_SUSPENDED,
AUDIO_MODULES_UPDATE,
ROUTING_UPDATED,
- UPDATE_UID_STATES
+ UPDATE_UID_STATES,
+ CHECK_SPATIALIZER
};
AudioCommandThread (String8 name, const wp<AudioPolicyService>& service);
@@ -532,6 +550,7 @@
void audioModulesUpdateCommand();
void routingChangedCommand();
void updateUidStatesCommand();
+ void checkSpatializerCommand();
void insertCommand_l(AudioCommand *command, int delayMs = 0);
private:
class AudioCommandData;
@@ -667,7 +686,8 @@
// The audio policy manager can check if the proposed parameters are suitable or not and act accordingly.
virtual status_t openOutput(audio_module_handle_t module,
audio_io_handle_t *output,
- audio_config_t *config,
+ audio_config_t *halConfig,
+ audio_config_base_t *mixerConfig,
const sp<DeviceDescriptorBase>& device,
uint32_t *latencyMs,
audio_output_flags_t flags);
@@ -985,6 +1005,8 @@
CaptureStateNotifier mCaptureStateNotifier;
+ sp<Spatializer> mSpatializer;
+
void *mLibraryHandle = nullptr;
CreateAudioPolicyManagerInstance mCreateAudioPolicyManager;
DestroyAudioPolicyManagerInstance mDestroyAudioPolicyManager;
diff --git a/services/audiopolicy/service/Spatializer.cpp b/services/audiopolicy/service/Spatializer.cpp
new file mode 100644
index 0000000..aa104a0
--- /dev/null
+++ b/services/audiopolicy/service/Spatializer.cpp
@@ -0,0 +1,377 @@
+/*
+**
+** Copyright 2021, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+
+
+#define LOG_TAG "Spatializer"
+//#define LOG_NDEBUG 0
+#include <utils/Log.h>
+
+#include <limits.h>
+#include <stdint.h>
+#include <sys/types.h>
+
+#include <android/content/AttributionSourceState.h>
+#include <audio_utils/fixedfft.h>
+#include <cutils/bitops.h>
+#include <media/ShmemCompat.h>
+#include <media/audiohal/EffectsFactoryHalInterface.h>
+#include <mediautils/ServiceUtilities.h>
+#include <utils/Thread.h>
+
+#include "Spatializer.h"
+
+namespace android {
+
+using aidl_utils::statusTFromBinderStatus;
+using aidl_utils::binderStatusFromStatusT;
+using android::content::AttributionSourceState;
+using binder::Status;
+using media::SpatializationLevel;
+using media::SpatializerHeadTrackingMode;
+
+#define VALUE_OR_RETURN_BINDER_STATUS(x) \
+ ({ auto _tmp = (x); \
+ if (!_tmp.ok()) return aidl_utils::binderStatusFromStatusT(_tmp.error()); \
+ std::move(_tmp.value()); })
+
+#define RETURN_IF_BINDER_ERROR(x) \
+ { \
+ binder::Status _tmp = (x); \
+ if (!_tmp.isOk()) return _tmp; \
+ }
+
+// ---------------------------------------------------------------------------
+
+sp<Spatializer> Spatializer::create(SpatializerPolicyCallback *callback) {
+ sp<Spatializer> spatializer;
+
+ sp<EffectsFactoryHalInterface> effectsFactoryHal = EffectsFactoryHalInterface::create();
+ if (effectsFactoryHal == nullptr) {
+ ALOGW("%s failed to create effect factory interface", __func__);
+ return spatializer;
+ }
+
+ std::vector<effect_descriptor_t> descriptors;
+ status_t status =
+ effectsFactoryHal->getDescriptors(FX_IID_SPATIALIZER, &descriptors);
+ if (status != NO_ERROR) {
+ ALOGW("%s failed to get spatializer descriptor, error %d", __func__, status);
+ return spatializer;
+ }
+ ALOG_ASSERT(!descriptors.empty(),
+ "%s getDescriptors() returned no error but empty list", __func__);
+
+ //TODO: get supported spatialization modes from FX engine or descriptor
+
+ sp<EffectHalInterface> effect;
+ status = effectsFactoryHal->createEffect(&descriptors[0].uuid, AUDIO_SESSION_OUTPUT_STAGE,
+ AUDIO_IO_HANDLE_NONE, AUDIO_PORT_HANDLE_NONE, &effect);
+ ALOGI("%s FX create status %d effect %p", __func__, status, effect.get());
+
+ if (status == NO_ERROR && effect != nullptr) {
+ spatializer = new Spatializer(descriptors[0], callback);
+ // TODO: Read supported config from engine
+ audio_config_base_t config = AUDIO_CONFIG_BASE_INITIALIZER;
+ config.channel_mask = AUDIO_CHANNEL_OUT_5POINT1;
+ spatializer->setAudioInConfig(config);
+ }
+
+ return spatializer;
+}
+
+Spatializer::Spatializer(effect_descriptor_t engineDescriptor,
+ SpatializerPolicyCallback *callback)
+ : mEngineDescriptor(engineDescriptor), mPolicyCallback(callback) {
+ ALOGV("%s", __func__);
+}
+
+Spatializer::~Spatializer() {
+ ALOGV("%s", __func__);
+}
+
+status_t Spatializer::registerCallback(
+ const sp<media::INativeSpatializerCallback>& callback) {
+ Mutex::Autolock _l(mLock);
+ if (callback == nullptr) {
+ return BAD_VALUE;
+ }
+
+ sp<IBinder> binder = IInterface::asBinder(callback);
+ status_t status = binder->linkToDeath(this);
+ if (status == NO_ERROR) {
+ mSpatializerCallback = callback;
+ }
+ ALOGV("%s status %d", __func__, status);
+ return status;
+}
+
+// IBinder::DeathRecipient
+void Spatializer::binderDied(__unused const wp<IBinder> &who) {
+ {
+ Mutex::Autolock _l(mLock);
+ mLevel = SpatializationLevel::NONE;
+ mSpatializerCallback.clear();
+ }
+ ALOGV("%s", __func__);
+ mPolicyCallback->onCheckSpatializer();
+}
+
+// ISpatializer
+Status Spatializer::getSupportedLevels(std::vector<SpatializationLevel> *levels) {
+ ALOGV("%s", __func__);
+ if (levels == nullptr) {
+ return binderStatusFromStatusT(BAD_VALUE);
+ }
+ //TODO: get this from engine
+ levels->push_back(SpatializationLevel::NONE);
+ levels->push_back(SpatializationLevel::SPATIALIZER_MULTICHANNEL);
+ return Status::ok();
+}
+
+Status Spatializer::setLevel(media::SpatializationLevel level) {
+ ALOGV("%s level %d", __func__, (int)level);
+ if (level != SpatializationLevel::NONE
+ && level != SpatializationLevel::SPATIALIZER_MULTICHANNEL) {
+ return binderStatusFromStatusT(BAD_VALUE);
+ }
+ sp<media::INativeSpatializerCallback> callback;
+ bool levelChanged = false;
+ {
+ Mutex::Autolock _l(mLock);
+ levelChanged = mLevel != level;
+ mLevel = level;
+ callback = mSpatializerCallback;
+ }
+
+ if (levelChanged) {
+ mPolicyCallback->onCheckSpatializer();
+ if (callback != nullptr) {
+ callback->onLevelChanged(level);
+ }
+ }
+ return Status::ok();
+}
+
+Status Spatializer::getLevel(media::SpatializationLevel *level) {
+ if (level == nullptr) {
+ return binderStatusFromStatusT(BAD_VALUE);
+ }
+ Mutex::Autolock _l(mLock);
+ *level = mLevel;
+ ALOGV("%s level %d", __func__, (int)*level);
+ return Status::ok();
+}
+
+Status Spatializer::getSupportedHeadTrackingModes(
+ std::vector<media::SpatializerHeadTrackingMode>* modes) {
+ ALOGV("%s", __func__);
+ if (modes == nullptr) {
+ return binderStatusFromStatusT(BAD_VALUE);
+ }
+ //TODO: get this from:
+ // - The engine capabilities
+ // - If a head tracking sensor is registered and linked to a connected audio device
+ // - if we have indications on the screen orientation
+ modes->push_back(SpatializerHeadTrackingMode::RELATIVE_WORLD);
+ return Status::ok();
+}
+
+Status Spatializer::setDesiredHeadTrackingMode(media::SpatializerHeadTrackingMode mode) {
+ ALOGV("%s level %d", __func__, (int)mode);
+ if (mode != SpatializerHeadTrackingMode::DISABLED
+ && mode != SpatializerHeadTrackingMode::RELATIVE_WORLD) {
+ return binderStatusFromStatusT(BAD_VALUE);
+ }
+ {
+ Mutex::Autolock _l(mLock);
+ mHeadTrackingMode = mode;
+ }
+ return Status::ok();
+}
+
+Status Spatializer::getActualHeadTrackingMode(media::SpatializerHeadTrackingMode *mode) {
+ if (mode == nullptr) {
+ return binderStatusFromStatusT(BAD_VALUE);
+ }
+ Mutex::Autolock _l(mLock);
+ *mode = mHeadTrackingMode;
+ ALOGV("%s mode %d", __func__, (int)*mode);
+ return Status::ok();
+}
+
+Status Spatializer::recenterHeadTracker() {
+ return Status::ok();
+}
+
+Status Spatializer::setGlobalTransform(const std::vector<float>& screenToStage) {
+ Mutex::Autolock _l(mLock);
+ mScreenToStageTransform = screenToStage;
+ ALOGV("%s", __func__);
+ return Status::ok();
+}
+
+Status Spatializer::release() {
+ ALOGV("%s", __func__);
+ bool levelChanged = false;
+ {
+ Mutex::Autolock _l(mLock);
+ if (mSpatializerCallback == nullptr) {
+ return binderStatusFromStatusT(INVALID_OPERATION);
+ }
+
+ sp<IBinder> binder = IInterface::asBinder(mSpatializerCallback);
+ binder->unlinkToDeath(this);
+ mSpatializerCallback.clear();
+
+ levelChanged = mLevel != SpatializationLevel::NONE;
+ mLevel = SpatializationLevel::NONE;
+ }
+
+ if (levelChanged) {
+ mPolicyCallback->onCheckSpatializer();
+ }
+ return Status::ok();
+}
+
+status_t Spatializer::attachOutput(audio_io_handle_t output) {
+ Mutex::Autolock _l(mLock);
+ ALOGV("%s output %d mOutput %d", __func__, (int)output, (int)mOutput);
+ if (mOutput != AUDIO_IO_HANDLE_NONE) {
+ LOG_ALWAYS_FATAL_IF(mEngine != nullptr, "%s output set without FX engine", __func__);
+ // remove FX instance
+ mEngine->setEnabled(false);
+ mEngine.clear();
+ }
+ // create FX instance on output
+ AttributionSourceState attributionSource = AttributionSourceState();
+ mEngine = new AudioEffect(attributionSource);
+ mEngine->set(nullptr, &mEngineDescriptor.uuid, 0, Spatializer::engineCallback /* cbf */,
+ this /* user */, AUDIO_SESSION_OUTPUT_STAGE, output, {} /* device */,
+ false /* probe */, true /* notifyFramesProcessed */);
+ status_t status = mEngine->initCheck();
+ ALOGV("%s mEngine create status %d", __func__, (int)status);
+ if (status != NO_ERROR) {
+ return status;
+ }
+ mEngine->setEnabled(true);
+ mOutput = output;
+ return NO_ERROR;
+}
+
+audio_io_handle_t Spatializer::detachOutput() {
+ Mutex::Autolock _l(mLock);
+ ALOGV("%s mOutput %d", __func__, (int)mOutput);
+ if (mOutput == AUDIO_IO_HANDLE_NONE) {
+ return AUDIO_IO_HANDLE_NONE;
+ }
+ // remove FX instance
+ mEngine->setEnabled(false);
+ mEngine.clear();
+ audio_io_handle_t output = mOutput;
+ mOutput = AUDIO_IO_HANDLE_NONE;
+ return output;
+}
+
+void Spatializer::engineCallback(int32_t event, void *user, void *info) {
+
+ if (user == nullptr) {
+ return;
+ }
+ const Spatializer * const me = reinterpret_cast<Spatializer *>(user);
+ switch (event) {
+ case AudioEffect::EVENT_FRAMES_PROCESSED: {
+ int frames = info == nullptr ? 0 : *(int *)info;
+ ALOGD("%s frames processed %d for me %p", __func__, frames, me);
+ } break;
+ default:
+ ALOGD("%s event %d", __func__, event);
+ break;
+ }
+}
+
+// ---------------------------------------------------------------------------
+
+Spatializer::EffectClient::EffectClient(const sp<media::IEffectClient>& effectClient,
+ Spatializer& parent)
+ : BnEffect(),
+ mEffectClient(effectClient), mParent(parent) {
+}
+
+Spatializer::EffectClient::~EffectClient() {
+}
+
+// IEffect
+
+#define RETURN(code) \
+ *_aidl_return = (code); \
+ return Status::ok();
+
+// Write a POD value into a vector of bytes (clears the previous buffer
+// content).
+template<typename T>
+void writeToBuffer(const T& value, std::vector<uint8_t>* buffer) {
+ buffer->clear();
+ appendToBuffer(value, buffer);
+}
+
+Status Spatializer::EffectClient::enable(int32_t* _aidl_return) {
+ RETURN(OK);
+}
+
+Status Spatializer::EffectClient::disable(int32_t* _aidl_return) {
+ RETURN(OK);
+}
+
+Status Spatializer::EffectClient::command(int32_t cmdCode,
+ const std::vector<uint8_t>& cmdData __unused,
+ int32_t maxResponseSize __unused,
+ std::vector<uint8_t>* response __unused,
+ int32_t* _aidl_return) {
+
+ // reject commands reserved for internal use by audio framework if coming from outside
+ // of audioserver
+ switch(cmdCode) {
+ case EFFECT_CMD_ENABLE:
+ case EFFECT_CMD_DISABLE:
+ case EFFECT_CMD_SET_PARAM_DEFERRED:
+ case EFFECT_CMD_SET_PARAM_COMMIT:
+ RETURN(BAD_VALUE);
+ case EFFECT_CMD_SET_PARAM:
+ case EFFECT_CMD_GET_PARAM:
+ break;
+ default:
+ if (cmdCode >= EFFECT_CMD_FIRST_PROPRIETARY) {
+ break;
+ }
+ android_errorWriteLog(0x534e4554, "62019992");
+ RETURN(BAD_VALUE);
+ }
+ (void)mParent;
+ RETURN(OK);
+}
+
+Status Spatializer::EffectClient::disconnect() {
+ mDisconnected = true;
+ return Status::ok();
+}
+
+Status Spatializer::EffectClient::getCblk(media::SharedFileRegion* _aidl_return) {
+ LOG_ALWAYS_FATAL_IF(!convertIMemoryToSharedFileRegion(mCblkMemory, _aidl_return));
+ return Status::ok();
+}
+
+} // namespace android
diff --git a/services/audiopolicy/service/Spatializer.h b/services/audiopolicy/service/Spatializer.h
new file mode 100644
index 0000000..768170a
--- /dev/null
+++ b/services/audiopolicy/service/Spatializer.h
@@ -0,0 +1,207 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_MEDIA_SPATIALIZER_H
+#define ANDROID_MEDIA_SPATIALIZER_H
+
+#include <android/media/BnEffect.h>
+#include <android/media/BnSpatializer.h>
+#include <android/media/SpatializerHeadTrackingMode.h>
+#include <android/media/SpatializationLevel.h>
+
+#include <media/AudioEffect.h>
+#include <system/audio_effects/effect_spatializer.h>
+
+
+namespace android {
+
+
+// ----------------------------------------------------------------------------
+
+/**
+ * A callback interface from the Spatializer object or its parent AudioPolicyService.
+ * This is implemented by the audio policy service hosting the Spatializer to perform
+ * actions needed when a state change inside the Spatializer requires some audio system
+ * changes that cannot be performed by the Spatializer. For instance opening or closing a
+ * spatializer output stream when the spatializer is enabled or disabled
+ */
+class SpatializerPolicyCallback {
+public:
+ /** Called when a stage change occurs that requires the parent audio policy service to take
+ * some action.
+ */
+ virtual void onCheckSpatializer() = 0;
+
+ virtual ~SpatializerPolicyCallback() = default;
+};
+/**
+ * The Spatializer class implements all functional controlling the multichannel spatializer
+ * with head tracking implementation in the native audio service: audio policy and audio flinger.
+ * It presents an AIDL interface available to the java audio service to discover the availability
+ * of the feature and options, control its state and register an active head tracking sensor.
+ * It maintains the current state of the platform spatializer and applies the stored parameters
+ * when the spatializer engine is created and enabled.
+ * Based on the requested spatializer level, it will request the creation of a specialized output
+ * mixer to the audio policy service which will in turn notify the Spatializer of the output
+ * stream on which a spatializer engine should be created, configured and enabled.
+ * The spatializer also hosts the head tracking management logic. This logic receives the
+ * desired head tracking mode and selected head tracking sensor, registers a sensor event listener
+ * and derives the compounded head pose information to the spatializer engine.
+ *
+ * Workflow:
+ * - Initialization: when the audio policy service starts, it checks if a spatializer effect
+ * engine exists and if the audio policy manager reports a dedicated spatializer output profile.
+ * If both conditions are met, a Spatializer object is created
+ * - Capabilities discovery: AudioService will call AudioSystem::canBeSpatialized() and if true,
+ * acquire an ISpatializer interface with AudioSystem::getSpatializer(). This interface
+ * will be used to query the implementation capabilities and configure the spatializer.
+ * - Enabling: when ISpatializer::setLevel() sets a level different from NONE the spatializer
+ * is considered enabled. The audio policy callback onCheckSpatializer() is called. This
+ * triggers a request to audio policy manager to open a spatialization output stream and a
+ * spatializer mixer is created in audio flinger. When an output is returned by audio policy
+ * manager, Spatializer::attachOutput() is called which creates and enables the spatializer
+ * stage engine on the specified output.
+ * - Disabling: when the spatialization level is set to NONE, the spatializer is considered
+ * disabled. The audio policy callback onCheckSpatializer() is called. This triggers a call
+ * to Spatializer::detachOutput() and the spatializer engine is released. Then a request is
+ * made to audio policy manager to release and close the spatializer output stream and the
+ * spatializer mixer thread is destroyed.
+ */
+class Spatializer : public media::BnSpatializer, public IBinder::DeathRecipient {
+public:
+
+ static sp<Spatializer> create(SpatializerPolicyCallback *callback);
+
+ ~Spatializer() override;
+
+ /** ISpatializer, see ISpatializer.aidl */
+ binder::Status release() override;
+ binder::Status getSupportedLevels(std::vector<media::SpatializationLevel>* levels) override;
+ binder::Status setLevel(media::SpatializationLevel level) override;
+ binder::Status getLevel(media::SpatializationLevel *level) override;
+ binder::Status getSupportedHeadTrackingModes(
+ std::vector<media::SpatializerHeadTrackingMode>* modes) override;
+ binder::Status setDesiredHeadTrackingMode(
+ media::SpatializerHeadTrackingMode mode) override;
+ binder::Status getActualHeadTrackingMode(
+ media::SpatializerHeadTrackingMode* mode) override;
+ binder::Status recenterHeadTracker() override;
+ binder::Status setGlobalTransform(const std::vector<float>& screenToStage) override;
+
+ /** IBinder::DeathRecipient. Listen to the death of the INativeSpatializerCallback. */
+ virtual void binderDied(const wp<IBinder>& who);
+
+ /** Registers a INativeSpatializerCallback when a client is attached to this Spatializer
+ * by audio policy service.
+ */
+ status_t registerCallback(const sp<media::INativeSpatializerCallback>& callback);
+
+ /** Level getter for use by local classes. */
+ media::SpatializationLevel getLevel() const { Mutex::Autolock _l(mLock); return mLevel; }
+
+ /** Called by audio policy service when the special output mixer dedicated to spatialization
+ * is opened and the spatializer engine must be created.
+ */
+ status_t attachOutput(audio_io_handle_t output);
+ /** Called by audio policy service when the special output mixer dedicated to spatialization
+ * is closed and the spatializer engine must be release.
+ */
+ audio_io_handle_t detachOutput();
+ /** Returns the output stream the spatializer is attached to. */
+ audio_io_handle_t getOutput() const { Mutex::Autolock _l(mLock); return mOutput; }
+
+ /** Sets the channel mask, sampling rate and format for the spatializer input. */
+ void setAudioInConfig(const audio_config_base_t& config) {
+ Mutex::Autolock _l(mLock);
+ mAudioInConfig = config;
+ }
+
+ /** Gets the channel mask, sampling rate and format set for the spatializer input. */
+ audio_config_base_t getAudioInConfig() const {
+ Mutex::Autolock _l(mLock);
+ return mAudioInConfig;
+ }
+
+ /** An implementation of an IEffect interface that can be used to pass advanced parameters to
+ * the spatializer engine. All APis are noop (i.e. the interface cannot be used to control
+ * the effect) except for passing parameters via the command() API. */
+ class EffectClient: public android::media::BnEffect {
+ public:
+
+ EffectClient(const sp<media::IEffectClient>& effectClient,
+ Spatializer& parent);
+ virtual ~EffectClient();
+
+ // IEffect
+ android::binder::Status enable(int32_t* _aidl_return) override;
+ android::binder::Status disable(int32_t* _aidl_return) override;
+ android::binder::Status command(int32_t cmdCode,
+ const std::vector<uint8_t>& cmdData,
+ int32_t maxResponseSize,
+ std::vector<uint8_t>* response,
+ int32_t* _aidl_return) override;
+ android::binder::Status disconnect() override;
+ android::binder::Status getCblk(media::SharedFileRegion* _aidl_return) override;
+
+ private:
+ const sp<media::IEffectClient> mEffectClient;
+ sp<IMemory> mCblkMemory;
+ const Spatializer& mParent;
+ bool mDisconnected = false;
+ };
+
+private:
+
+ Spatializer(effect_descriptor_t engineDescriptor,
+ SpatializerPolicyCallback *callback);
+
+
+ static void engineCallback(int32_t event, void* user, void *info);
+
+ /** Effect engine descriptor */
+ const effect_descriptor_t mEngineDescriptor;
+ /** Callback interface to parent audio policy service */
+ SpatializerPolicyCallback* mPolicyCallback;
+
+ /** Mutex protecting internal state */
+ mutable Mutex mLock;
+
+ /** Client AudioEffect for the engine */
+ sp<AudioEffect> mEngine GUARDED_BY(mLock);
+ /** Output stream the spatializer mixer thread is attached to */
+ audio_io_handle_t mOutput GUARDED_BY(mLock) = AUDIO_IO_HANDLE_NONE;
+ /** Virtualizer engine input configuration */
+ audio_config_base_t mAudioInConfig GUARDED_BY(mLock) = AUDIO_CONFIG_BASE_INITIALIZER;
+
+ /** Callback interface to the client (AudioService) controlling this`Spatializer */
+ sp<media::INativeSpatializerCallback> mSpatializerCallback GUARDED_BY(mLock);
+
+ /** Requested spatialization level */
+ media::SpatializationLevel mLevel GUARDED_BY(mLock) = media::SpatializationLevel::NONE;
+ /** Requested head tracking mode */
+ media::SpatializerHeadTrackingMode mHeadTrackingMode GUARDED_BY(mLock)
+ = media::SpatializerHeadTrackingMode::DISABLED;
+ /** Configured screen to stage transform */
+ std::vector<float> mScreenToStageTransform GUARDED_BY(mLock);
+
+ /** Extended IEffect interface is one has been created */
+ sp<EffectClient> mEffectClient GUARDED_BY(mLock);
+};
+
+
+}; // namespace android
+
+#endif // ANDROID_MEDIA_SPATIALIZER_H
diff --git a/services/audiopolicy/service/SpatializerPoseController.cpp b/services/audiopolicy/service/SpatializerPoseController.cpp
new file mode 100644
index 0000000..f0d7f7c
--- /dev/null
+++ b/services/audiopolicy/service/SpatializerPoseController.cpp
@@ -0,0 +1,196 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "SpatializerPoseController.h"
+
+#define LOG_TAG "VirtualizerStageController"
+//#define LOG_NDEBUG 0
+#include <utils/Log.h>
+#include <utils/SystemClock.h>
+
+namespace android {
+
+using media::createHeadTrackingProcessor;
+using media::HeadTrackingMode;
+using media::HeadTrackingProcessor;
+using media::Pose3f;
+using media::SensorPoseProvider;
+using media::Twist3f;
+
+using namespace std::chrono_literals;
+
+namespace {
+
+// This is how fast, in m/s, we allow position to shift during rate-limiting.
+constexpr auto kMaxTranslationalVelocity = 2 ;
+
+// This is how fast, in rad/s, we allow rotation angle to shift during rate-limiting.
+constexpr auto kMaxRotationalVelocity = 4 * M_PI ;
+
+// This should be set to the typical time scale that the translation sensors used drift in. This
+// means, loosely, for how long we can trust the reading to be "accurate enough". This would
+// determine the time constants used for high-pass filtering those readings. If the value is set
+// too high, we may experience drift. If it is set too low, we may experience poses tending toward
+// identity too fast.
+constexpr auto kTranslationalDriftTimeConstant = 20s;
+
+// This should be set to the typical time scale that the rotation sensors used drift in. This
+// means, loosely, for how long we can trust the reading to be "accurate enough". This would
+// determine the time constants used for high-pass filtering those readings. If the value is set
+// too high, we may experience drift. If it is set too low, we may experience poses tending toward
+// identity too fast.
+constexpr auto kRotationalDriftTimeConstant = 20s;
+
+// This is how far into the future we predict the head pose, using linear extrapolation based on
+// twist (velocity). It should be set to a value that matches the characteristic durations of moving
+// one's head. The higher we set this, the more latency we are able to reduce, but setting this too
+// high will result in high prediction errors whenever the head accelerates (changes velocity).
+constexpr auto kPredictionDuration = 10ms;
+
+// After losing this many consecutive samples from either sensor, we would treat the measurement as
+// stale;
+constexpr auto kMaxLostSamples = 4;
+
+// Time units for system clock ticks. This is what the Sensor Framework timestamps represent and
+// what we use for pose filtering.
+using Ticks = std::chrono::nanoseconds;
+
+// How many ticks in a second.
+constexpr auto kTicksPerSecond = Ticks::period::den;
+
+} // namespace
+
+SpatializerPoseController::SpatializerPoseController(Listener* listener,
+ std::chrono::microseconds sensorPeriod,
+ std::chrono::microseconds maxUpdatePeriod)
+ : mListener(listener),
+ mSensorPeriod(sensorPeriod),
+ mPoseProvider(SensorPoseProvider::create("headtracker", this)),
+ mProcessor(createHeadTrackingProcessor(HeadTrackingProcessor::Options{
+ .maxTranslationalVelocity = kMaxTranslationalVelocity / kTicksPerSecond,
+ .maxRotationalVelocity = kMaxRotationalVelocity / kTicksPerSecond,
+ .translationalDriftTimeConstant = Ticks(kTranslationalDriftTimeConstant).count(),
+ .rotationalDriftTimeConstant = Ticks(kRotationalDriftTimeConstant).count(),
+ .freshnessTimeout = Ticks(sensorPeriod * kMaxLostSamples).count(),
+ .predictionDuration = Ticks(kPredictionDuration).count(),
+ })),
+ mThread([this, maxUpdatePeriod] {
+ while (true) {
+ {
+ std::unique_lock lock(mMutex);
+ mCondVar.wait_for(lock, maxUpdatePeriod,
+ [this] { return mShouldExit || mShouldCalculate; });
+ if (mShouldExit) {
+ ALOGV("Exiting thread");
+ return;
+ }
+ calculate_l();
+ if (!mCalculated) {
+ mCalculated = true;
+ mCondVar.notify_all();
+ }
+ mShouldCalculate = false;
+ }
+ }
+ }) {}
+
+SpatializerPoseController::~SpatializerPoseController() {
+ {
+ std::unique_lock lock(mMutex);
+ mShouldExit = true;
+ mCondVar.notify_all();
+ }
+ mThread.join();
+}
+
+void SpatializerPoseController::setHeadSensor(const ASensor* sensor) {
+ std::lock_guard lock(mMutex);
+ // Stop current sensor, if valid.
+ if (mHeadSensor != SensorPoseProvider::INVALID_HANDLE) {
+ mPoseProvider->stopSensor(mHeadSensor);
+ }
+ // Start new sensor, if valid.
+ mHeadSensor = sensor != nullptr ? mPoseProvider->startSensor(sensor, mSensorPeriod)
+ : SensorPoseProvider::INVALID_HANDLE;
+ mProcessor->recenter();
+}
+
+void SpatializerPoseController::setScreenSensor(const ASensor* sensor) {
+ std::lock_guard lock(mMutex);
+ // Stop current sensor, if valid.
+ if (mScreenSensor != SensorPoseProvider::INVALID_HANDLE) {
+ mPoseProvider->stopSensor(mScreenSensor);
+ }
+ // Start new sensor, if valid.
+ mScreenSensor = sensor != nullptr ? mPoseProvider->startSensor(sensor, mSensorPeriod)
+ : SensorPoseProvider::INVALID_HANDLE;
+ mProcessor->recenter();
+}
+
+void SpatializerPoseController::setDesiredMode(HeadTrackingMode mode) {
+ std::lock_guard lock(mMutex);
+ mProcessor->setDesiredMode(mode);
+}
+
+void SpatializerPoseController::setScreenToStagePose(const Pose3f& screenToStage) {
+ std::lock_guard lock(mMutex);
+ mProcessor->setScreenToStagePose(screenToStage);
+}
+
+void SpatializerPoseController::setDisplayOrientation(float physicalToLogicalAngle) {
+ std::lock_guard lock(mMutex);
+ mProcessor->setDisplayOrientation(physicalToLogicalAngle);
+}
+
+void SpatializerPoseController::calculateAsync() {
+ std::lock_guard lock(mMutex);
+ mShouldCalculate = true;
+ mCondVar.notify_all();
+}
+
+void SpatializerPoseController::waitUntilCalculated() {
+ std::unique_lock lock(mMutex);
+ mCondVar.wait(lock, [this] { return mCalculated; });
+}
+
+void SpatializerPoseController::calculate_l() {
+ Pose3f headToStage;
+ HeadTrackingMode mode;
+ mProcessor->calculate(elapsedRealtimeNano());
+ headToStage = mProcessor->getHeadToStagePose();
+ mode = mProcessor->getActualMode();
+ mListener->onHeadToStagePose(headToStage);
+ if (!mActualMode.has_value() || mActualMode.value() != mode) {
+ mActualMode = mode;
+ mListener->onActualModeChange(mode);
+ }
+}
+
+void SpatializerPoseController::recenter() {
+ std::lock_guard lock(mMutex);
+ mProcessor->recenter();
+}
+
+void SpatializerPoseController::onPose(int64_t timestamp, int32_t sensor, const Pose3f& pose,
+ const std::optional<Twist3f>& twist) {
+ std::lock_guard lock(mMutex);
+ if (sensor == mHeadSensor) {
+ mProcessor->setWorldToHeadPose(timestamp, pose, twist.value_or(Twist3f()));
+ } else if (sensor == mScreenSensor) {
+ mProcessor->setWorldToScreenPose(timestamp, pose);
+ }
+}
+
+} // namespace android
diff --git a/services/audiopolicy/service/SpatializerPoseController.h b/services/audiopolicy/service/SpatializerPoseController.h
new file mode 100644
index 0000000..619dc7b
--- /dev/null
+++ b/services/audiopolicy/service/SpatializerPoseController.h
@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <chrono>
+#include <condition_variable>
+#include <limits>
+#include <memory>
+#include <mutex>
+#include <thread>
+
+#include <media/HeadTrackingProcessor.h>
+#include <media/SensorPoseProvider.h>
+
+namespace android {
+
+/**
+ * This class encapsulates the logic for pose processing, intended for driving a spatializer effect.
+ * This includes integration with the Sensor sub-system for retrieving sensor data, doing all the
+ * necessary processing, etc.
+ *
+ * Calculations happen on a dedicated thread and published to the client via the Listener interface.
+ * A calculation may be triggered in one of two ways:
+ * - By calling calculateAsync() - calculation will be kicked off in the background.
+ * - By setting a timeout in the ctor, a calculation will be triggered after the timeout elapsed
+ * from the last calculateAsync() call.
+ *
+ * This class is thread-safe. Callbacks are invoked with the lock held, so it is illegal to call
+ * into this module from the callbacks.
+ */
+class SpatializerPoseController : private media::SensorPoseProvider::Listener {
+ public:
+ /**
+ * Listener interface for getting pose and mode updates.
+ * Methods will always be invoked from a designated thread. Calling into the parent class from
+ * within the callbacks is disallowed (will result in a deadlock).
+ */
+ class Listener {
+ public:
+ virtual ~Listener() = default;
+
+ virtual void onHeadToStagePose(const media::Pose3f&) = 0;
+ virtual void onActualModeChange(media::HeadTrackingMode) = 0;
+ };
+
+ /**
+ * Ctor.
+ * sensorPeriod determines how often to receive updates from the sensors (input rate).
+ * maxUpdatePeriod determines how often to produce an output when calculateAsync() isn't
+ * invoked.
+ */
+ SpatializerPoseController(Listener* listener, std::chrono::microseconds sensorPeriod,
+ std::chrono::microseconds maxUpdatePeriod);
+
+ /** Dtor. */
+ ~SpatializerPoseController();
+
+ /**
+ * Set the sensor that is to be used for head-tracking.
+ * nullptr can be used to disable head-tracking.
+ */
+ void setHeadSensor(const ASensor* sensor);
+
+ /**
+ * Set the sensor that is to be used for screen-tracking.
+ * nullptr can be used to disable screen-tracking.
+ */
+ void setScreenSensor(const ASensor* sensor);
+
+ /** Sets the desired head-tracking mode. */
+ void setDesiredMode(media::HeadTrackingMode mode);
+
+ /**
+ * Set the screen-to-stage pose, used in all modes.
+ */
+ void setScreenToStagePose(const media::Pose3f& screenToStage);
+
+ /**
+ * Sets the display orientation.
+ * Orientation is expressed in the angle of rotation from the physical "up" side of the screen
+ * to the logical "up" side of the content displayed the screen. Counterclockwise angles, as
+ * viewed while facing the screen are positive.
+ */
+ void setDisplayOrientation(float physicalToLogicalAngle);
+
+ /**
+ * This causes the current poses for both the head and screen to be considered "center".
+ */
+ void recenter();
+
+ /**
+ * This call triggers the recalculation of the output and the invocation of the relevant
+ * callbacks. This call is async and the callbacks will be triggered shortly after.
+ */
+ void calculateAsync();
+
+ /**
+ * Blocks until calculation and invocation of the respective callbacks has happened at least
+ * once.
+ */
+ void waitUntilCalculated();
+
+ private:
+ mutable std::mutex mMutex;
+ Listener* const mListener;
+ const std::chrono::microseconds mSensorPeriod;
+ std::unique_ptr<media::SensorPoseProvider> mPoseProvider;
+ std::unique_ptr<media::HeadTrackingProcessor> mProcessor;
+ int32_t mHeadSensor = media::SensorPoseProvider::INVALID_HANDLE;
+ int32_t mScreenSensor = media::SensorPoseProvider::INVALID_HANDLE;
+ std::optional<media::HeadTrackingMode> mActualMode;
+ std::thread mThread;
+ std::condition_variable mCondVar;
+ bool mShouldCalculate = true;
+ bool mShouldExit = false;
+ bool mCalculated = false;
+
+ void onPose(int64_t timestamp, int32_t sensor, const media::Pose3f& pose,
+ const std::optional<media::Twist3f>& twist) override;
+
+ void calculate_l();
+};
+
+} // namespace android
diff --git a/services/audiopolicy/tests/AudioPolicyManagerTestClient.h b/services/audiopolicy/tests/AudioPolicyManagerTestClient.h
index f7b0565..84b40d2 100644
--- a/services/audiopolicy/tests/AudioPolicyManagerTestClient.h
+++ b/services/audiopolicy/tests/AudioPolicyManagerTestClient.h
@@ -37,7 +37,8 @@
status_t openOutput(audio_module_handle_t module,
audio_io_handle_t *output,
- audio_config_t * /*config*/,
+ audio_config_t * /*halConfig*/,
+ audio_config_base_t * /*mixerConfig*/,
const sp<DeviceDescriptorBase>& /*device*/,
uint32_t * /*latencyMs*/,
audio_output_flags_t /*flags*/) override {
diff --git a/services/audiopolicy/tests/AudioPolicyTestClient.h b/services/audiopolicy/tests/AudioPolicyTestClient.h
index 1384864..4e0735b 100644
--- a/services/audiopolicy/tests/AudioPolicyTestClient.h
+++ b/services/audiopolicy/tests/AudioPolicyTestClient.h
@@ -30,7 +30,8 @@
}
status_t openOutput(audio_module_handle_t /*module*/,
audio_io_handle_t* /*output*/,
- audio_config_t* /*config*/,
+ audio_config_t* /*halConfig*/,
+ audio_config_base_t* /*mixerConfig*/,
const sp<DeviceDescriptorBase>& /*device*/,
uint32_t* /*latencyMs*/,
audio_output_flags_t /*flags*/) override { return NO_INIT; }
diff --git a/services/mediametrics/Android.bp b/services/mediametrics/Android.bp
index 5989181..0351d2d 100644
--- a/services/mediametrics/Android.bp
+++ b/services/mediametrics/Android.bp
@@ -148,7 +148,8 @@
"statsd_mediaparser.cpp",
"statsd_nuplayer.cpp",
"statsd_recorder.cpp",
- "StringUtils.cpp"
+ "StringUtils.cpp",
+ "ValidateId.cpp",
],
proto: {
diff --git a/services/mediametrics/AudioAnalytics.cpp b/services/mediametrics/AudioAnalytics.cpp
index 45c9f56..270fe2f 100644
--- a/services/mediametrics/AudioAnalytics.cpp
+++ b/services/mediametrics/AudioAnalytics.cpp
@@ -29,6 +29,7 @@
#include "AudioTypes.h" // string to int conversions
#include "MediaMetricsService.h" // package info
#include "StringUtils.h"
+#include "ValidateId.h"
#define PROP_AUDIO_ANALYTICS_CLOUD_ENABLED "persist.audio.analytics.cloud.enabled"
@@ -563,7 +564,7 @@
const auto flagsForStats = types::lookup<types::INPUT_FLAG, short_enum_type_t>(flags);
const auto sourceForStats = types::lookup<types::SOURCE_TYPE, short_enum_type_t>(source);
// Android S
- const auto logSessionIdForStats = stringutils::sanitizeLogSessionId(logSessionId);
+ const auto logSessionIdForStats = ValidateId::get()->validateId(logSessionId);
LOG(LOG_LEVEL) << "key:" << key
<< " id:" << id
@@ -718,7 +719,7 @@
types::lookup<types::TRACK_TRAITS, short_enum_type_t>(traits);
const auto usageForStats = types::lookup<types::USAGE, short_enum_type_t>(usage);
// Android S
- const auto logSessionIdForStats = stringutils::sanitizeLogSessionId(logSessionId);
+ const auto logSessionIdForStats = ValidateId::get()->validateId(logSessionId);
LOG(LOG_LEVEL) << "key:" << key
<< " id:" << id
diff --git a/services/mediametrics/LruSet.h b/services/mediametrics/LruSet.h
new file mode 100644
index 0000000..1f0ab60
--- /dev/null
+++ b/services/mediametrics/LruSet.h
@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <list>
+#include <sstream>
+#include <unordered_map>
+
+namespace android::mediametrics {
+
+/**
+ * LruSet keeps a set of the last "Size" elements added or accessed.
+ *
+ * (Lru stands for least-recently-used eviction policy).
+ *
+ * Runs in O(1) time for add, remove, and check. Internally implemented
+ * with an unordered_map and a list. In order to remove elements,
+ * a list iterator is stored in the unordered_map
+ * (noting that std::list::erase() contractually
+ * does not affect iterators other than the one erased).
+ */
+
+template <typename T>
+class LruSet {
+ const size_t mMaxSize;
+ std::list<T> mAccessOrder; // front is the most recent, back is the oldest.
+ // item T with its access order iterator.
+ std::unordered_map<T, typename std::list<T>::iterator> mMap;
+
+public:
+ /**
+ * Constructs a LruSet which checks whether the element was
+ * accessed or added recently.
+ *
+ * The parameter maxSize is used to cap growth of LruSet;
+ * eviction is based on least recently used LRU.
+ * If maxSize is zero, the LruSet contains no elements
+ * and check() always returns false.
+ *
+ * \param maxSize the maximum number of elements that are tracked.
+ */
+ explicit LruSet(size_t maxSize) : mMaxSize(maxSize) {}
+
+ /**
+ * Returns the number of entries in the LruSet.
+ *
+ * This is a number between 0 and maxSize.
+ */
+ size_t size() const {
+ return mMap.size();
+ }
+
+ /** Clears the container contents. */
+ void clear() {
+ mMap.clear();
+ mAccessOrder.clear();
+ }
+
+ /** Returns a string dump of the last n entries. */
+ std::string dump(size_t n) const {
+ std::stringstream ss;
+ auto it = mAccessOrder.cbegin();
+ for (size_t i = 0; i < n && it != mAccessOrder.cend(); ++i) {
+ ss << *it++ << "\n";
+ }
+ return ss.str();
+ }
+
+ /** Adds a new item to the set. */
+ void add(const T& t) {
+ if (mMaxSize == 0) return;
+ auto it = mMap.find(t);
+ if (it != mMap.end()) { // already exists.
+ mAccessOrder.erase(it->second); // move-to-front on the chronologically ordered list.
+ } else if (mAccessOrder.size() >= mMaxSize) {
+ const T last = mAccessOrder.back();
+ mAccessOrder.pop_back();
+ mMap.erase(last);
+ }
+ mAccessOrder.push_front(t);
+ mMap[t] = mAccessOrder.begin();
+ }
+
+ /**
+ * Removes an item from the set.
+ *
+ * \param t item to be removed.
+ * \return false if the item doesn't exist.
+ */
+ bool remove(const T& t) {
+ auto it = mMap.find(t);
+ if (it == mMap.end()) return false;
+ mAccessOrder.erase(it->second);
+ mMap.erase(it);
+ return true;
+ }
+
+ /** Returns true if t is present (and moves the access order of t to the front). */
+ bool check(const T& t) { // not const, as it adjusts the least-recently-used order.
+ auto it = mMap.find(t);
+ if (it == mMap.end()) return false;
+ mAccessOrder.erase(it->second);
+ mAccessOrder.push_front(it->first);
+ it->second = mAccessOrder.begin();
+ return true;
+ }
+};
+
+} // namespace android::mediametrics
diff --git a/services/mediametrics/MediaMetricsService.cpp b/services/mediametrics/MediaMetricsService.cpp
index 1d64878..35e0ae4 100644
--- a/services/mediametrics/MediaMetricsService.cpp
+++ b/services/mediametrics/MediaMetricsService.cpp
@@ -19,6 +19,7 @@
#include <utils/Log.h>
#include "MediaMetricsService.h"
+#include "ValidateId.h"
#include "iface_statsd.h"
#include <pwd.h> //getpwuid
@@ -204,6 +205,15 @@
// now attach either the item or its dup to a const shared pointer
std::shared_ptr<const mediametrics::Item> sitem(release ? item : item->dup());
+ // register log session ids with singleton.
+ if (startsWith(item->getKey(), "metrics.manager")) {
+ std::string logSessionId;
+ if (item->get("logSessionId", &logSessionId)
+ && mediametrics::stringutils::isLogSessionId(logSessionId.c_str())) {
+ mediametrics::ValidateId::get()->registerId(logSessionId);
+ }
+ }
+
(void)mAudioAnalytics.submit(sitem, isTrusted);
(void)dump2Statsd(sitem, mStatsdLog); // failure should be logged in function.
@@ -309,6 +319,9 @@
result << "-- some lines may be truncated --\n";
}
+ result << "LogSessionId:\n"
+ << mediametrics::ValidateId::get()->dump();
+
// Dump the statsd atoms we sent out.
result << "Statsd atoms:\n"
<< mStatsdLog->dumpToString(" " /* prefix */,
diff --git a/services/mediametrics/ValidateId.cpp b/services/mediametrics/ValidateId.cpp
new file mode 100644
index 0000000..0cc8593
--- /dev/null
+++ b/services/mediametrics/ValidateId.cpp
@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "MediaMetricsService" // not ValidateId
+#include <utils/Log.h>
+
+#include "ValidateId.h"
+
+namespace android::mediametrics {
+
+std::string ValidateId::dump() const
+{
+ std::stringstream ss;
+ ss << "Entries:" << mIdSet.size() << " InvalidIds:" << mInvalidIds << "\n";
+ ss << mIdSet.dump(10);
+ return ss.str();
+}
+
+void ValidateId::registerId(const std::string& id)
+{
+ if (id.empty()) return;
+ if (!mediametrics::stringutils::isLogSessionId(id.c_str())) {
+ ALOGW("%s: rejecting malformed id %s", __func__, id.c_str());
+ return;
+ }
+ ALOGV("%s: registering %s", __func__, id.c_str());
+ mIdSet.add(id);
+}
+
+const std::string& ValidateId::validateId(const std::string& id)
+{
+ static const std::string empty{};
+ if (id.empty()) return empty;
+
+ // reject because the id is malformed
+ if (!mediametrics::stringutils::isLogSessionId(id.c_str())) {
+ ALOGW("%s: rejecting malformed id %s", __func__, id.c_str());
+ ++mInvalidIds;
+ return empty;
+ }
+
+ // reject because the id is unregistered
+ if (!mIdSet.check(id)) {
+ ALOGW("%s: rejecting unregistered id %s", __func__, id.c_str());
+ ++mInvalidIds;
+ return empty;
+ }
+ return id;
+}
+
+} // namespace android::mediametrics
diff --git a/services/mediametrics/ValidateId.h b/services/mediametrics/ValidateId.h
new file mode 100644
index 0000000..166b39a
--- /dev/null
+++ b/services/mediametrics/ValidateId.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "LruSet.h"
+#include "StringUtils.h"
+#include "Wrap.h"
+
+namespace android::mediametrics {
+
+/*
+ * ValidateId is used to check whether the log session id is properly formed
+ * and has been registered (i.e. from the Java MediaMetricsManagerService).
+ *
+ * The default memory window to track registered ids is set to SINGLETON_LRU_SET_SIZE.
+ *
+ * This class is not thread-safe, but the singleton returned by get() uses LockWrap<>
+ * to ensure thread-safety.
+ */
+class ValidateId {
+ mediametrics::LruSet<std::string> mIdSet;
+ size_t mInvalidIds = 0; // count invalid ids encountered.
+public:
+ /** Creates a ValidateId object with size memory window. */
+ explicit ValidateId(size_t size) : mIdSet{size} {}
+
+ /** Returns a string dump of recent contents and stats. */
+ std::string dump() const;
+
+ /**
+ * Registers the id string.
+ *
+ * If id string is malformed (not 16 Base64Url chars), it is ignored.
+ * Once registered, calling validateId() will return id (instead of the empty string).
+ * ValidateId may "forget" the id after not encountering it within the past N ids,
+ * where N is the size set in the constructor.
+ *
+ * param id string (from MediaMetricsManagerService).
+ */
+ void registerId(const std::string& id);
+
+ /**
+ * Returns the empty string if id string is malformed (not 16 Base64Url chars)
+ * or if id string has not been seen (in the recent size ids);
+ * otherwise it returns the same id parameter.
+ *
+ * \param id string (to be sent to statsd).
+ */
+ const std::string& validateId(const std::string& id);
+
+ /** Singleton set size */
+ static inline constexpr size_t SINGLETON_LRU_SET_SIZE = 2000;
+
+ using LockedValidateId = mediametrics::LockWrap<ValidateId>;
+ /**
+ * Returns a singleton locked ValidateId object that is thread-safe using LockWrap<>.
+ *
+ * The Singleton ValidateId object is created with size LRU_SET_SIZE (during first call).
+ */
+ static inline LockedValidateId& get() {
+ static LockedValidateId privateSet{SINGLETON_LRU_SET_SIZE};
+ return privateSet;
+ }
+};
+
+} // namespace android::mediametrics
diff --git a/services/mediametrics/statsd_audiorecord.cpp b/services/mediametrics/statsd_audiorecord.cpp
index 41efcaa..c53b6f3 100644
--- a/services/mediametrics/statsd_audiorecord.cpp
+++ b/services/mediametrics/statsd_audiorecord.cpp
@@ -32,7 +32,7 @@
#include <statslog.h>
#include "MediaMetricsService.h"
-#include "StringUtils.h"
+#include "ValidateId.h"
#include "frameworks/proto_logging/stats/message/mediametrics_message.pb.h"
#include "iface_statsd.h"
@@ -143,8 +143,7 @@
// log_session_id (string)
std::string logSessionId;
(void)item->getString("android.media.audiorecord.logSessionId", &logSessionId);
- const auto log_session_id =
- mediametrics::stringutils::sanitizeLogSessionId(logSessionId);
+ const auto log_session_id = mediametrics::ValidateId::get()->validateId(logSessionId);
android::util::BytesField bf_serialized( serialized.c_str(), serialized.size());
int result = android::util::stats_write(android::util::MEDIAMETRICS_AUDIORECORD_REPORTED,
diff --git a/services/mediametrics/statsd_audiotrack.cpp b/services/mediametrics/statsd_audiotrack.cpp
index 59627ae..707effd 100644
--- a/services/mediametrics/statsd_audiotrack.cpp
+++ b/services/mediametrics/statsd_audiotrack.cpp
@@ -32,7 +32,7 @@
#include <statslog.h>
#include "MediaMetricsService.h"
-#include "StringUtils.h"
+#include "ValidateId.h"
#include "frameworks/proto_logging/stats/message/mediametrics_message.pb.h"
#include "iface_statsd.h"
@@ -137,8 +137,7 @@
// log_session_id (string)
std::string logSessionId;
(void)item->getString("android.media.audiotrack.logSessionId", &logSessionId);
- const auto log_session_id =
- mediametrics::stringutils::sanitizeLogSessionId(logSessionId);
+ const auto log_session_id = mediametrics::ValidateId::get()->validateId(logSessionId);
android::util::BytesField bf_serialized( serialized.c_str(), serialized.size());
int result = android::util::stats_write(android::util::MEDIAMETRICS_AUDIOTRACK_REPORTED,
diff --git a/services/mediametrics/statsd_codec.cpp b/services/mediametrics/statsd_codec.cpp
index 46cbdc8..8581437 100644
--- a/services/mediametrics/statsd_codec.cpp
+++ b/services/mediametrics/statsd_codec.cpp
@@ -34,7 +34,7 @@
#include "cleaner.h"
#include "MediaMetricsService.h"
-#include "StringUtils.h"
+#include "ValidateId.h"
#include "frameworks/proto_logging/stats/message/mediametrics_message.pb.h"
#include "iface_statsd.h"
@@ -228,7 +228,7 @@
std::string sessionId;
if (item->getString("android.media.mediacodec.log-session-id", &sessionId)) {
- sessionId = mediametrics::stringutils::sanitizeLogSessionId(sessionId);
+ sessionId = mediametrics::ValidateId::get()->validateId(sessionId);
metrics_proto.set_log_session_id(sessionId);
}
AStatsEvent_writeString(event, codec.c_str());
diff --git a/services/mediametrics/statsd_extractor.cpp b/services/mediametrics/statsd_extractor.cpp
index bcf2e0a..a8bfeaa 100644
--- a/services/mediametrics/statsd_extractor.cpp
+++ b/services/mediametrics/statsd_extractor.cpp
@@ -32,7 +32,7 @@
#include <statslog.h>
#include "MediaMetricsService.h"
-#include "StringUtils.h"
+#include "ValidateId.h"
#include "frameworks/proto_logging/stats/message/mediametrics_message.pb.h"
#include "iface_statsd.h"
@@ -86,7 +86,7 @@
std::string log_session_id;
if (item->getString("android.media.mediaextractor.logSessionId", &log_session_id)) {
- log_session_id = mediametrics::stringutils::sanitizeLogSessionId(log_session_id);
+ log_session_id = mediametrics::ValidateId::get()->validateId(log_session_id);
metrics_proto.set_log_session_id(log_session_id);
}
diff --git a/services/mediametrics/statsd_mediaparser.cpp b/services/mediametrics/statsd_mediaparser.cpp
index 921b320..67ca874b 100644
--- a/services/mediametrics/statsd_mediaparser.cpp
+++ b/services/mediametrics/statsd_mediaparser.cpp
@@ -31,7 +31,7 @@
#include <statslog.h>
#include "MediaMetricsService.h"
-#include "StringUtils.h"
+#include "ValidateId.h"
#include "frameworks/proto_logging/stats/enums/stats/mediametrics/mediametrics.pb.h"
#include "iface_statsd.h"
@@ -81,7 +81,7 @@
std::string logSessionId;
item->getString("android.media.mediaparser.logSessionId", &logSessionId);
- logSessionId = mediametrics::stringutils::sanitizeLogSessionId(logSessionId);
+ logSessionId = mediametrics::ValidateId::get()->validateId(logSessionId);
int result = android::util::stats_write(android::util::MEDIAMETRICS_MEDIAPARSER_REPORTED,
timestamp_nanos,
diff --git a/services/mediametrics/statsd_recorder.cpp b/services/mediametrics/statsd_recorder.cpp
index b29ad73..5f54a68 100644
--- a/services/mediametrics/statsd_recorder.cpp
+++ b/services/mediametrics/statsd_recorder.cpp
@@ -32,7 +32,7 @@
#include <statslog.h>
#include "MediaMetricsService.h"
-#include "StringUtils.h"
+#include "ValidateId.h"
#include "frameworks/proto_logging/stats/message/mediametrics_message.pb.h"
#include "iface_statsd.h"
@@ -59,7 +59,7 @@
// string kRecorderLogSessionId = "android.media.mediarecorder.log-session-id";
std::string log_session_id;
if (item->getString("android.media.mediarecorder.log-session-id", &log_session_id)) {
- log_session_id = mediametrics::stringutils::sanitizeLogSessionId(log_session_id);
+ log_session_id = mediametrics::ValidateId::get()->validateId(log_session_id);
metrics_proto.set_log_session_id(log_session_id);
}
// string kRecorderAudioMime = "android.media.mediarecorder.audio.mime";
diff --git a/services/mediametrics/tests/mediametrics_tests.cpp b/services/mediametrics/tests/mediametrics_tests.cpp
index 2336d6f..69ec947 100644
--- a/services/mediametrics/tests/mediametrics_tests.cpp
+++ b/services/mediametrics/tests/mediametrics_tests.cpp
@@ -28,6 +28,7 @@
#include "AudioTypes.h"
#include "StringUtils.h"
+#include "ValidateId.h"
using namespace android;
@@ -1127,3 +1128,100 @@
validId2[3] = '!';
ASSERT_EQ("", mediametrics::stringutils::sanitizeLogSessionId(validId2));
}
+
+TEST(mediametrics_tests, LruSet) {
+ constexpr size_t LRU_SET_SIZE = 2;
+ mediametrics::LruSet<std::string> lruSet(LRU_SET_SIZE);
+
+ // test adding a couple strings.
+ lruSet.add("abc");
+ ASSERT_EQ(1u, lruSet.size());
+ ASSERT_TRUE(lruSet.check("abc"));
+ lruSet.add("def");
+ ASSERT_EQ(2u, lruSet.size());
+
+ // now adding the third string causes eviction of the oldest.
+ lruSet.add("ghi");
+ ASSERT_FALSE(lruSet.check("abc"));
+ ASSERT_TRUE(lruSet.check("ghi"));
+ ASSERT_TRUE(lruSet.check("def")); // "def" is most recent.
+ ASSERT_EQ(2u, lruSet.size()); // "abc" is correctly discarded.
+
+ // adding another string will evict the oldest.
+ lruSet.add("foo");
+ ASSERT_FALSE(lruSet.check("ghi")); // note: "ghi" discarded when "foo" added.
+ ASSERT_TRUE(lruSet.check("foo"));
+ ASSERT_TRUE(lruSet.check("def"));
+
+ // manual removing of a string works, too.
+ ASSERT_TRUE(lruSet.remove("def"));
+ ASSERT_FALSE(lruSet.check("def")); // we manually removed "def".
+ ASSERT_TRUE(lruSet.check("foo")); // "foo" is still there.
+ ASSERT_EQ(1u, lruSet.size());
+
+ // you can't remove a string that has not been added.
+ ASSERT_FALSE(lruSet.remove("bar")); // Note: "bar" doesn't exist, so remove returns false.
+ ASSERT_EQ(1u, lruSet.size());
+
+ lruSet.add("foo"); // adding "foo" (which already exists) doesn't change size.
+ ASSERT_EQ(1u, lruSet.size());
+ lruSet.add("bar"); // add "bar"
+ ASSERT_EQ(2u, lruSet.size());
+ lruSet.add("glorp"); // add "glorp" evicts "foo".
+ ASSERT_EQ(2u, lruSet.size());
+ ASSERT_TRUE(lruSet.check("bar"));
+ ASSERT_TRUE(lruSet.check("glorp"));
+ ASSERT_FALSE(lruSet.check("foo"));
+}
+
+TEST(mediametrics_tests, LruSet0) {
+ constexpr size_t LRU_SET_SIZE = 0;
+ mediametrics::LruSet<std::string> lruSet(LRU_SET_SIZE);
+
+ lruSet.add("a");
+ ASSERT_EQ(0u, lruSet.size());
+ ASSERT_FALSE(lruSet.check("a"));
+ ASSERT_FALSE(lruSet.remove("a")); // never added.
+ ASSERT_EQ(0u, lruSet.size());
+}
+
+// Returns a 16 Base64Url string representing the decimal representation of value
+// (with leading 0s) e.g. 0000000000000000, 0000000000000001, 0000000000000002, ...
+static std::string generateId(size_t value)
+{
+ char id[16 + 1]; // to be filled with 16 Base64Url chars (and zero termination)
+ char *sptr = id + 16; // start at the end.
+ *sptr-- = 0; // zero terminate.
+ // output the digits from least significant to most significant.
+ while (value) {
+ *sptr-- = value % 10;
+ value /= 10;
+ }
+ // add leading 0's
+ while (sptr > id) {
+ *sptr-- = '0';
+ }
+ return std::string(id);
+}
+
+TEST(mediametrics_tests, ValidateId) {
+ constexpr size_t LRU_SET_SIZE = 3;
+ constexpr size_t IDS = 10;
+ static_assert(IDS > LRU_SET_SIZE); // IDS must be greater than LRU_SET_SIZE.
+ mediametrics::ValidateId validateId(LRU_SET_SIZE);
+
+
+ // register IDs as integer strings counting from 0.
+ for (size_t i = 0; i < IDS; ++i) {
+ validateId.registerId(generateId(i));
+ }
+
+ // only the last LRU_SET_SIZE exist.
+ for (size_t i = 0; i < IDS - LRU_SET_SIZE; ++i) {
+ ASSERT_EQ("", validateId.validateId(generateId(i)));
+ }
+ for (size_t i = IDS - LRU_SET_SIZE; i < IDS; ++i) {
+ const std::string id = generateId(i);
+ ASSERT_EQ(id, validateId.validateId(id));
+ }
+}
diff --git a/services/oboeservice/AAudioServiceEndpoint.cpp b/services/oboeservice/AAudioServiceEndpoint.cpp
index 13dd3d3..340076e 100644
--- a/services/oboeservice/AAudioServiceEndpoint.cpp
+++ b/services/oboeservice/AAudioServiceEndpoint.cpp
@@ -59,6 +59,7 @@
result << " Device Id: " << getDeviceId() << "\n";
result << " Sample Rate: " << getSampleRate() << "\n";
result << " Channel Count: " << getSamplesPerFrame() << "\n";
+ result << " Channel Mask: 0x" << std::hex << getChannelMask() << std::dec << "\n";
result << " Format: " << getFormat() << "\n";
result << " Frames Per Burst: " << mFramesPerBurst << "\n";
result << " Usage: " << getUsage() << "\n";
@@ -164,6 +165,10 @@
configuration.getSamplesPerFrame() != getSamplesPerFrame()) {
return false;
}
+ if (configuration.getChannelMask() != AAUDIO_UNSPECIFIED &&
+ configuration.getChannelMask() != getChannelMask()) {
+ return false;
+ }
return true;
}
diff --git a/services/oboeservice/AAudioServiceEndpointMMAP.cpp b/services/oboeservice/AAudioServiceEndpointMMAP.cpp
index a08098c..35a0890 100644
--- a/services/oboeservice/AAudioServiceEndpointMMAP.cpp
+++ b/services/oboeservice/AAudioServiceEndpointMMAP.cpp
@@ -126,20 +126,15 @@
}
config.sample_rate = aaudioSampleRate;
- int32_t aaudioSamplesPerFrame = getSamplesPerFrame();
-
const aaudio_direction_t direction = getDirection();
+ config.channel_mask = AAudio_getChannelMaskForOpen(
+ getChannelMask(), getSamplesPerFrame(), direction == AAUDIO_DIRECTION_INPUT);
+
if (direction == AAUDIO_DIRECTION_OUTPUT) {
- config.channel_mask = (aaudioSamplesPerFrame == AAUDIO_UNSPECIFIED)
- ? AUDIO_CHANNEL_OUT_STEREO
- : audio_channel_out_mask_from_count(aaudioSamplesPerFrame);
mHardwareTimeOffsetNanos = OUTPUT_ESTIMATED_HARDWARE_OFFSET_NANOS; // frames at DAC later
} else if (direction == AAUDIO_DIRECTION_INPUT) {
- config.channel_mask = (aaudioSamplesPerFrame == AAUDIO_UNSPECIFIED)
- ? AUDIO_CHANNEL_IN_STEREO
- : audio_channel_in_mask_from_count(aaudioSamplesPerFrame);
mHardwareTimeOffsetNanos = INPUT_ESTIMATED_HARDWARE_OFFSET_NANOS; // frames at ADC earlier
} else {
@@ -225,9 +220,9 @@
}
// Get information about the stream and pass it back to the caller.
- setSamplesPerFrame((direction == AAUDIO_DIRECTION_OUTPUT)
- ? audio_channel_count_from_out_mask(config.channel_mask)
- : audio_channel_count_from_in_mask(config.channel_mask));
+ setChannelMask(AAudioConvert_androidToAAudioChannelMask(
+ config.channel_mask, getDirection() == AAUDIO_DIRECTION_INPUT,
+ AAudio_isChannelIndexMask(config.channel_mask)));
// AAudio creates a copy of this FD and retains ownership of the copy.
// Assume that AudioFlinger will close the original shared_memory_fd.
@@ -247,9 +242,9 @@
setFormat(config.format);
setSampleRate(config.sample_rate);
- ALOGD("%s() actual rate = %d, channels = %d"
- ", deviceId = %d, capacity = %d\n",
- __func__, getSampleRate(), getSamplesPerFrame(), deviceId, getBufferCapacity());
+ ALOGD("%s() actual rate = %d, channels = %d channelMask = %#x, deviceId = %d, capacity = %d\n",
+ __func__, getSampleRate(), getSamplesPerFrame(), getChannelMask(),
+ deviceId, getBufferCapacity());
ALOGD("%s() format = 0x%08x, frame size = %d, burst size = %d",
__func__, getFormat(), calculateBytesPerFrame(), mFramesPerBurst);
diff --git a/services/oboeservice/AAudioServiceEndpointShared.cpp b/services/oboeservice/AAudioServiceEndpointShared.cpp
index 5fbcadb..5af0a91 100644
--- a/services/oboeservice/AAudioServiceEndpointShared.cpp
+++ b/services/oboeservice/AAudioServiceEndpointShared.cpp
@@ -78,7 +78,7 @@
result = mStreamInternal->open(builder);
setSampleRate(mStreamInternal->getSampleRate());
- setSamplesPerFrame(mStreamInternal->getSamplesPerFrame());
+ setChannelMask(mStreamInternal->getChannelMask());
setDeviceId(mStreamInternal->getDeviceId());
setSessionId(mStreamInternal->getSessionId());
setFormat(AUDIO_FORMAT_PCM_FLOAT); // force for mixer
diff --git a/services/oboeservice/AAudioServiceStreamBase.cpp b/services/oboeservice/AAudioServiceStreamBase.cpp
index 34ddd4d..4ffc127 100644
--- a/services/oboeservice/AAudioServiceStreamBase.cpp
+++ b/services/oboeservice/AAudioServiceStreamBase.cpp
@@ -73,7 +73,8 @@
}
std::string AAudioServiceStreamBase::dumpHeader() {
- return std::string(" T Handle UId Port Run State Format Burst Chan Capacity");
+ return std::string(
+ " T Handle UId Port Run State Format Burst Chan Mask Capacity");
}
std::string AAudioServiceStreamBase::dump() const {
@@ -88,6 +89,7 @@
result << std::setw(7) << getFormat();
result << std::setw(6) << mFramesPerBurst;
result << std::setw(5) << getSamplesPerFrame();
+ result << std::setw(8) << std::hex << getChannelMask() << std::dec;
result << std::setw(9) << getBufferCapacity();
return result.str();
diff --git a/services/oboeservice/AAudioServiceStreamShared.cpp b/services/oboeservice/AAudioServiceStreamShared.cpp
index c665cda..ad06d97 100644
--- a/services/oboeservice/AAudioServiceStreamShared.cpp
+++ b/services/oboeservice/AAudioServiceStreamShared.cpp
@@ -164,11 +164,11 @@
goto error;
}
- setSamplesPerFrame(configurationInput.getSamplesPerFrame());
- if (getSamplesPerFrame() == AAUDIO_UNSPECIFIED) {
- setSamplesPerFrame(endpoint->getSamplesPerFrame());
+ setChannelMask(configurationInput.getChannelMask());
+ if (getChannelMask() == AAUDIO_UNSPECIFIED) {
+ setChannelMask(endpoint->getChannelMask());
} else if (getSamplesPerFrame() != endpoint->getSamplesPerFrame()) {
- ALOGD("%s() mSamplesPerFrame = %d, need %d",
+ ALOGD("%s() mSamplesPerFrame = %#x, need %#x",
__func__, getSamplesPerFrame(), endpoint->getSamplesPerFrame());
result = AAUDIO_ERROR_OUT_OF_RANGE;
goto error;
diff --git a/services/oboeservice/fuzzer/README.md b/services/oboeservice/fuzzer/README.md
index 00b85df..ae7af3eb 100644
--- a/services/oboeservice/fuzzer/README.md
+++ b/services/oboeservice/fuzzer/README.md
@@ -15,7 +15,7 @@
4. InService
5. DeviceId
6. SampleRate
-7. SamplesPerFrame
+7. ChannelMask
8. Direction
9. SharingMode
10. Usage
@@ -31,7 +31,7 @@
| `InService` | `bool` | Value obtained from FuzzedDataProvider |
| `DeviceId` | `INT32_MIN` to `INT32_MAX` | Value obtained from FuzzedDataProvider |
| `SampleRate` | `INT32_MIN` to `INT32_MAX` | Value obtained from FuzzedDataProvider |
-| `SamplesPerFrame` | `INT32_MIN` to `INT32_MAX` | Value obtained from FuzzedDataProvider |
+| `ChannelMask` | `AAUDIO_UNSPECIFIED`, `AAUDIO_CHANNEL_INDEX_MASK_1`, `AAUDIO_CHANNEL_INDEX_MASK_2`, `AAUDIO_CHANNEL_INDEX_MASK_3`, `AAUDIO_CHANNEL_INDEX_MASK_4`, `AAUDIO_CHANNEL_INDEX_MASK_5`, `AAUDIO_CHANNEL_INDEX_MASK_6`, `AAUDIO_CHANNEL_INDEX_MASK_7`, `AAUDIO_CHANNEL_INDEX_MASK_8`, `AAUDIO_CHANNEL_INDEX_MASK_9`, `AAUDIO_CHANNEL_INDEX_MASK_10`, `AAUDIO_CHANNEL_INDEX_MASK_11`, `AAUDIO_CHANNEL_INDEX_MASK_12`, `AAUDIO_CHANNEL_INDEX_MASK_13`, `AAUDIO_CHANNEL_INDEX_MASK_14`, `AAUDIO_CHANNEL_INDEX_MASK_15`, `AAUDIO_CHANNEL_INDEX_MASK_16`, `AAUDIO_CHANNEL_INDEX_MASK_17`, `AAUDIO_CHANNEL_INDEX_MASK_18`, `AAUDIO_CHANNEL_INDEX_MASK_19`, `AAUDIO_CHANNEL_INDEX_MASK_20`, `AAUDIO_CHANNEL_INDEX_MASK_21`, `AAUDIO_CHANNEL_INDEX_MASK_22`, `AAUDIO_CHANNEL_INDEX_MASK_23`, `AAUDIO_CHANNEL_INDEX_MASK_24`, `AAUDIO_CHANNEL_MONO`, `AAUDIO_CHANNEL_STEREO`, `AAUDIO_CHANNEL_FRONT_BACK`, `AAUDIO_CHANNEL_2POINT0POINT2`, `AAUDIO_CHANNEL_2POINT1POINT2`, `AAUDIO_CHANNEL_3POINT0POINT2`, `AAUDIO_CHANNEL_3POINT1POINT2`, `AAUDIO_CHANNEL_5POINT1`, `AAUDIO_CHANNEL_MONO`, `AAUDIO_CHANNEL_STEREO`, `AAUDIO_CHANNEL_2POINT1`, `AAUDIO_CHANNEL_TRI`, `AAUDIO_CHANNEL_TRI_BACK`, `AAUDIO_CHANNEL_3POINT1`, `AAUDIO_CHANNEL_2POINT0POINT2`, `AAUDIO_CHANNEL_2POINT1POINT2`, `AAUDIO_CHANNEL_3POINT0POINT2`, `AAUDIO_CHANNEL_3POINT1POINT2`, `AAUDIO_CHANNEL_QUAD`, `AAUDIO_CHANNEL_QUAD_SIDE`, `AAUDIO_CHANNEL_SURROUND`, `AAUDIO_CHANNEL_PENTA`, `AAUDIO_CHANNEL_5POINT1`, `AAUDIO_CHANNEL_5POINT1_SIDE`, `AAUDIO_CHANNEL_5POINT1POINT2`, `AAUDIO_CHANNEL_5POINT1POINT4`, `AAUDIO_CHANNEL_6POINT1`, `AAUDIO_CHANNEL_7POINT1`, `AAUDIO_CHANNEL_7POINT1POINT2`, `AAUDIO_CHANNEL_7POINT1POINT4`, `AAUDIO_CHANNEL_9POINT1POINT4`, `AAUDIO_CHANNEL_9POINT1POINT6` | Value obtained from FuzzedDataProvider |
| `Direction` | `AAUDIO_DIRECTION_OUTPUT`, `AAUDIO_DIRECTION_INPUT` | Value chosen from valid values by obtaining index from FuzzedDataProvider |
| `SharingMode` | `AAUDIO_SHARING_MODE_EXCLUSIVE`, `AAUDIO_SHARING_MODE_SHARED` | Value chosen from valid values by obtaining index from FuzzedDataProvider |
| `Usage` | `AAUDIO_USAGE_MEDIA`, `AAUDIO_USAGE_VOICE_COMMUNICATION`, `AAUDIO_USAGE_VOICE_COMMUNICATION_SIGNALLING`, `AAUDIO_USAGE_ALARM`, `AAUDIO_USAGE_NOTIFICATION`, `AAUDIO_USAGE_NOTIFICATION_RINGTONE`, `AAUDIO_USAGE_NOTIFICATION_EVENT`, `AAUDIO_USAGE_ASSISTANCE_ACCESSIBILITY`, `AAUDIO_USAGE_ASSISTANCE_NAVIGATION_GUIDANCE`, `AAUDIO_USAGE_ASSISTANCE_SONIFICATION`, `AAUDIO_USAGE_GAME`, `AAUDIO_USAGE_ASSISTANT`, `AAUDIO_SYSTEM_USAGE_EMERGENCY`, `AAUDIO_SYSTEM_USAGE_SAFETY`, `AAUDIO_SYSTEM_USAGE_VEHICLE_STATUS`, `AAUDIO_SYSTEM_USAGE_ANNOUNCEMENT` | Value chosen from valid values by obtaining index from FuzzedDataProvider |
diff --git a/services/oboeservice/fuzzer/oboeservice_fuzzer.cpp b/services/oboeservice/fuzzer/oboeservice_fuzzer.cpp
index 4bc661c..17e8d36 100644
--- a/services/oboeservice/fuzzer/oboeservice_fuzzer.cpp
+++ b/services/oboeservice/fuzzer/oboeservice_fuzzer.cpp
@@ -68,10 +68,71 @@
AAUDIO_INPUT_PRESET_UNPROCESSED, AAUDIO_INPUT_PRESET_VOICE_PERFORMANCE,
};
+aaudio_channel_mask_t kAAudioChannelMasks[] = {
+ AAUDIO_UNSPECIFIED,
+ AAUDIO_CHANNEL_INDEX_MASK_1,
+ AAUDIO_CHANNEL_INDEX_MASK_2,
+ AAUDIO_CHANNEL_INDEX_MASK_3,
+ AAUDIO_CHANNEL_INDEX_MASK_4,
+ AAUDIO_CHANNEL_INDEX_MASK_5,
+ AAUDIO_CHANNEL_INDEX_MASK_6,
+ AAUDIO_CHANNEL_INDEX_MASK_7,
+ AAUDIO_CHANNEL_INDEX_MASK_8,
+ AAUDIO_CHANNEL_INDEX_MASK_9,
+ AAUDIO_CHANNEL_INDEX_MASK_10,
+ AAUDIO_CHANNEL_INDEX_MASK_11,
+ AAUDIO_CHANNEL_INDEX_MASK_12,
+ AAUDIO_CHANNEL_INDEX_MASK_13,
+ AAUDIO_CHANNEL_INDEX_MASK_14,
+ AAUDIO_CHANNEL_INDEX_MASK_15,
+ AAUDIO_CHANNEL_INDEX_MASK_16,
+ AAUDIO_CHANNEL_INDEX_MASK_17,
+ AAUDIO_CHANNEL_INDEX_MASK_18,
+ AAUDIO_CHANNEL_INDEX_MASK_19,
+ AAUDIO_CHANNEL_INDEX_MASK_20,
+ AAUDIO_CHANNEL_INDEX_MASK_21,
+ AAUDIO_CHANNEL_INDEX_MASK_22,
+ AAUDIO_CHANNEL_INDEX_MASK_23,
+ AAUDIO_CHANNEL_INDEX_MASK_24,
+ AAUDIO_CHANNEL_MONO,
+ AAUDIO_CHANNEL_STEREO,
+ AAUDIO_CHANNEL_FRONT_BACK,
+ AAUDIO_CHANNEL_2POINT0POINT2,
+ AAUDIO_CHANNEL_2POINT1POINT2,
+ AAUDIO_CHANNEL_3POINT0POINT2,
+ AAUDIO_CHANNEL_3POINT1POINT2,
+ AAUDIO_CHANNEL_5POINT1,
+ AAUDIO_CHANNEL_MONO,
+ AAUDIO_CHANNEL_STEREO,
+ AAUDIO_CHANNEL_2POINT1,
+ AAUDIO_CHANNEL_TRI,
+ AAUDIO_CHANNEL_TRI_BACK,
+ AAUDIO_CHANNEL_3POINT1,
+ AAUDIO_CHANNEL_2POINT0POINT2,
+ AAUDIO_CHANNEL_2POINT1POINT2,
+ AAUDIO_CHANNEL_3POINT0POINT2,
+ AAUDIO_CHANNEL_3POINT1POINT2,
+ AAUDIO_CHANNEL_QUAD,
+ AAUDIO_CHANNEL_QUAD_SIDE,
+ AAUDIO_CHANNEL_SURROUND,
+ AAUDIO_CHANNEL_PENTA,
+ AAUDIO_CHANNEL_5POINT1,
+ AAUDIO_CHANNEL_5POINT1_SIDE,
+ AAUDIO_CHANNEL_5POINT1POINT2,
+ AAUDIO_CHANNEL_5POINT1POINT4,
+ AAUDIO_CHANNEL_6POINT1,
+ AAUDIO_CHANNEL_7POINT1,
+ AAUDIO_CHANNEL_7POINT1POINT2,
+ AAUDIO_CHANNEL_7POINT1POINT4,
+ AAUDIO_CHANNEL_9POINT1POINT4,
+ AAUDIO_CHANNEL_9POINT1POINT6,
+};
+
const size_t kNumAAudioFormats = std::size(kAAudioFormats);
const size_t kNumAAudioUsages = std::size(kAAudioUsages);
const size_t kNumAAudioContentTypes = std::size(kAAudioContentTypes);
const size_t kNumAAudioInputPresets = std::size(kAAudioInputPresets);
+const size_t kNumAAudioChannelMasks = std::size(kAAudioChannelMasks);
class FuzzAAudioClient : public virtual RefBase, public AAudioServiceInterface {
public:
@@ -305,7 +366,11 @@
request.getConfiguration().setDeviceId(fdp.ConsumeIntegral<int32_t>());
request.getConfiguration().setSampleRate(fdp.ConsumeIntegral<int32_t>());
- request.getConfiguration().setSamplesPerFrame(fdp.ConsumeIntegral<int32_t>());
+ request.getConfiguration().setChannelMask((aaudio_channel_mask_t)(
+ fdp.ConsumeBool()
+ ? fdp.ConsumeIntegral<int32_t>()
+ : kAAudioChannelMasks[fdp.ConsumeIntegralInRange<int32_t>(
+ 0, kNumAAudioChannelMasks - 1)]));
request.getConfiguration().setDirection(
fdp.ConsumeBool() ? fdp.ConsumeIntegral<int32_t>()
: (fdp.ConsumeBool() ? AAUDIO_DIRECTION_OUTPUT : AAUDIO_DIRECTION_INPUT));
