audio policy: add Spatializer effect control
Add APIs to discover support for audio spatialization function and control a
spatializer output mixer and effect creation to Audio Policy Service.
When supported, a system service can retrieve a ISpatializer
interface to query supported features and set a spatialization mode.
When spatialization is enabled, the corresponding specialized output mixer is
created and the effect applied.
The audio policy manager will attach clients which audio attributes and
format qualifying for spatialization to the specialized output mixer.
Bug: 188502620
Test: make
Change-Id: Ie2734fb9bb3ef9b945431c6c9bd110b1434a79cd
Merged-In: Ie2734fb9bb3ef9b945431c6c9bd110b1434a79cd
diff --git a/media/libaudioclient/Android.bp b/media/libaudioclient/Android.bp
index 9c307ff..4ab9b28 100644
--- a/media/libaudioclient/Android.bp
+++ b/media/libaudioclient/Android.bp
@@ -388,7 +388,9 @@
"aidl/android/media/AudioProductStrategy.aidl",
"aidl/android/media/AudioVolumeGroup.aidl",
"aidl/android/media/DeviceRole.aidl",
+ "aidl/android/media/HeadTrackingMode.aidl",
"aidl/android/media/SoundTriggerSession.aidl",
+ "aidl/android/media/SpatializationLevel.aidl",
],
imports: [
"audio_common-aidl",
@@ -459,10 +461,13 @@
srcs: [
"aidl/android/media/GetInputForAttrResponse.aidl",
"aidl/android/media/GetOutputForAttrResponse.aidl",
+ "aidl/android/media/GetSpatializerResponse.aidl",
"aidl/android/media/Int.aidl",
"aidl/android/media/RecordClientInfo.aidl",
"aidl/android/media/IAudioPolicyService.aidl",
"aidl/android/media/IAudioPolicyServiceClient.aidl",
+ "aidl/android/media/INativeSpatializerCallback.aidl",
+ "aidl/android/media/ISpatializer.aidl",
],
imports: [
"audio_common-aidl",
diff --git a/media/libaudioclient/AudioSystem.cpp b/media/libaudioclient/AudioSystem.cpp
index 792be31..c7967e5 100644
--- a/media/libaudioclient/AudioSystem.cpp
+++ b/media/libaudioclient/AudioSystem.cpp
@@ -2240,6 +2240,47 @@
return OK;
}
+status_t AudioSystem::getSpatializer(const sp<media::INativeSpatializerCallback>& callback,
+ sp<media::ISpatializer>* spatializer) {
+ const sp<IAudioPolicyService>& aps = AudioSystem::get_audio_policy_service();
+ if (spatializer == nullptr) {
+ return BAD_VALUE;
+ }
+ if (aps == 0) {
+ return PERMISSION_DENIED;
+ }
+ media::GetSpatializerResponse response;
+ RETURN_STATUS_IF_ERROR(statusTFromBinderStatus(
+ aps->getSpatializer(callback, &response)));
+
+ *spatializer = response.spatializer;
+ return OK;
+}
+
+status_t AudioSystem::canBeSpatialized(const audio_attributes_t *attr,
+ const audio_config_t *config,
+ const AudioDeviceTypeAddrVector &devices,
+ bool *canBeSpatialized) {
+ const sp<IAudioPolicyService>& aps = AudioSystem::get_audio_policy_service();
+ if (aps == 0) {
+ return PERMISSION_DENIED;
+ }
+ audio_attributes_t attributes = attr != nullptr ? *attr : AUDIO_ATTRIBUTES_INITIALIZER;
+ audio_config_t configuration = config != nullptr ? *config : AUDIO_CONFIG_INITIALIZER;
+
+ std::optional<media::AudioAttributesInternal> attrAidl = VALUE_OR_RETURN_STATUS(
+ legacy2aidl_audio_attributes_t_AudioAttributesInternal(attributes));
+ std::optional<media::AudioConfig> configAidl = VALUE_OR_RETURN_STATUS(
+ legacy2aidl_audio_config_t_AudioConfig(configuration));
+ std::vector<media::AudioDevice> devicesAidl = VALUE_OR_RETURN_STATUS(
+ convertContainer<std::vector<media::AudioDevice>>(devices,
+ legacy2aidl_AudioDeviceTypeAddress));
+ RETURN_STATUS_IF_ERROR(statusTFromBinderStatus(
+ aps->canBeSpatialized(attrAidl, configAidl, devicesAidl, canBeSpatialized)));
+ return OK;
+}
+
+
class CaptureStateListenerImpl : public media::BnCaptureStateListener,
public IBinder::DeathRecipient {
public:
diff --git a/media/libaudioclient/aidl/android/media/GetSpatializerResponse.aidl b/media/libaudioclient/aidl/android/media/GetSpatializerResponse.aidl
new file mode 100644
index 0000000..25115ac
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/GetSpatializerResponse.aidl
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+import android.media.ISpatializer;
+
+/**
+ * Used as a return value for IAudioPolicyService.getSpatializer() method
+ * {@hide}
+ */
+ parcelable GetSpatializerResponse {
+ /* The ISpatializer interface if successful, null if not */
+ @nullable ISpatializer spatializer;
+}
diff --git a/media/libaudioclient/aidl/android/media/HeadTrackingMode.aidl b/media/libaudioclient/aidl/android/media/HeadTrackingMode.aidl
new file mode 100644
index 0000000..d6cf410
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/HeadTrackingMode.aidl
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+
+/**
+ * The head tracking mode supported by the spatializer stage effect implementation.
+ * Used by methods of the ISpatializer interface.
+ * {@hide}
+ */
+@Backing(type="byte")
+enum HeadTrackingMode {
+ /** Head tracking is active in a mode not listed below (forward compatibility) */
+ OTHER = 0,
+ /** Head tracking is disabled */
+ DISABLED = 1,
+ /** Head tracking is performed relative to the real work environment */
+ RELATIVE_WORLD = 2,
+ /** Head tracking is performed relative to the device's screen */
+ RELATIVE_SCREEN = 3,
+}
diff --git a/media/libaudioclient/aidl/android/media/IAudioPolicyService.aidl b/media/libaudioclient/aidl/android/media/IAudioPolicyService.aidl
index 4c3955a..5f0a1de 100644
--- a/media/libaudioclient/aidl/android/media/IAudioPolicyService.aidl
+++ b/media/libaudioclient/aidl/android/media/IAudioPolicyService.aidl
@@ -47,8 +47,10 @@
import android.media.EffectDescriptor;
import android.media.GetInputForAttrResponse;
import android.media.GetOutputForAttrResponse;
+import android.media.GetSpatializerResponse;
import android.media.IAudioPolicyServiceClient;
import android.media.ICaptureStateListener;
+import android.media.INativeSpatializerCallback;
import android.media.Int;
import android.media.SoundTriggerSession;
@@ -348,4 +350,29 @@
DeviceRole role);
boolean registerSoundTriggerCaptureStateListener(ICaptureStateListener listener);
+
+ /** If a spatializer stage effect is present on the platform, this will return an
+ * ISpatializer interface (see GetSpatializerResponse,aidl) to control this
+ * feature.
+ * If no spatializer stage is present, a null interface is returned.
+ * The INativeSpatializerCallback passed must not be null.
+ * Only one ISpatializer interface can exist at a given time. The native audio policy
+ * service will reject the request if an interface was already acquired and previous owner
+ * did not die or call ISpatializer.release().
+ */
+ GetSpatializerResponse getSpatializer(INativeSpatializerCallback callback);
+
+ /** Queries if some kind of spatialization will be performed if the audio playback context
+ * described by the provided arguments is present.
+ * The context is made of:
+ * - The audio attributes describing the playback use case.
+ * - The audio configuration describing the audio format, channels, sampling rate...
+ * - The devices describing the sink audio device selected for playback.
+ * All arguments are optional and only the specified arguments are used to match against
+ * supported criteria. For instance, supplying no argument will tell if spatialization is
+ * supported or not in general.
+ */
+ boolean canBeSpatialized(in @nullable AudioAttributesInternal attr,
+ in @nullable AudioConfig config,
+ in AudioDevice[] devices);
}
diff --git a/media/libaudioclient/aidl/android/media/INativeSpatializerCallback.aidl b/media/libaudioclient/aidl/android/media/INativeSpatializerCallback.aidl
new file mode 100644
index 0000000..f34df05
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/INativeSpatializerCallback.aidl
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+import android.media.SpatializationLevel;
+
+/**
+ * The INativeSpatializerCallback interface is a callback associated to the
+ * ISpatializer interface. The callback is used by the spatializer stage
+ * implementation in native audio server to communicate stage changes to the
+ * client controlling the spatializer with the ISpatializer interface.
+ * {@hide}
+ */
+interface INativeSpatializerCallback {
+ /** Called when the spatialization level applied by the vitualizer stage changes
+ * (e.g. when the spatializer is enabled or disabled)
+ */
+ oneway void onLevelChanged(SpatializationLevel level);
+}
diff --git a/media/libaudioclient/aidl/android/media/ISpatializer.aidl b/media/libaudioclient/aidl/android/media/ISpatializer.aidl
new file mode 100644
index 0000000..212d8fe
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/ISpatializer.aidl
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+import android.media.HeadTrackingMode;
+import android.media.SpatializationLevel;
+
+/**
+ * The ISpatializer interface is used to control the native audio service implementation
+ * of the spatializer stage with headtracking when present on a platform.
+ * It is intended for exclusive use by the java AudioService running in system_server.
+ * It provides APIs to discover the feature availability and options as well as control and report
+ * the active state and modes of the spatializer and head tracking effect.
+ * {@hide}
+ */
+interface ISpatializer {
+ /** Releases a ISpatializer interface previously acquired. */
+ void release();
+
+ /** Reports the list of supported spatialization levels (see SpatializationLevel.aidl).
+ * The list should never be empty if an ISpatializer interface was successfully
+ * retrieved with IAudioPolicyService.getSpatializer().
+ */
+ SpatializationLevel[] getSupportedLevels();
+
+ /** Selects the desired spatialization level (see SpatializationLevel.aidl). Selecting a level
+ * different from SpatializationLevel.NONE with create the specialized multichannel output
+ * mixer, create and enable the spatializer effect and let the audio policy attach eligible
+ * AudioTrack to this output stream.
+ */
+ void setLevel(SpatializationLevel level);
+
+ /** Gets the selected spatialization level (see SpatializationLevel.aidl) */
+ SpatializationLevel getLevel();
+
+ /** Reports the list of supported head tracking modes (see HeadTrackingMode.aidl). The list can
+ * be empty if the spatializer implementation does not support head tracking or if no
+ * head tracking device is connected.
+ */
+ HeadTrackingMode[] getSupportedHeadTrackingModes();
+
+ /** Selects the desired head tracking mode (see HeadTrackingMode.aidl) */
+ void setDesiredHeadTrackingMode(HeadTrackingMode mode);
+
+ /** Gets the actual head tracking mode. Can be different from the desired mode if conditions to
+ * enable the desired mode are not met (e.g if the head tracking device was removed)
+ */
+ HeadTrackingMode getActualHeadTrackingMode();
+
+ /** Reset the head tracking algorithm to consider current head pose as neutral */
+ void recenterHeadtracker();
+ /** Set the screen to stage transform to use by the head tracking algorithm */
+ void setGlobalTransform(in float[] screenToStage);
+}
diff --git a/media/libaudioclient/aidl/android/media/SpatializationLevel.aidl b/media/libaudioclient/aidl/android/media/SpatializationLevel.aidl
new file mode 100644
index 0000000..cef42bb
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/SpatializationLevel.aidl
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+/**
+ * The spatialization level or mode supported by the spatializer stage effect implementation.
+ * Used by methods of the ISpatializer interface.
+ * {@hide}
+ */
+@Backing(type="byte")
+enum SpatializationLevel {
+ /** Spatialization is disabled. */
+ NONE = 0,
+ /** The spatializer accepts audio with positional multichannel masks (e.g 5.1). */
+ SPATIALIZER_MULTICHANNEL = 1,
+ /** The spatializer accepts audio made of a channel bed of positional multichannels (e.g 5.1)
+ * and audio objects positioned independently via meta data.
+ */
+ SPATIALIZER_MCHAN_BED_PLUS_OBJECTS = 2,
+}
diff --git a/media/libaudioclient/include/media/AudioSystem.h b/media/libaudioclient/include/media/AudioSystem.h
index 9187de1..869bd6e 100644
--- a/media/libaudioclient/include/media/AudioSystem.h
+++ b/media/libaudioclient/include/media/AudioSystem.h
@@ -22,6 +22,8 @@
#include <android/media/AudioVibratorInfo.h>
#include <android/media/BnAudioFlingerClient.h>
#include <android/media/BnAudioPolicyServiceClient.h>
+#include <android/media/INativeSpatializerCallback.h>
+#include <android/media/ISpatializer.h>
#include <android/content/AttributionSourceState.h>
#include <media/AidlConversionUtil.h>
#include <media/AudioDeviceTypeAddr.h>
@@ -488,6 +490,49 @@
static status_t getDeviceForStrategy(product_strategy_t strategy,
AudioDeviceTypeAddr &device);
+
+ /**
+ * If a spatializer stage effect is present on the platform, this will return an
+ * ISpatializer interface to control this feature.
+ * If no spatializer stage is present, a null interface is returned.
+ * The INativeSpatializerCallback passed must not be null.
+ * Only one ISpatializer interface can exist at a given time. The native audio policy
+ * service will reject the request if an interface was already acquired and previous owner
+ * did not die or call ISpatializer.release().
+ * @param callback in: the callback to receive state updates if the ISpatializer
+ * interface is acquired.
+ * @param spatializer out: the ISpatializer interface made available to control the
+ * platform spatializer
+ * @return NO_ERROR in case of success, DEAD_OBJECT, NO_INIT, PERMISSION_DENIED, BAD_VALUE
+ * in case of error.
+ */
+ static status_t getSpatializer(const sp<media::INativeSpatializerCallback>& callback,
+ sp<media::ISpatializer>* spatializer);
+
+ /**
+ * Queries if some kind of spatialization will be performed if the audio playback context
+ * described by the provided arguments is present.
+ * The context is made of:
+ * - The audio attributes describing the playback use case.
+ * - The audio configuration describing the audio format, channels, sampling rate ...
+ * - The devices describing the sink audio device selected for playback.
+ * All arguments are optional and only the specified arguments are used to match against
+ * supported criteria. For instance, supplying no argument will tell if spatialization is
+ * supported or not in general.
+ * @param attr audio attributes describing the playback use case
+ * @param config audio configuration describing the audio format, channels, sampling rate...
+ * @param devices the sink audio device selected for playback
+ * @param canBeSpatialized out: true if spatialization is enabled for this context,
+ * false otherwise
+ * @return NO_ERROR in case of success, DEAD_OBJECT, NO_INIT, BAD_VALUE
+ * in case of error.
+ */
+ static status_t canBeSpatialized(const audio_attributes_t *attr,
+ const audio_config_t *config,
+ const AudioDeviceTypeAddrVector &devices,
+ bool *canBeSpatialized);
+
+
// A listener for capture state changes.
class CaptureStateListener : public RefBase {
public:
@@ -500,11 +545,11 @@
virtual ~CaptureStateListener() = default;
};
- // Regiseters a listener for sound trigger capture state changes.
+ // Registers a listener for sound trigger capture state changes.
// There may only be one such listener registered at any point.
- // The listener onStateChanged() method will be invoked sychronously from
+ // The listener onStateChanged() method will be invoked synchronously from
// this call with the initial value.
- // The listener onServiceDied() method will be invoked sychronously from
+ // The listener onServiceDied() method will be invoked synchronously from
// this call if initial attempt to register failed.
// If the audio policy service cannot be reached, this method will return
// PERMISSION_DENIED and will not invoke the callback, otherwise, it will