blob: 233a48444ed57249dd51d55094cd0aa921cd5538 [file] [log] [blame]
Eric Laurent6d607012021-07-05 11:54:40 +02001/*
2**
3** Copyright 2021, The Android Open Source Project
4**
5** Licensed under the Apache License, Version 2.0 (the "License");
6** you may not use this file except in compliance with the License.
7** You may obtain a copy of the License at
8**
9** http://www.apache.org/licenses/LICENSE-2.0
10**
11** Unless required by applicable law or agreed to in writing, software
12** distributed under the License is distributed on an "AS IS" BASIS,
13** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14** See the License for the specific language governing permissions and
15** limitations under the License.
16*/
17
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +000018#include <string>
Eric Laurent6d607012021-07-05 11:54:40 +020019#define LOG_TAG "Spatializer"
20//#define LOG_NDEBUG 0
21#include <utils/Log.h>
22
Andy Hung2490b2d2023-03-09 20:45:36 -080023#include <algorithm>
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +000024#include <inttypes.h>
Eric Laurent6d607012021-07-05 11:54:40 +020025#include <limits.h>
26#include <stdint.h>
27#include <sys/types.h>
28
29#include <android/content/AttributionSourceState.h>
Eric Laurentbe21a942023-11-14 16:05:00 +010030#include <android/sysprop/BluetoothProperties.sysprop.h>
Eric Laurent6d607012021-07-05 11:54:40 +020031#include <audio_utils/fixedfft.h>
Eric Laurentbe21a942023-11-14 16:05:00 +010032#include <com_android_media_audio.h>
Eric Laurent6d607012021-07-05 11:54:40 +020033#include <cutils/bitops.h>
Nikhil Bhanu8f4ea772024-01-31 17:15:52 -080034#include <cutils/properties.h>
Eric Laurent2be8b292021-08-23 09:44:33 -070035#include <hardware/sensors.h>
Eric Laurent8a4259f2021-09-14 16:04:00 +020036#include <media/stagefright/foundation/AHandler.h>
37#include <media/stagefright/foundation/AMessage.h>
Andy Hunga461a002022-05-17 10:36:02 -070038#include <media/MediaMetricsItem.h>
Andy Hung560addd2023-01-30 11:58:44 -080039#include <media/QuaternionUtil.h>
Eric Laurent8a4259f2021-09-14 16:04:00 +020040#include <media/ShmemCompat.h>
Andy Hung41ccf7f2022-12-14 14:25:49 -080041#include <mediautils/SchedulingPolicyService.h>
Eric Laurent6d607012021-07-05 11:54:40 +020042#include <mediautils/ServiceUtilities.h>
43#include <utils/Thread.h>
44
45#include "Spatializer.h"
46
47namespace android {
48
Eric Laurent6d607012021-07-05 11:54:40 +020049using aidl_utils::binderStatusFromStatusT;
Shunkai Yao49bc61f2023-10-10 19:31:10 +000050using aidl_utils::statusTFromBinderStatus;
Eric Laurent6d607012021-07-05 11:54:40 +020051using android::content::AttributionSourceState;
52using binder::Status;
Shunkai Yao2ee06f62024-02-16 22:00:22 +000053using internal::ToString;
Eric Laurent2be8b292021-08-23 09:44:33 -070054using media::HeadTrackingMode;
55using media::Pose3f;
Eric Laurent2be8b292021-08-23 09:44:33 -070056using media::SensorPoseProvider;
Shunkai Yao49bc61f2023-10-10 19:31:10 +000057using media::audio::common::HeadTracking;
58using media::audio::common::Spatialization;
Eric Laurent2be8b292021-08-23 09:44:33 -070059
Eric Laurent2be8b292021-08-23 09:44:33 -070060using namespace std::chrono_literals;
Eric Laurent6d607012021-07-05 11:54:40 +020061
62#define VALUE_OR_RETURN_BINDER_STATUS(x) \
63 ({ auto _tmp = (x); \
64 if (!_tmp.ok()) return aidl_utils::binderStatusFromStatusT(_tmp.error()); \
65 std::move(_tmp.value()); })
66
Andy Hung4e2547c2022-08-29 14:14:58 -070067static audio_channel_mask_t getMaxChannelMask(
68 const std::vector<audio_channel_mask_t>& masks, size_t channelLimit = SIZE_MAX) {
Andy Hunga461a002022-05-17 10:36:02 -070069 uint32_t maxCount = 0;
70 audio_channel_mask_t maxMask = AUDIO_CHANNEL_NONE;
71 for (auto mask : masks) {
72 const size_t count = audio_channel_count_from_out_mask(mask);
Andy Hung4e2547c2022-08-29 14:14:58 -070073 if (count > channelLimit) continue; // ignore masks greater than channelLimit
Andy Hunga461a002022-05-17 10:36:02 -070074 if (count > maxCount) {
75 maxMask = mask;
76 maxCount = count;
77 }
78 }
79 return maxMask;
80}
81
Andy Hung560addd2023-01-30 11:58:44 -080082static std::vector<float> recordFromTranslationRotationVector(
83 const std::vector<float>& trVector) {
84 auto headToStageOpt = Pose3f::fromVector(trVector);
85 if (!headToStageOpt) return {};
86
87 const auto stageToHead = headToStageOpt.value().inverse();
88 const auto stageToHeadTranslation = stageToHead.translation();
Andy Hunga367cb22023-01-30 11:58:44 -080089 constexpr float RAD_TO_DEGREE = 180.f / M_PI;
90 std::vector<float> record{
Andy Hung560addd2023-01-30 11:58:44 -080091 stageToHeadTranslation[0], stageToHeadTranslation[1], stageToHeadTranslation[2],
92 0.f, 0.f, 0.f};
93 media::quaternionToAngles(stageToHead.rotation(), &record[3], &record[4], &record[5]);
94 record[3] *= RAD_TO_DEGREE;
95 record[4] *= RAD_TO_DEGREE;
96 record[5] *= RAD_TO_DEGREE;
Andy Hunga367cb22023-01-30 11:58:44 -080097 return record;
98}
99
Andy Hung2490b2d2023-03-09 20:45:36 -0800100template<typename T>
101static constexpr const T& safe_clamp(const T& value, const T& low, const T& high) {
102 if constexpr (std::is_floating_point_v<T>) {
103 return value != value /* constexpr isnan */
104 ? low : std::clamp(value, low, high);
105 } else /* constexpr */ {
106 return std::clamp(value, low, high);
107 }
108}
109
Eric Laurent6d607012021-07-05 11:54:40 +0200110// ---------------------------------------------------------------------------
111
Eric Laurent8a4259f2021-09-14 16:04:00 +0200112class Spatializer::EngineCallbackHandler : public AHandler {
113public:
114 EngineCallbackHandler(wp<Spatializer> spatializer)
115 : mSpatializer(spatializer) {
116 }
117
118 enum {
119 // Device state callbacks
120 kWhatOnFramesProcessed, // AudioEffect::EVENT_FRAMES_PROCESSED
121 kWhatOnHeadToStagePose, // SpatializerPoseController::Listener::onHeadToStagePose
122 kWhatOnActualModeChange, // SpatializerPoseController::Listener::onActualModeChange
Eric Laurent9c04de92022-07-20 13:49:47 +0200123 kWhatOnLatencyModesChanged, // Spatializer::onSupportedLatencyModesChanged
Eric Laurent8a4259f2021-09-14 16:04:00 +0200124 };
125 static constexpr const char *kNumFramesKey = "numFrames";
126 static constexpr const char *kModeKey = "mode";
127 static constexpr const char *kTranslation0Key = "translation0";
128 static constexpr const char *kTranslation1Key = "translation1";
129 static constexpr const char *kTranslation2Key = "translation2";
130 static constexpr const char *kRotation0Key = "rotation0";
131 static constexpr const char *kRotation1Key = "rotation1";
132 static constexpr const char *kRotation2Key = "rotation2";
Eric Laurent9c04de92022-07-20 13:49:47 +0200133 static constexpr const char *kLatencyModesKey = "latencyModes";
134
135 class LatencyModes : public RefBase {
136 public:
137 LatencyModes(audio_io_handle_t output,
138 const std::vector<audio_latency_mode_t>& latencyModes)
139 : mOutput(output), mLatencyModes(latencyModes) {}
140 ~LatencyModes() = default;
141
142 audio_io_handle_t mOutput;
143 std::vector<audio_latency_mode_t> mLatencyModes;
144 };
Eric Laurent8a4259f2021-09-14 16:04:00 +0200145
146 void onMessageReceived(const sp<AMessage> &msg) override {
Andy Hung41ccf7f2022-12-14 14:25:49 -0800147 // No ALooper method to get the tid so update
148 // Spatializer priority on the first message received.
149 std::call_once(mPrioritySetFlag, [](){
150 const pid_t pid = getpid();
151 const pid_t tid = gettid();
152 (void)requestSpatializerPriority(pid, tid);
153 });
154
Eric Laurent9c04de92022-07-20 13:49:47 +0200155 sp<Spatializer> spatializer = mSpatializer.promote();
156 if (spatializer == nullptr) {
157 ALOGW("%s: Cannot promote spatializer", __func__);
158 return;
159 }
Eric Laurent8a4259f2021-09-14 16:04:00 +0200160 switch (msg->what()) {
161 case kWhatOnFramesProcessed: {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200162 int numFrames;
163 if (!msg->findInt32(kNumFramesKey, &numFrames)) {
164 ALOGE("%s: Cannot find num frames!", __func__);
165 return;
166 }
167 if (numFrames > 0) {
168 spatializer->calculateHeadPose();
169 }
170 } break;
171 case kWhatOnHeadToStagePose: {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200172 std::vector<float> headToStage(sHeadPoseKeys.size());
173 for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
174 if (!msg->findFloat(sHeadPoseKeys[i], &headToStage[i])) {
175 ALOGE("%s: Cannot find kTranslation0Key!", __func__);
176 return;
177 }
178 }
179 spatializer->onHeadToStagePoseMsg(headToStage);
180 } break;
181 case kWhatOnActualModeChange: {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200182 int mode;
Eric Laurent9c04de92022-07-20 13:49:47 +0200183 if (!msg->findInt32(kModeKey, &mode)) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200184 ALOGE("%s: Cannot find actualMode!", __func__);
185 return;
186 }
187 spatializer->onActualModeChangeMsg(static_cast<HeadTrackingMode>(mode));
188 } break;
Eric Laurent9c04de92022-07-20 13:49:47 +0200189
190 case kWhatOnLatencyModesChanged: {
191 sp<RefBase> object;
192 if (!msg->findObject(kLatencyModesKey, &object)) {
193 ALOGE("%s: Cannot find latency modes!", __func__);
194 return;
195 }
196 sp<LatencyModes> latencyModes = static_cast<LatencyModes*>(object.get());
197 spatializer->onSupportedLatencyModesChangedMsg(
198 latencyModes->mOutput, std::move(latencyModes->mLatencyModes));
199 } break;
200
Eric Laurent8a4259f2021-09-14 16:04:00 +0200201 default:
202 LOG_ALWAYS_FATAL("Invalid callback message %d", msg->what());
203 }
204 }
205private:
206 wp<Spatializer> mSpatializer;
Andy Hung41ccf7f2022-12-14 14:25:49 -0800207 std::once_flag mPrioritySetFlag;
Eric Laurent8a4259f2021-09-14 16:04:00 +0200208};
209
210const std::vector<const char *> Spatializer::sHeadPoseKeys = {
211 Spatializer::EngineCallbackHandler::kTranslation0Key,
212 Spatializer::EngineCallbackHandler::kTranslation1Key,
213 Spatializer::EngineCallbackHandler::kTranslation2Key,
214 Spatializer::EngineCallbackHandler::kRotation0Key,
215 Spatializer::EngineCallbackHandler::kRotation1Key,
216 Spatializer::EngineCallbackHandler::kRotation2Key,
217};
218
Eric Laurentbe21a942023-11-14 16:05:00 +0100219// Mapping table between strings read form property bluetooth.core.le.dsa_transport_preference
220// and low latency modes emums.
221//TODO b/273373363: use AIDL enum when available
222const std::map<std::string, audio_latency_mode_t> Spatializer::sStringToLatencyModeMap = {
223 {"le-acl", AUDIO_LATENCY_MODE_LOW},
224 {"iso-sw", AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_SOFTWARE},
225 {"iso-hw", AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_HARDWARE},
226};
227
228void Spatializer::loadOrderedLowLatencyModes() {
229 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
230 return;
231 }
232 auto latencyModesStrs = android::sysprop::BluetoothProperties::dsa_transport_preference();
Andy Hung79eacdb2023-11-30 19:34:24 -0800233 audio_utils::lock_guard lock(mMutex);
Eric Laurentbe21a942023-11-14 16:05:00 +0100234 // First load preferred low latency modes ordered from the property
235 for (auto str : latencyModesStrs) {
236 if (!str.has_value()) continue;
237 if (auto it = sStringToLatencyModeMap.find(str.value());
238 it != sStringToLatencyModeMap.end()) {
239 mOrderedLowLatencyModes.push_back(it->second);
240 }
241 }
242 // Then add unlisted latency modes at the end of the ordered list
243 for (auto it : sStringToLatencyModeMap) {
244 if (std::find(mOrderedLowLatencyModes.begin(), mOrderedLowLatencyModes.end(), it.second)
245 == mOrderedLowLatencyModes.end()) {
246 mOrderedLowLatencyModes.push_back(it.second);
247 }
248 }
249}
250
Eric Laurent8a4259f2021-09-14 16:04:00 +0200251// ---------------------------------------------------------------------------
Shunkai Yao8d6489a2023-04-18 23:14:25 +0000252sp<Spatializer> Spatializer::create(SpatializerPolicyCallback* callback,
253 const sp<EffectsFactoryHalInterface>& effectsFactoryHal) {
Eric Laurent6d607012021-07-05 11:54:40 +0200254 sp<Spatializer> spatializer;
255
Eric Laurent6d607012021-07-05 11:54:40 +0200256 if (effectsFactoryHal == nullptr) {
257 ALOGW("%s failed to create effect factory interface", __func__);
258 return spatializer;
259 }
260
261 std::vector<effect_descriptor_t> descriptors;
Shunkai Yao8d6489a2023-04-18 23:14:25 +0000262 status_t status = effectsFactoryHal->getDescriptors(FX_IID_SPATIALIZER, &descriptors);
Eric Laurent6d607012021-07-05 11:54:40 +0200263 if (status != NO_ERROR) {
264 ALOGW("%s failed to get spatializer descriptor, error %d", __func__, status);
265 return spatializer;
266 }
267 ALOG_ASSERT(!descriptors.empty(),
268 "%s getDescriptors() returned no error but empty list", __func__);
269
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000270 // TODO: get supported spatialization modes from FX engine or descriptor
Eric Laurent6d607012021-07-05 11:54:40 +0200271 sp<EffectHalInterface> effect;
272 status = effectsFactoryHal->createEffect(&descriptors[0].uuid, AUDIO_SESSION_OUTPUT_STAGE,
273 AUDIO_IO_HANDLE_NONE, AUDIO_PORT_HANDLE_NONE, &effect);
Mikhail Naganov89c22e42023-06-14 15:49:59 -0700274 ALOGI("%s FX create status %d effect %p", __func__, status, effect.get());
Eric Laurent6d607012021-07-05 11:54:40 +0200275
276 if (status == NO_ERROR && effect != nullptr) {
277 spatializer = new Spatializer(descriptors[0], callback);
Eric Laurent2be8b292021-08-23 09:44:33 -0700278 if (spatializer->loadEngineConfiguration(effect) != NO_ERROR) {
279 spatializer.clear();
Mikhail Naganov89c22e42023-06-14 15:49:59 -0700280 ALOGW("%s loadEngine error: %d effect %p", __func__, status, effect.get());
Andy Hung4e2547c2022-08-29 14:14:58 -0700281 } else {
Eric Laurentbe21a942023-11-14 16:05:00 +0100282 spatializer->loadOrderedLowLatencyModes();
Mikhail Naganov89c22e42023-06-14 15:49:59 -0700283 spatializer->mLocalLog.log("%s with effect Id %p", __func__, effect.get());
Eric Laurent2be8b292021-08-23 09:44:33 -0700284 }
Eric Laurent6d607012021-07-05 11:54:40 +0200285 }
286
287 return spatializer;
288}
289
Eric Laurent2be8b292021-08-23 09:44:33 -0700290Spatializer::Spatializer(effect_descriptor_t engineDescriptor, SpatializerPolicyCallback* callback)
291 : mEngineDescriptor(engineDescriptor),
292 mPolicyCallback(callback) {
Eric Laurent6d607012021-07-05 11:54:40 +0200293 ALOGV("%s", __func__);
Andy Hung225aef62022-12-06 16:33:20 -0800294 setMinSchedulerPolicy(SCHED_NORMAL, ANDROID_PRIORITY_AUDIO);
Eric Laurent6d607012021-07-05 11:54:40 +0200295}
296
Eric Laurent8a4259f2021-09-14 16:04:00 +0200297void Spatializer::onFirstRef() {
298 mLooper = new ALooper;
299 mLooper->setName("Spatializer-looper");
300 mLooper->start(
301 /*runOnCallingThread*/false,
302 /*canCallJava*/ false,
Andy Hungbf1777b2022-11-07 20:09:20 -0800303 PRIORITY_URGENT_AUDIO);
Eric Laurent8a4259f2021-09-14 16:04:00 +0200304
305 mHandler = new EngineCallbackHandler(this);
306 mLooper->registerHandler(mHandler);
307}
308
Eric Laurent6d607012021-07-05 11:54:40 +0200309Spatializer::~Spatializer() {
310 ALOGV("%s", __func__);
Eric Laurent8a4259f2021-09-14 16:04:00 +0200311 if (mLooper != nullptr) {
312 mLooper->stop();
313 mLooper->unregisterHandler(mHandler->id());
314 }
315 mLooper.clear();
316 mHandler.clear();
Eric Laurent6d607012021-07-05 11:54:40 +0200317}
318
Andy Hung8a7ecff2022-08-17 17:27:32 -0700319static std::string channelMaskVectorToString(
320 const std::vector<audio_channel_mask_t>& masks) {
321 std::stringstream ss;
322 for (const auto &mask : masks) {
323 if (ss.tellp() != 0) ss << "|";
324 ss << mask;
325 }
326 return ss.str();
327}
328
Eric Laurent2be8b292021-08-23 09:44:33 -0700329status_t Spatializer::loadEngineConfiguration(sp<EffectHalInterface> effect) {
330 ALOGV("%s", __func__);
331
332 std::vector<bool> supportsHeadTracking;
333 status_t status = getHalParameter<false>(effect, SPATIALIZER_PARAM_HEADTRACKING_SUPPORTED,
334 &supportsHeadTracking);
335 if (status != NO_ERROR) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700336 ALOGW("%s: cannot get SPATIALIZER_PARAM_HEADTRACKING_SUPPORTED", __func__);
Eric Laurent2be8b292021-08-23 09:44:33 -0700337 return status;
338 }
339 mSupportsHeadTracking = supportsHeadTracking[0];
340
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000341 std::vector<Spatialization::Level> spatializationLevels;
Andy Hung119dbdb2022-05-11 19:20:13 -0700342 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_LEVELS,
343 &spatializationLevels);
Eric Laurent2be8b292021-08-23 09:44:33 -0700344 if (status != NO_ERROR) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700345 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_LEVELS", __func__);
Eric Laurent2be8b292021-08-23 09:44:33 -0700346 return status;
347 }
Andy Hung119dbdb2022-05-11 19:20:13 -0700348 bool noneLevelFound = false;
349 bool activeLevelFound = false;
350 for (const auto spatializationLevel : spatializationLevels) {
351 if (!aidl_utils::isValidEnum(spatializationLevel)) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000352 ALOGW("%s: ignoring spatializationLevel:%s", __func__,
353 ToString(spatializationLevel).c_str());
Andy Hung119dbdb2022-05-11 19:20:13 -0700354 continue;
355 }
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000356 if (spatializationLevel == Spatialization::Level::NONE) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700357 noneLevelFound = true;
358 } else {
359 activeLevelFound = true;
360 }
361 // we don't detect duplicates.
362 mLevels.emplace_back(spatializationLevel);
363 }
364 if (!noneLevelFound || !activeLevelFound) {
365 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_LEVELS must include NONE"
366 " and another valid level", __func__);
367 return BAD_VALUE;
368 }
369
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000370 std::vector<Spatialization::Mode> spatializationModes;
Eric Laurent2be8b292021-08-23 09:44:33 -0700371 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES,
Andy Hung119dbdb2022-05-11 19:20:13 -0700372 &spatializationModes);
Eric Laurent2be8b292021-08-23 09:44:33 -0700373 if (status != NO_ERROR) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700374 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES", __func__);
Eric Laurent2be8b292021-08-23 09:44:33 -0700375 return status;
376 }
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000377
Andy Hung119dbdb2022-05-11 19:20:13 -0700378 for (const auto spatializationMode : spatializationModes) {
379 if (!aidl_utils::isValidEnum(spatializationMode)) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000380 ALOGW("%s: ignoring spatializationMode:%s", __func__,
381 ToString(spatializationMode).c_str());
Andy Hung119dbdb2022-05-11 19:20:13 -0700382 continue;
383 }
384 // we don't detect duplicates.
385 mSpatializationModes.emplace_back(spatializationMode);
386 }
387 if (mSpatializationModes.empty()) {
388 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES reports empty", __func__);
389 return BAD_VALUE;
390 }
391
392 std::vector<audio_channel_mask_t> channelMasks;
393 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS,
394 &channelMasks);
395 if (status != NO_ERROR) {
396 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS", __func__);
397 return status;
398 }
399 for (const auto channelMask : channelMasks) {
Nikhil Bhanu8f4ea772024-01-31 17:15:52 -0800400 static const bool stereo_spatialization_enabled =
401 property_get_bool("ro.audio.stereo_spatialization_enabled", false);
Andy Hung481bfe32023-12-18 14:00:29 -0800402 const bool channel_mask_spatialized =
Nikhil Bhanu8f4ea772024-01-31 17:15:52 -0800403 (stereo_spatialization_enabled && com_android_media_audio_stereo_spatialization())
Andy Hung481bfe32023-12-18 14:00:29 -0800404 ? audio_channel_mask_contains_stereo(channelMask)
405 : audio_is_channel_mask_spatialized(channelMask);
406 if (!channel_mask_spatialized) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700407 ALOGW("%s: ignoring channelMask:%#x", __func__, channelMask);
408 continue;
409 }
410 // we don't detect duplicates.
411 mChannelMasks.emplace_back(channelMask);
412 }
413 if (mChannelMasks.empty()) {
414 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS reports empty", __func__);
415 return BAD_VALUE;
416 }
Andy Hunga461a002022-05-17 10:36:02 -0700417
Eric Laurentbe21a942023-11-14 16:05:00 +0100418 if (com::android::media::audio::dsa_over_bt_le_audio()
419 && mSupportsHeadTracking) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000420 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED;
421 std::vector<HeadTracking::ConnectionMode> headtrackingConnectionModes;
Eric Laurentbe21a942023-11-14 16:05:00 +0100422 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_HEADTRACKING_CONNECTION,
423 &headtrackingConnectionModes);
424 if (status == NO_ERROR) {
425 for (const auto htConnectionMode : headtrackingConnectionModes) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000426 if (htConnectionMode < HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED ||
427 htConnectionMode > HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_TUNNEL) {
428 ALOGW("%s: ignoring HT connection mode:%s", __func__,
429 ToString(htConnectionMode).c_str());
Eric Laurentbe21a942023-11-14 16:05:00 +0100430 continue;
431 }
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000432 mSupportedHeadtrackingConnectionModes.insert(htConnectionMode);
Eric Laurentbe21a942023-11-14 16:05:00 +0100433 }
434 ALOGW_IF(mSupportedHeadtrackingConnectionModes.find(
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000435 HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED) ==
436 mSupportedHeadtrackingConnectionModes.end(),
437 "%s: Headtracking FRAMEWORK_PROCESSED not reported", __func__);
Eric Laurentbe21a942023-11-14 16:05:00 +0100438 }
439 }
440
Andy Hunga461a002022-05-17 10:36:02 -0700441 // Currently we expose only RELATIVE_WORLD.
442 // This is a limitation of the head tracking library based on a UX choice.
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000443 mHeadTrackingModes.push_back(HeadTracking::Mode::DISABLED);
Andy Hunga461a002022-05-17 10:36:02 -0700444 if (mSupportsHeadTracking) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000445 mHeadTrackingModes.push_back(HeadTracking::Mode::RELATIVE_WORLD);
Andy Hunga461a002022-05-17 10:36:02 -0700446 }
447 mediametrics::LogItem(mMetricsId)
448 .set(AMEDIAMETRICS_PROP_EVENT, AMEDIAMETRICS_PROP_EVENT_VALUE_CREATE)
Andy Hung8a7ecff2022-08-17 17:27:32 -0700449 .set(AMEDIAMETRICS_PROP_CHANNELMASKS, channelMaskVectorToString(mChannelMasks))
Andy Hunga461a002022-05-17 10:36:02 -0700450 .set(AMEDIAMETRICS_PROP_LEVELS, aidl_utils::enumsToString(mLevels))
451 .set(AMEDIAMETRICS_PROP_MODES, aidl_utils::enumsToString(mSpatializationModes))
452 .set(AMEDIAMETRICS_PROP_HEADTRACKINGMODES, aidl_utils::enumsToString(mHeadTrackingModes))
453 .set(AMEDIAMETRICS_PROP_STATUS, (int32_t)status)
454 .record();
Andy Hung119dbdb2022-05-11 19:20:13 -0700455 return NO_ERROR;
Eric Laurent2be8b292021-08-23 09:44:33 -0700456}
457
Andy Hung8aa43c02022-09-13 18:53:06 -0700458/* static */
459void Spatializer::sendEmptyCreateSpatializerMetricWithStatus(status_t status) {
460 mediametrics::LogItem(kDefaultMetricsId)
461 .set(AMEDIAMETRICS_PROP_EVENT, AMEDIAMETRICS_PROP_EVENT_VALUE_CREATE)
462 .set(AMEDIAMETRICS_PROP_CHANNELMASKS, "")
463 .set(AMEDIAMETRICS_PROP_LEVELS, "")
464 .set(AMEDIAMETRICS_PROP_MODES, "")
465 .set(AMEDIAMETRICS_PROP_HEADTRACKINGMODES, "")
466 .set(AMEDIAMETRICS_PROP_STATUS, (int32_t)status)
467 .record();
468}
469
Eric Laurent2be8b292021-08-23 09:44:33 -0700470/** Gets the channel mask, sampling rate and format set for the spatializer input. */
471audio_config_base_t Spatializer::getAudioInConfig() const {
Andy Hung79eacdb2023-11-30 19:34:24 -0800472 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700473 audio_config_base_t config = AUDIO_CONFIG_BASE_INITIALIZER;
474 // For now use highest supported channel count
Andy Hung4e2547c2022-08-29 14:14:58 -0700475 config.channel_mask = getMaxChannelMask(mChannelMasks, FCC_LIMIT);
Eric Laurent2be8b292021-08-23 09:44:33 -0700476 return config;
477}
478
Eric Laurent6d607012021-07-05 11:54:40 +0200479status_t Spatializer::registerCallback(
480 const sp<media::INativeSpatializerCallback>& callback) {
Andy Hung79eacdb2023-11-30 19:34:24 -0800481 audio_utils::lock_guard lock(mMutex);
Eric Laurent6d607012021-07-05 11:54:40 +0200482 if (callback == nullptr) {
483 return BAD_VALUE;
484 }
485
Eric Laurentd6bee3a2022-08-31 16:07:50 +0200486 if (mSpatializerCallback != nullptr) {
487 if (IInterface::asBinder(callback) == IInterface::asBinder(mSpatializerCallback)) {
488 ALOGW("%s: Registering callback %p again",
489 __func__, mSpatializerCallback.get());
490 return NO_ERROR;
491 }
492 ALOGE("%s: Already one client registered with callback %p",
493 __func__, mSpatializerCallback.get());
494 return INVALID_OPERATION;
495 }
496
Eric Laurent6d607012021-07-05 11:54:40 +0200497 sp<IBinder> binder = IInterface::asBinder(callback);
498 status_t status = binder->linkToDeath(this);
499 if (status == NO_ERROR) {
500 mSpatializerCallback = callback;
501 }
502 ALOGV("%s status %d", __func__, status);
503 return status;
504}
505
506// IBinder::DeathRecipient
507void Spatializer::binderDied(__unused const wp<IBinder> &who) {
508 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800509 audio_utils::lock_guard lock(mMutex);
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000510 mLevel = Spatialization::Level::NONE;
Eric Laurent6d607012021-07-05 11:54:40 +0200511 mSpatializerCallback.clear();
512 }
513 ALOGV("%s", __func__);
514 mPolicyCallback->onCheckSpatializer();
515}
516
517// ISpatializer
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000518Status Spatializer::getSupportedLevels(std::vector<Spatialization::Level> *levels) {
Eric Laurent6d607012021-07-05 11:54:40 +0200519 ALOGV("%s", __func__);
520 if (levels == nullptr) {
521 return binderStatusFromStatusT(BAD_VALUE);
522 }
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000523 // Spatialization::Level::NONE is already required from the effect or we don't load it.
Eric Laurent2be8b292021-08-23 09:44:33 -0700524 levels->insert(levels->end(), mLevels.begin(), mLevels.end());
Eric Laurent6d607012021-07-05 11:54:40 +0200525 return Status::ok();
526}
527
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000528Status Spatializer::setLevel(Spatialization::Level level) {
529 ALOGV("%s level %s", __func__, ToString(level).c_str());
530 mLocalLog.log("%s with %s", __func__, ToString(level).c_str());
531 if (level != Spatialization::Level::NONE
Eric Laurent2be8b292021-08-23 09:44:33 -0700532 && std::find(mLevels.begin(), mLevels.end(), level) == mLevels.end()) {
Eric Laurent6d607012021-07-05 11:54:40 +0200533 return binderStatusFromStatusT(BAD_VALUE);
534 }
535 sp<media::INativeSpatializerCallback> callback;
536 bool levelChanged = false;
537 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800538 audio_utils::lock_guard lock(mMutex);
Eric Laurent6d607012021-07-05 11:54:40 +0200539 levelChanged = mLevel != level;
540 mLevel = level;
541 callback = mSpatializerCallback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700542
543 if (levelChanged && mEngine != nullptr) {
Eric Laurent7ea0d1b2022-04-01 14:23:44 +0200544 checkEngineState_l();
Eric Laurent2be8b292021-08-23 09:44:33 -0700545 }
Eric Laurent9249d342022-03-18 11:55:56 +0100546 checkSensorsState_l();
Eric Laurent6d607012021-07-05 11:54:40 +0200547 }
548
549 if (levelChanged) {
550 mPolicyCallback->onCheckSpatializer();
551 if (callback != nullptr) {
552 callback->onLevelChanged(level);
553 }
554 }
555 return Status::ok();
556}
557
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000558Status Spatializer::getLevel(Spatialization::Level *level) {
Eric Laurent6d607012021-07-05 11:54:40 +0200559 if (level == nullptr) {
560 return binderStatusFromStatusT(BAD_VALUE);
561 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800562 audio_utils::lock_guard lock(mMutex);
Eric Laurent6d607012021-07-05 11:54:40 +0200563 *level = mLevel;
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000564 ALOGV("%s level %s", __func__, ToString(*level).c_str());
Eric Laurent6d607012021-07-05 11:54:40 +0200565 return Status::ok();
566}
567
Eric Laurentc87402b2021-09-17 16:49:42 +0200568Status Spatializer::isHeadTrackingSupported(bool *supports) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000569 ALOGV("%s mSupportsHeadTracking %s", __func__, ToString(mSupportsHeadTracking).c_str());
Eric Laurentc87402b2021-09-17 16:49:42 +0200570 if (supports == nullptr) {
571 return binderStatusFromStatusT(BAD_VALUE);
572 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800573 audio_utils::lock_guard lock(mMutex);
Eric Laurentc87402b2021-09-17 16:49:42 +0200574 *supports = mSupportsHeadTracking;
575 return Status::ok();
576}
577
Eric Laurent6d607012021-07-05 11:54:40 +0200578Status Spatializer::getSupportedHeadTrackingModes(
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000579 std::vector<HeadTracking::Mode>* modes) {
Andy Hung79eacdb2023-11-30 19:34:24 -0800580 audio_utils::lock_guard lock(mMutex);
Eric Laurent6d607012021-07-05 11:54:40 +0200581 ALOGV("%s", __func__);
582 if (modes == nullptr) {
583 return binderStatusFromStatusT(BAD_VALUE);
584 }
Andy Hunga461a002022-05-17 10:36:02 -0700585 modes->insert(modes->end(), mHeadTrackingModes.begin(), mHeadTrackingModes.end());
Eric Laurent6d607012021-07-05 11:54:40 +0200586 return Status::ok();
587}
588
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000589Status Spatializer::setDesiredHeadTrackingMode(HeadTracking::Mode mode) {
590 ALOGV("%s mode %s", __func__, ToString(mode).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700591
592 if (!mSupportsHeadTracking) {
593 return binderStatusFromStatusT(INVALID_OPERATION);
594 }
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000595 mLocalLog.log("%s with %s", __func__, ToString(mode).c_str());
Andy Hung79eacdb2023-11-30 19:34:24 -0800596 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700597 switch (mode) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000598 case HeadTracking::Mode::OTHER:
Eric Laurent2be8b292021-08-23 09:44:33 -0700599 return binderStatusFromStatusT(BAD_VALUE);
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000600 case HeadTracking::Mode::DISABLED:
Eric Laurent2be8b292021-08-23 09:44:33 -0700601 mDesiredHeadTrackingMode = HeadTrackingMode::STATIC;
602 break;
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000603 case HeadTracking::Mode::RELATIVE_WORLD:
Eric Laurent2be8b292021-08-23 09:44:33 -0700604 mDesiredHeadTrackingMode = HeadTrackingMode::WORLD_RELATIVE;
605 break;
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000606 case HeadTracking::Mode::RELATIVE_SCREEN:
Eric Laurent2be8b292021-08-23 09:44:33 -0700607 mDesiredHeadTrackingMode = HeadTrackingMode::SCREEN_RELATIVE;
608 break;
609 }
610
Eric Laurent11094172022-04-05 18:27:42 +0200611 checkPoseController_l();
612 checkSensorsState_l();
Eric Laurent2be8b292021-08-23 09:44:33 -0700613
614 return Status::ok();
615}
616
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000617Status Spatializer::getActualHeadTrackingMode(HeadTracking::Mode *mode) {
Eric Laurent6d607012021-07-05 11:54:40 +0200618 if (mode == nullptr) {
619 return binderStatusFromStatusT(BAD_VALUE);
620 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800621 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700622 *mode = mActualHeadTrackingMode;
Eric Laurent6d607012021-07-05 11:54:40 +0200623 ALOGV("%s mode %d", __func__, (int)*mode);
624 return Status::ok();
625}
626
Ytai Ben-Tsvia16a9df2021-08-05 08:57:06 -0700627Status Spatializer::recenterHeadTracker() {
Eric Laurent780be4a2021-09-16 10:44:24 +0200628 if (!mSupportsHeadTracking) {
629 return binderStatusFromStatusT(INVALID_OPERATION);
630 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800631 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700632 if (mPoseController != nullptr) {
633 mPoseController->recenter();
634 }
Eric Laurent6d607012021-07-05 11:54:40 +0200635 return Status::ok();
636}
637
638Status Spatializer::setGlobalTransform(const std::vector<float>& screenToStage) {
Eric Laurent6d607012021-07-05 11:54:40 +0200639 ALOGV("%s", __func__);
Eric Laurent780be4a2021-09-16 10:44:24 +0200640 if (!mSupportsHeadTracking) {
641 return binderStatusFromStatusT(INVALID_OPERATION);
642 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700643 std::optional<Pose3f> maybePose = Pose3f::fromVector(screenToStage);
644 if (!maybePose.has_value()) {
645 ALOGW("Invalid screenToStage vector.");
646 return binderStatusFromStatusT(BAD_VALUE);
647 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800648 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700649 if (mPoseController != nullptr) {
Andy Hunga367cb22023-01-30 11:58:44 -0800650 mLocalLog.log("%s with screenToStage %s", __func__,
651 media::VectorRecorder::toString<float>(screenToStage).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700652 mPoseController->setScreenToStagePose(maybePose.value());
653 }
Eric Laurent6d607012021-07-05 11:54:40 +0200654 return Status::ok();
655}
656
657Status Spatializer::release() {
658 ALOGV("%s", __func__);
659 bool levelChanged = false;
660 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800661 audio_utils::lock_guard lock(mMutex);
Eric Laurent6d607012021-07-05 11:54:40 +0200662 if (mSpatializerCallback == nullptr) {
663 return binderStatusFromStatusT(INVALID_OPERATION);
664 }
665
666 sp<IBinder> binder = IInterface::asBinder(mSpatializerCallback);
667 binder->unlinkToDeath(this);
668 mSpatializerCallback.clear();
669
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000670 levelChanged = mLevel != Spatialization::Level::NONE;
671 mLevel = Spatialization::Level::NONE;
Eric Laurent6d607012021-07-05 11:54:40 +0200672 }
673
674 if (levelChanged) {
675 mPolicyCallback->onCheckSpatializer();
676 }
677 return Status::ok();
678}
679
Eric Laurent2be8b292021-08-23 09:44:33 -0700680Status Spatializer::setHeadSensor(int sensorHandle) {
681 ALOGV("%s sensorHandle %d", __func__, sensorHandle);
Eric Laurent780be4a2021-09-16 10:44:24 +0200682 if (!mSupportsHeadTracking) {
683 return binderStatusFromStatusT(INVALID_OPERATION);
684 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800685 audio_utils::lock_guard lock(mMutex);
Andy Hungba2a61a2022-05-20 12:00:28 -0700686 if (mHeadSensor != sensorHandle) {
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000687 mLocalLog.log("%s with 0x%08x", __func__, sensorHandle);
Andy Hungba2a61a2022-05-20 12:00:28 -0700688 mHeadSensor = sensorHandle;
689 checkPoseController_l();
690 checkSensorsState_l();
691 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700692 return Status::ok();
693}
694
695Status Spatializer::setScreenSensor(int sensorHandle) {
696 ALOGV("%s sensorHandle %d", __func__, sensorHandle);
Eric Laurent780be4a2021-09-16 10:44:24 +0200697 if (!mSupportsHeadTracking) {
698 return binderStatusFromStatusT(INVALID_OPERATION);
699 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800700 audio_utils::lock_guard lock(mMutex);
Andy Hungba2a61a2022-05-20 12:00:28 -0700701 if (mScreenSensor != sensorHandle) {
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000702 mLocalLog.log("%s with 0x%08x", __func__, sensorHandle);
Andy Hungba2a61a2022-05-20 12:00:28 -0700703 mScreenSensor = sensorHandle;
704 // TODO: consider a new method setHeadAndScreenSensor()
705 // because we generally set both at the same time.
706 // This will avoid duplicated work and recentering.
707 checkSensorsState_l();
708 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700709 return Status::ok();
710}
711
712Status Spatializer::setDisplayOrientation(float physicalToLogicalAngle) {
713 ALOGV("%s physicalToLogicalAngle %f", __func__, physicalToLogicalAngle);
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000714 mLocalLog.log("%s with %f", __func__, physicalToLogicalAngle);
Andy Hung2490b2d2023-03-09 20:45:36 -0800715 const float angle = safe_clamp(physicalToLogicalAngle, 0.f, (float)(2. * M_PI));
716 // It is possible due to numerical inaccuracies to exceed the boundaries of 0 to 2 * M_PI.
717 ALOGI_IF(angle != physicalToLogicalAngle,
718 "%s: clamping %f to %f", __func__, physicalToLogicalAngle, angle);
Andy Hung79eacdb2023-11-30 19:34:24 -0800719 audio_utils::lock_guard lock(mMutex);
Andy Hung2490b2d2023-03-09 20:45:36 -0800720 mDisplayOrientation = angle;
Eric Laurent2be8b292021-08-23 09:44:33 -0700721 if (mPoseController != nullptr) {
Andy Hung2490b2d2023-03-09 20:45:36 -0800722 // This turns on the rate-limiter.
723 mPoseController->setDisplayOrientation(angle);
Eric Laurent2be8b292021-08-23 09:44:33 -0700724 }
Eric Laurent16ddaf42021-09-17 15:00:35 +0200725 if (mEngine != nullptr) {
726 setEffectParameter_l(
Andy Hung2490b2d2023-03-09 20:45:36 -0800727 SPATIALIZER_PARAM_DISPLAY_ORIENTATION, std::vector<float>{angle});
Eric Laurent16ddaf42021-09-17 15:00:35 +0200728 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700729 return Status::ok();
730}
731
732Status Spatializer::setHingeAngle(float hingeAngle) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700733 ALOGV("%s hingeAngle %f", __func__, hingeAngle);
Andy Hung2490b2d2023-03-09 20:45:36 -0800734 mLocalLog.log("%s with %f", __func__, hingeAngle);
735 const float angle = safe_clamp(hingeAngle, 0.f, (float)(2. * M_PI));
736 // It is possible due to numerical inaccuracies to exceed the boundaries of 0 to 2 * M_PI.
737 ALOGI_IF(angle != hingeAngle,
738 "%s: clamping %f to %f", __func__, hingeAngle, angle);
Andy Hung79eacdb2023-11-30 19:34:24 -0800739 audio_utils::lock_guard lock(mMutex);
Andy Hung2490b2d2023-03-09 20:45:36 -0800740 mHingeAngle = angle;
Eric Laurent2be8b292021-08-23 09:44:33 -0700741 if (mEngine != nullptr) {
Andy Hung2490b2d2023-03-09 20:45:36 -0800742 setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE, std::vector<float>{angle});
743 }
744 return Status::ok();
745}
746
747Status Spatializer::setFoldState(bool folded) {
748 ALOGV("%s foldState %d", __func__, (int)folded);
749 mLocalLog.log("%s with %d", __func__, (int)folded);
Andy Hung79eacdb2023-11-30 19:34:24 -0800750 audio_utils::lock_guard lock(mMutex);
Andy Hung2490b2d2023-03-09 20:45:36 -0800751 mFoldedState = folded;
752 if (mEngine != nullptr) {
753 // we don't suppress multiple calls with the same folded state - that's
754 // done at the caller.
755 setEffectParameter_l(SPATIALIZER_PARAM_FOLD_STATE, std::vector<uint8_t>{mFoldedState});
Eric Laurent2be8b292021-08-23 09:44:33 -0700756 }
757 return Status::ok();
758}
759
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000760Status Spatializer::getSupportedModes(std::vector<Spatialization::Mode> *modes) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700761 ALOGV("%s", __func__);
762 if (modes == nullptr) {
763 return binderStatusFromStatusT(BAD_VALUE);
764 }
765 *modes = mSpatializationModes;
766 return Status::ok();
767}
768
Eric Laurent67816e32021-09-16 15:18:40 +0200769Status Spatializer::registerHeadTrackingCallback(
770 const sp<media::ISpatializerHeadTrackingCallback>& callback) {
771 ALOGV("%s callback %p", __func__, callback.get());
Andy Hung79eacdb2023-11-30 19:34:24 -0800772 audio_utils::lock_guard lock(mMutex);
Eric Laurent67816e32021-09-16 15:18:40 +0200773 if (!mSupportsHeadTracking) {
774 return binderStatusFromStatusT(INVALID_OPERATION);
775 }
776 mHeadTrackingCallback = callback;
777 return Status::ok();
778}
779
Eric Laurentc87402b2021-09-17 16:49:42 +0200780Status Spatializer::setParameter(int key, const std::vector<unsigned char>& value) {
781 ALOGV("%s key %d", __func__, key);
Andy Hung79eacdb2023-11-30 19:34:24 -0800782 audio_utils::lock_guard lock(mMutex);
Eric Laurentc87402b2021-09-17 16:49:42 +0200783 status_t status = INVALID_OPERATION;
784 if (mEngine != nullptr) {
785 status = setEffectParameter_l(key, value);
786 }
787 return binderStatusFromStatusT(status);
788}
789
790Status Spatializer::getParameter(int key, std::vector<unsigned char> *value) {
Greg Kaiserf7249f82021-09-21 07:10:12 -0700791 ALOGV("%s key %d value size %d", __func__, key,
792 (value != nullptr ? (int)value->size() : -1));
Eric Laurentc87402b2021-09-17 16:49:42 +0200793 if (value == nullptr) {
George Burgess IV22386222021-09-22 12:09:31 -0700794 return binderStatusFromStatusT(BAD_VALUE);
Eric Laurentc87402b2021-09-17 16:49:42 +0200795 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800796 audio_utils::lock_guard lock(mMutex);
Eric Laurentc87402b2021-09-17 16:49:42 +0200797 status_t status = INVALID_OPERATION;
798 if (mEngine != nullptr) {
799 ALOGV("%s key %d mEngine %p", __func__, key, mEngine.get());
800 status = getEffectParameter_l(key, value);
801 }
802 return binderStatusFromStatusT(status);
803}
804
805Status Spatializer::getOutput(int *output) {
806 ALOGV("%s", __func__);
807 if (output == nullptr) {
808 binderStatusFromStatusT(BAD_VALUE);
809 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800810 audio_utils::lock_guard lock(mMutex);
Eric Laurentc87402b2021-09-17 16:49:42 +0200811 *output = VALUE_OR_RETURN_BINDER_STATUS(legacy2aidl_audio_io_handle_t_int32_t(mOutput));
812 ALOGV("%s got output %d", __func__, *output);
813 return Status::ok();
814}
815
Eric Laurent2be8b292021-08-23 09:44:33 -0700816// SpatializerPoseController::Listener
817void Spatializer::onHeadToStagePose(const Pose3f& headToStage) {
818 ALOGV("%s", __func__);
Eric Laurent780be4a2021-09-16 10:44:24 +0200819 LOG_ALWAYS_FATAL_IF(!mSupportsHeadTracking,
820 "onHeadToStagePose() called with no head tracking support!");
821
Eric Laurent2be8b292021-08-23 09:44:33 -0700822 auto vec = headToStage.toVector();
Eric Laurent8a4259f2021-09-14 16:04:00 +0200823 LOG_ALWAYS_FATAL_IF(vec.size() != sHeadPoseKeys.size(),
824 "%s invalid head to stage vector size %zu", __func__, vec.size());
Eric Laurent8a4259f2021-09-14 16:04:00 +0200825 sp<AMessage> msg =
826 new AMessage(EngineCallbackHandler::kWhatOnHeadToStagePose, mHandler);
827 for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
828 msg->setFloat(sHeadPoseKeys[i], vec[i]);
829 }
830 msg->post();
831}
832
Eric Laurentbdecc052022-10-21 11:28:32 +0200833void Spatializer::resetEngineHeadPose_l() {
834 ALOGV("%s mEngine %p", __func__, mEngine.get());
835 if (mEngine == nullptr) {
836 return;
837 }
838 const std::vector<float> headToStage(6, 0.0);
839 setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, headToStage);
840 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000841 std::vector<HeadTracking::Mode>{HeadTracking::Mode::DISABLED});
Eric Laurentbdecc052022-10-21 11:28:32 +0200842}
843
Eric Laurent8a4259f2021-09-14 16:04:00 +0200844void Spatializer::onHeadToStagePoseMsg(const std::vector<float>& headToStage) {
845 ALOGV("%s", __func__);
Eric Laurent67816e32021-09-16 15:18:40 +0200846 sp<media::ISpatializerHeadTrackingCallback> callback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700847 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800848 audio_utils::lock_guard lock(mMutex);
Eric Laurent67816e32021-09-16 15:18:40 +0200849 callback = mHeadTrackingCallback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700850 if (mEngine != nullptr) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200851 setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, headToStage);
Andy Hung560addd2023-01-30 11:58:44 -0800852 const auto record = recordFromTranslationRotationVector(headToStage);
853 mPoseRecorder.record(record);
854 mPoseDurableRecorder.record(record);
Eric Laurent2be8b292021-08-23 09:44:33 -0700855 }
856 }
857
858 if (callback != nullptr) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200859 callback->onHeadToSoundStagePoseUpdated(headToStage);
Eric Laurent2be8b292021-08-23 09:44:33 -0700860 }
861}
862
863void Spatializer::onActualModeChange(HeadTrackingMode mode) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000864 std::string modeStr = ToString(mode);
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000865 ALOGV("%s(%s)", __func__, modeStr.c_str());
Shunkai Yao56e43f02022-08-30 03:14:50 +0000866 sp<AMessage> msg = new AMessage(EngineCallbackHandler::kWhatOnActualModeChange, mHandler);
Eric Laurent8a4259f2021-09-14 16:04:00 +0200867 msg->setInt32(EngineCallbackHandler::kModeKey, static_cast<int>(mode));
868 msg->post();
869}
870
871void Spatializer::onActualModeChangeMsg(HeadTrackingMode mode) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000872 ALOGV("%s(%s)", __func__, ToString(mode).c_str());
Eric Laurent67816e32021-09-16 15:18:40 +0200873 sp<media::ISpatializerHeadTrackingCallback> callback;
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000874 HeadTracking::Mode spatializerMode;
Eric Laurent2be8b292021-08-23 09:44:33 -0700875 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800876 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700877 if (!mSupportsHeadTracking) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000878 spatializerMode = HeadTracking::Mode::DISABLED;
Eric Laurent2be8b292021-08-23 09:44:33 -0700879 } else {
880 switch (mode) {
881 case HeadTrackingMode::STATIC:
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000882 spatializerMode = HeadTracking::Mode::DISABLED;
Eric Laurent2be8b292021-08-23 09:44:33 -0700883 break;
884 case HeadTrackingMode::WORLD_RELATIVE:
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000885 spatializerMode = HeadTracking::Mode::RELATIVE_WORLD;
Eric Laurent2be8b292021-08-23 09:44:33 -0700886 break;
887 case HeadTrackingMode::SCREEN_RELATIVE:
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000888 spatializerMode = HeadTracking::Mode::RELATIVE_SCREEN;
Eric Laurent2be8b292021-08-23 09:44:33 -0700889 break;
890 default:
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000891 LOG_ALWAYS_FATAL("Unknown mode: %s", ToString(mode).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700892 }
893 }
894 mActualHeadTrackingMode = spatializerMode;
Eric Laurente51f80e2022-04-14 10:20:38 +0200895 if (mEngine != nullptr) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000896 if (spatializerMode == HeadTracking::Mode::DISABLED) {
Eric Laurentbdecc052022-10-21 11:28:32 +0200897 resetEngineHeadPose_l();
898 } else {
899 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000900 std::vector<HeadTracking::Mode>{spatializerMode});
Eric Laurentbe21a942023-11-14 16:05:00 +0100901 setEngineHeadtrackingConnectionMode_l();
Eric Laurentbdecc052022-10-21 11:28:32 +0200902 }
Eric Laurent7ea0d1b2022-04-01 14:23:44 +0200903 }
Eric Laurent67816e32021-09-16 15:18:40 +0200904 callback = mHeadTrackingCallback;
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000905 mLocalLog.log("%s: updating mode to %s", __func__, ToString(mode).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700906 }
Eric Laurente51f80e2022-04-14 10:20:38 +0200907 if (callback != nullptr) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700908 callback->onHeadTrackingModeChanged(spatializerMode);
909 }
910}
911
Eric Laurentbe21a942023-11-14 16:05:00 +0100912void Spatializer::setEngineHeadtrackingConnectionMode_l() {
913 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
914 return;
915 }
916 if (mActualHeadTrackingMode != HeadTracking::Mode::DISABLED
917 && !mSupportedHeadtrackingConnectionModes.empty()) {
918 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_CONNECTION,
919 static_cast<uint8_t>(mHeadtrackingConnectionMode),
920 static_cast<uint32_t>(mHeadSensor));
921 }
922}
923
924void Spatializer::sortSupportedLatencyModes_l() {
925 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
926 return;
927 }
928 std::sort(mSupportedLatencyModes.begin(), mSupportedLatencyModes.end(),
929 [this](audio_latency_mode_t x, audio_latency_mode_t y) {
930 auto itX = std::find(mOrderedLowLatencyModes.begin(),
931 mOrderedLowLatencyModes.end(), x);
932 auto itY = std::find(mOrderedLowLatencyModes.begin(),
933 mOrderedLowLatencyModes.end(), y);
934 return itX < itY;
935 });
936}
937
Eric Laurent15903592022-02-24 20:44:36 +0100938status_t Spatializer::attachOutput(audio_io_handle_t output, size_t numActiveTracks) {
Eric Laurent4a872862021-10-11 17:06:47 +0200939 bool outputChanged = false;
940 sp<media::INativeSpatializerCallback> callback;
941
Eric Laurent2be8b292021-08-23 09:44:33 -0700942 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800943 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700944 ALOGV("%s output %d mOutput %d", __func__, (int)output, (int)mOutput);
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000945 mLocalLog.log("%s with output %d tracks %zu (mOutput %d)", __func__, (int)output,
946 numActiveTracks, (int)mOutput);
Eric Laurent2be8b292021-08-23 09:44:33 -0700947 if (mOutput != AUDIO_IO_HANDLE_NONE) {
948 LOG_ALWAYS_FATAL_IF(mEngine == nullptr, "%s output set without FX engine", __func__);
949 // remove FX instance
950 mEngine->setEnabled(false);
951 mEngine.clear();
Eric Laurent15903592022-02-24 20:44:36 +0100952 mPoseController.reset();
Eric Laurentee398ad2022-05-03 18:19:35 +0200953 AudioSystem::removeSupportedLatencyModesCallback(this);
Eric Laurent2be8b292021-08-23 09:44:33 -0700954 }
Eric Laurentee398ad2022-05-03 18:19:35 +0200955
Eric Laurent2be8b292021-08-23 09:44:33 -0700956 // create FX instance on output
957 AttributionSourceState attributionSource = AttributionSourceState();
958 mEngine = new AudioEffect(attributionSource);
Atneya Nair20e6cc82022-05-17 20:12:37 -0400959 mEngine->set(nullptr /* type */, &mEngineDescriptor.uuid, 0 /* priority */,
960 wp<AudioEffect::IAudioEffectCallback>::fromExisting(this),
961 AUDIO_SESSION_OUTPUT_STAGE, output, {} /* device */, false /* probe */,
962 true /* notifyFramesProcessed */);
Eric Laurent2be8b292021-08-23 09:44:33 -0700963 status_t status = mEngine->initCheck();
964 ALOGV("%s mEngine create status %d", __func__, (int)status);
965 if (status != NO_ERROR) {
966 return status;
967 }
968
Eric Laurent4a872862021-10-11 17:06:47 +0200969 outputChanged = mOutput != output;
Eric Laurent2be8b292021-08-23 09:44:33 -0700970 mOutput = output;
Eric Laurent11094172022-04-05 18:27:42 +0200971 mNumActiveTracks = numActiveTracks;
Eric Laurentee398ad2022-05-03 18:19:35 +0200972 AudioSystem::addSupportedLatencyModesCallback(this);
973
974 std::vector<audio_latency_mode_t> latencyModes;
975 status = AudioSystem::getSupportedLatencyModes(mOutput, &latencyModes);
976 if (status == OK) {
977 mSupportedLatencyModes = latencyModes;
Eric Laurentbe21a942023-11-14 16:05:00 +0100978 sortSupportedLatencyModes_l();
Eric Laurentee398ad2022-05-03 18:19:35 +0200979 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700980
Eric Laurent11094172022-04-05 18:27:42 +0200981 checkEngineState_l();
Eric Laurent780be4a2021-09-16 10:44:24 +0200982 if (mSupportsHeadTracking) {
Eric Laurent11094172022-04-05 18:27:42 +0200983 checkPoseController_l();
Eric Laurent9249d342022-03-18 11:55:56 +0100984 checkSensorsState_l();
Eric Laurent780be4a2021-09-16 10:44:24 +0200985 }
Eric Laurent4a872862021-10-11 17:06:47 +0200986 callback = mSpatializerCallback;
Andy Hung2490b2d2023-03-09 20:45:36 -0800987
988 // Restore common effect state.
989 setEffectParameter_l(SPATIALIZER_PARAM_DISPLAY_ORIENTATION,
990 std::vector<float>{mDisplayOrientation});
991 setEffectParameter_l(SPATIALIZER_PARAM_FOLD_STATE,
992 std::vector<uint8_t>{mFoldedState});
993 setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE,
994 std::vector<float>{mHingeAngle});
Eric Laurent6d607012021-07-05 11:54:40 +0200995 }
Eric Laurent4a872862021-10-11 17:06:47 +0200996
997 if (outputChanged && callback != nullptr) {
998 callback->onOutputChanged(output);
999 }
1000
Eric Laurent6d607012021-07-05 11:54:40 +02001001 return NO_ERROR;
1002}
1003
1004audio_io_handle_t Spatializer::detachOutput() {
Eric Laurent2be8b292021-08-23 09:44:33 -07001005 audio_io_handle_t output = AUDIO_IO_HANDLE_NONE;
Eric Laurent4a872862021-10-11 17:06:47 +02001006 sp<media::INativeSpatializerCallback> callback;
1007
1008 {
Andy Hung79eacdb2023-11-30 19:34:24 -08001009 audio_utils::lock_guard lock(mMutex);
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001010 mLocalLog.log("%s with output %d tracks %zu", __func__, (int)mOutput, mNumActiveTracks);
Eric Laurent4a872862021-10-11 17:06:47 +02001011 ALOGV("%s mOutput %d", __func__, (int)mOutput);
1012 if (mOutput == AUDIO_IO_HANDLE_NONE) {
1013 return output;
1014 }
1015 // remove FX instance
1016 mEngine->setEnabled(false);
1017 mEngine.clear();
Eric Laurentee398ad2022-05-03 18:19:35 +02001018 AudioSystem::removeSupportedLatencyModesCallback(this);
Eric Laurent4a872862021-10-11 17:06:47 +02001019 output = mOutput;
1020 mOutput = AUDIO_IO_HANDLE_NONE;
1021 mPoseController.reset();
Eric Laurent4a872862021-10-11 17:06:47 +02001022 callback = mSpatializerCallback;
Eric Laurent6d607012021-07-05 11:54:40 +02001023 }
Eric Laurent4a872862021-10-11 17:06:47 +02001024
1025 if (callback != nullptr) {
1026 callback->onOutputChanged(AUDIO_IO_HANDLE_NONE);
1027 }
Eric Laurent6d607012021-07-05 11:54:40 +02001028 return output;
1029}
1030
Eric Laurentee398ad2022-05-03 18:19:35 +02001031void Spatializer::onSupportedLatencyModesChanged(
1032 audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes) {
Eric Laurent9c04de92022-07-20 13:49:47 +02001033 ALOGV("%s output %d num modes %zu", __func__, (int)output, modes.size());
1034 sp<AMessage> msg =
1035 new AMessage(EngineCallbackHandler::kWhatOnLatencyModesChanged, mHandler);
1036 msg->setObject(EngineCallbackHandler::kLatencyModesKey,
1037 sp<EngineCallbackHandler::LatencyModes>::make(output, modes));
1038 msg->post();
1039}
1040
1041void Spatializer::onSupportedLatencyModesChangedMsg(
1042 audio_io_handle_t output, std::vector<audio_latency_mode_t>&& modes) {
Andy Hung79eacdb2023-11-30 19:34:24 -08001043 audio_utils::lock_guard lock(mMutex);
Eric Laurent9c04de92022-07-20 13:49:47 +02001044 ALOGV("%s output %d mOutput %d num modes %zu",
1045 __func__, (int)output, (int)mOutput, modes.size());
Eric Laurentee398ad2022-05-03 18:19:35 +02001046 if (output == mOutput) {
Eric Laurent9c04de92022-07-20 13:49:47 +02001047 mSupportedLatencyModes = std::move(modes);
Eric Laurentbe21a942023-11-14 16:05:00 +01001048 sortSupportedLatencyModes_l();
Eric Laurentee398ad2022-05-03 18:19:35 +02001049 checkSensorsState_l();
1050 }
1051}
1052
Eric Laurent15903592022-02-24 20:44:36 +01001053void Spatializer::updateActiveTracks(size_t numActiveTracks) {
Andy Hung79eacdb2023-11-30 19:34:24 -08001054 audio_utils::lock_guard lock(mMutex);
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001055 if (mNumActiveTracks != numActiveTracks) {
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001056 mLocalLog.log("%s from %zu to %zu", __func__, mNumActiveTracks, numActiveTracks);
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001057 mNumActiveTracks = numActiveTracks;
1058 checkEngineState_l();
1059 checkSensorsState_l();
1060 }
Eric Laurent15903592022-02-24 20:44:36 +01001061}
1062
Eric Laurentbe21a942023-11-14 16:05:00 +01001063audio_latency_mode_t Spatializer::selectHeadtrackingConnectionMode_l() {
1064 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
1065 return AUDIO_LATENCY_MODE_LOW;
1066 }
1067 // mSupportedLatencyModes is ordered according to system preferences loaded in
1068 // mOrderedLowLatencyModes
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001069 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED;
Eric Laurentbe21a942023-11-14 16:05:00 +01001070 audio_latency_mode_t requestedLatencyMode = mSupportedLatencyModes[0];
1071 if (requestedLatencyMode == AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_HARDWARE) {
1072 if (mSupportedHeadtrackingConnectionModes.find(
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001073 HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_TUNNEL)
Eric Laurentbe21a942023-11-14 16:05:00 +01001074 != mSupportedHeadtrackingConnectionModes.end()) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001075 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_TUNNEL;
Eric Laurentbe21a942023-11-14 16:05:00 +01001076 } else if (mSupportedHeadtrackingConnectionModes.find(
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001077 HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_SW)
Eric Laurentbe21a942023-11-14 16:05:00 +01001078 != mSupportedHeadtrackingConnectionModes.end()) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001079 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_SW;
Eric Laurentbe21a942023-11-14 16:05:00 +01001080 } else {
1081 // if the engine does not support direct reading of IMU data, do not allow
1082 // DYNAMIC_SPATIAL_AUDIO_HARDWARE mode and fallback to next mode
1083 if (mSupportedLatencyModes.size() > 1) {
1084 requestedLatencyMode = mSupportedLatencyModes[1];
1085 } else {
1086 // If only DYNAMIC_SPATIAL_AUDIO_HARDWARE mode is reported by the
1087 // HAL and the engine does not support it, assert as this is a
1088 // product configuration error
1089 LOG_ALWAYS_FATAL("%s: the audio HAL reported only low latency with"
1090 "HW HID tunneling but the spatializer does not support it",
1091 __func__);
1092 }
1093 }
1094 }
1095 return requestedLatencyMode;
1096}
1097
Eric Laurent9249d342022-03-18 11:55:56 +01001098void Spatializer::checkSensorsState_l() {
Eric Laurent82d1f452024-03-07 18:41:00 +01001099 mRequestedLatencyMode = AUDIO_LATENCY_MODE_FREE;
Andy Hungdae53212022-10-21 17:30:30 -07001100 const bool supportsSetLatencyMode = !mSupportedLatencyModes.empty();
Eric Laurentbe21a942023-11-14 16:05:00 +01001101 bool supportsLowLatencyMode;
1102 if (com::android::media::audio::dsa_over_bt_le_audio()) {
1103 // mSupportedLatencyModes is ordered with MODE_FREE always at the end:
1104 // the first entry is never MODE_FREE if at least one low ltency mode is supported.
1105 supportsLowLatencyMode = supportsSetLatencyMode
1106 && mSupportedLatencyModes[0] != AUDIO_LATENCY_MODE_FREE;
1107 } else {
1108 supportsLowLatencyMode = supportsSetLatencyMode && std::find(
Andy Hungdae53212022-10-21 17:30:30 -07001109 mSupportedLatencyModes.begin(), mSupportedLatencyModes.end(),
1110 AUDIO_LATENCY_MODE_LOW) != mSupportedLatencyModes.end();
Eric Laurentbe21a942023-11-14 16:05:00 +01001111 }
Eric Laurentbdecc052022-10-21 11:28:32 +02001112 if (mSupportsHeadTracking) {
1113 if (mPoseController != nullptr) {
Andy Hungdae53212022-10-21 17:30:30 -07001114 // TODO(b/253297301, b/255433067) reenable low latency condition check
1115 // for Head Tracking after Bluetooth HAL supports it correctly.
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001116 if (mNumActiveTracks > 0 && mLevel != Spatialization::Level::NONE
Eric Laurentbe21a942023-11-14 16:05:00 +01001117 && mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
1118 && mHeadSensor != SpatializerPoseController::INVALID_SENSOR) {
1119 if (supportsLowLatencyMode) {
Eric Laurent82d1f452024-03-07 18:41:00 +01001120 mRequestedLatencyMode = selectHeadtrackingConnectionMode_l();
Eric Laurentbe21a942023-11-14 16:05:00 +01001121 }
Eric Laurentbdecc052022-10-21 11:28:32 +02001122 if (mEngine != nullptr) {
1123 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001124 std::vector<HeadTracking::Mode>{mActualHeadTrackingMode});
Eric Laurentbe21a942023-11-14 16:05:00 +01001125 setEngineHeadtrackingConnectionMode_l();
Eric Laurentbdecc052022-10-21 11:28:32 +02001126 }
Eric Laurentbe21a942023-11-14 16:05:00 +01001127 // TODO: b/307588546: configure mPoseController according to selected
1128 // mHeadtrackingConnectionMode
Eric Laurentbdecc052022-10-21 11:28:32 +02001129 mPoseController->setHeadSensor(mHeadSensor);
1130 mPoseController->setScreenSensor(mScreenSensor);
Eric Laurentbdecc052022-10-21 11:28:32 +02001131 } else {
1132 mPoseController->setHeadSensor(SpatializerPoseController::INVALID_SENSOR);
1133 mPoseController->setScreenSensor(SpatializerPoseController::INVALID_SENSOR);
1134 resetEngineHeadPose_l();
1135 }
Eric Laurent15903592022-02-24 20:44:36 +01001136 } else {
Eric Laurentbdecc052022-10-21 11:28:32 +02001137 resetEngineHeadPose_l();
Eric Laurent15903592022-02-24 20:44:36 +01001138 }
1139 }
Andy Hungdae53212022-10-21 17:30:30 -07001140 if (mOutput != AUDIO_IO_HANDLE_NONE && supportsSetLatencyMode) {
Andy Hung4bd53e72022-11-17 17:21:45 -08001141 const status_t status =
Eric Laurent82d1f452024-03-07 18:41:00 +01001142 AudioSystem::setRequestedLatencyMode(mOutput, mRequestedLatencyMode);
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001143 ALOGD("%s: setRequestedLatencyMode for output thread(%d) to %s returned %d", __func__,
Eric Laurent82d1f452024-03-07 18:41:00 +01001144 mOutput, toString(mRequestedLatencyMode).c_str(), status);
Eric Laurentee398ad2022-05-03 18:19:35 +02001145 }
Eric Laurent15903592022-02-24 20:44:36 +01001146}
1147
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001148void Spatializer::checkEngineState_l() {
1149 if (mEngine != nullptr) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001150 if (mLevel != Spatialization::Level::NONE && mNumActiveTracks > 0) {
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001151 mEngine->setEnabled(true);
1152 setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001153 std::vector<Spatialization::Level>{mLevel});
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001154 } else {
1155 setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001156 std::vector<Spatialization::Level>{Spatialization::Level::NONE});
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001157 mEngine->setEnabled(false);
1158 }
1159 }
1160}
1161
Eric Laurent11094172022-04-05 18:27:42 +02001162void Spatializer::checkPoseController_l() {
1163 bool isControllerNeeded = mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
1164 && mHeadSensor != SpatializerPoseController::INVALID_SENSOR;
1165
1166 if (isControllerNeeded && mPoseController == nullptr) {
1167 mPoseController = std::make_shared<SpatializerPoseController>(
1168 static_cast<SpatializerPoseController::Listener*>(this),
Eric Laurente51f80e2022-04-14 10:20:38 +02001169 10ms, std::nullopt);
Eric Laurent11094172022-04-05 18:27:42 +02001170 LOG_ALWAYS_FATAL_IF(mPoseController == nullptr,
1171 "%s could not allocate pose controller", __func__);
1172 mPoseController->setDisplayOrientation(mDisplayOrientation);
1173 } else if (!isControllerNeeded && mPoseController != nullptr) {
1174 mPoseController.reset();
Eric Laurentbdecc052022-10-21 11:28:32 +02001175 resetEngineHeadPose_l();
Eric Laurent11094172022-04-05 18:27:42 +02001176 }
1177 if (mPoseController != nullptr) {
1178 mPoseController->setDesiredMode(mDesiredHeadTrackingMode);
1179 }
1180}
1181
Eric Laurent2be8b292021-08-23 09:44:33 -07001182void Spatializer::calculateHeadPose() {
1183 ALOGV("%s", __func__);
Andy Hung79eacdb2023-11-30 19:34:24 -08001184 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -07001185 if (mPoseController != nullptr) {
1186 mPoseController->calculateAsync();
1187 }
1188}
Eric Laurent6d607012021-07-05 11:54:40 +02001189
Atneya Nair20e6cc82022-05-17 20:12:37 -04001190void Spatializer::onFramesProcessed(int32_t framesProcessed) {
Eric Laurent8a4259f2021-09-14 16:04:00 +02001191 sp<AMessage> msg =
1192 new AMessage(EngineCallbackHandler::kWhatOnFramesProcessed, mHandler);
Atneya Nair20e6cc82022-05-17 20:12:37 -04001193 msg->setInt32(EngineCallbackHandler::kNumFramesKey, framesProcessed);
Eric Laurent8a4259f2021-09-14 16:04:00 +02001194 msg->post();
1195}
1196
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001197std::string Spatializer::toString(unsigned level) const {
Andy Hung560addd2023-01-30 11:58:44 -08001198 std::string prefixSpace(level, ' ');
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001199 std::string ss = prefixSpace + "Spatializer:\n";
1200 bool needUnlock = false;
1201
1202 prefixSpace += ' ';
Andy Hung79eacdb2023-11-30 19:34:24 -08001203 if (!mMutex.try_lock()) {
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001204 // dumpsys even try_lock failed, information dump can be useful although may not accurate
1205 ss.append(prefixSpace).append("try_lock failed, dumpsys below maybe INACCURATE!\n");
1206 } else {
1207 needUnlock = true;
1208 }
1209
1210 // Spatializer class information.
1211 // 1. Capabilities (mLevels, mHeadTrackingModes, mSpatializationModes, mChannelMasks, etc)
1212 ss.append(prefixSpace).append("Supported levels: [");
1213 for (auto& level : mLevels) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001214 base::StringAppendF(&ss, " %s", ToString(level).c_str());
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001215 }
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001216 base::StringAppendF(&ss, "], mLevel: %s", ToString(mLevel).c_str());
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001217
1218 base::StringAppendF(&ss, "\n%smHeadTrackingModes: [", prefixSpace.c_str());
1219 for (auto& mode : mHeadTrackingModes) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001220 base::StringAppendF(&ss, " %s", ToString(mode).c_str());
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001221 }
1222 base::StringAppendF(&ss, "], Desired: %s, Actual %s\n",
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001223 ToString(mDesiredHeadTrackingMode).c_str(),
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001224 ToString(mActualHeadTrackingMode).c_str());
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001225
1226 base::StringAppendF(&ss, "%smSpatializationModes: [", prefixSpace.c_str());
1227 for (auto& mode : mSpatializationModes) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001228 base::StringAppendF(&ss, " %s", ToString(mode).c_str());
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001229 }
1230 ss += "]\n";
1231
1232 base::StringAppendF(&ss, "%smChannelMasks: ", prefixSpace.c_str());
1233 for (auto& mask : mChannelMasks) {
1234 base::StringAppendF(&ss, "%s", audio_channel_out_mask_to_string(mask));
1235 }
1236 base::StringAppendF(&ss, "\n%smSupportsHeadTracking: %s\n", prefixSpace.c_str(),
1237 mSupportsHeadTracking ? "true" : "false");
1238 // 2. Settings (Output, tracks)
1239 base::StringAppendF(&ss, "%smNumActiveTracks: %zu\n", prefixSpace.c_str(), mNumActiveTracks);
1240 base::StringAppendF(&ss, "%sOutputStreamHandle: %d\n", prefixSpace.c_str(), (int)mOutput);
1241
1242 // 3. Sensors, Effect information.
1243 base::StringAppendF(&ss, "%sHeadSensorHandle: 0x%08x\n", prefixSpace.c_str(), mHeadSensor);
1244 base::StringAppendF(&ss, "%sScreenSensorHandle: 0x%08x\n", prefixSpace.c_str(), mScreenSensor);
1245 base::StringAppendF(&ss, "%sEffectHandle: %p\n", prefixSpace.c_str(), mEngine.get());
1246 base::StringAppendF(&ss, "%sDisplayOrientation: %f\n", prefixSpace.c_str(),
1247 mDisplayOrientation);
1248
Andy Hung481bfe32023-12-18 14:00:29 -08001249 // 4. Show flag or property state.
1250 base::StringAppendF(&ss, "%sStereo Spatialization: %s\n", prefixSpace.c_str(),
1251 com_android_media_audio_stereo_spatialization() ? "true" : "false");
1252
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001253 ss.append(prefixSpace + "CommandLog:\n");
1254 ss += mLocalLog.dumpToString((prefixSpace + " ").c_str(), mMaxLocalLogLine);
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001255
1256 // PostController dump.
1257 if (mPoseController != nullptr) {
Andy Hung560addd2023-01-30 11:58:44 -08001258 ss.append(mPoseController->toString(level + 1))
1259 .append(prefixSpace)
Andy Hunga69459c2023-02-27 13:36:23 -08001260 .append("Pose (active stage-to-head) [tx, ty, tz : pitch, roll, yaw]:\n")
Andy Hung560addd2023-01-30 11:58:44 -08001261 .append(prefixSpace)
1262 .append(" PerMinuteHistory:\n")
Andy Hunga69459c2023-02-27 13:36:23 -08001263 .append(mPoseDurableRecorder.toString(level + 3))
Andy Hung560addd2023-01-30 11:58:44 -08001264 .append(prefixSpace)
1265 .append(" PerSecondHistory:\n")
Andy Hunga69459c2023-02-27 13:36:23 -08001266 .append(mPoseRecorder.toString(level + 3));
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001267 } else {
1268 ss.append(prefixSpace).append("SpatializerPoseController not exist\n");
1269 }
1270
1271 if (needUnlock) {
Andy Hung79eacdb2023-11-30 19:34:24 -08001272 mMutex.unlock();
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001273 }
1274 return ss;
1275}
1276
Eric Laurent6d607012021-07-05 11:54:40 +02001277} // namespace android