blob: c98f8df124dce769adfc8eaa209c2a9af2f5c422 [file] [log] [blame]
Eric Laurent6d607012021-07-05 11:54:40 +02001/*
2**
3** Copyright 2021, The Android Open Source Project
4**
5** Licensed under the Apache License, Version 2.0 (the "License");
6** you may not use this file except in compliance with the License.
7** You may obtain a copy of the License at
8**
9** http://www.apache.org/licenses/LICENSE-2.0
10**
11** Unless required by applicable law or agreed to in writing, software
12** distributed under the License is distributed on an "AS IS" BASIS,
13** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14** See the License for the specific language governing permissions and
15** limitations under the License.
16*/
17
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +000018#include <string>
Eric Laurent6d607012021-07-05 11:54:40 +020019#define LOG_TAG "Spatializer"
20//#define LOG_NDEBUG 0
21#include <utils/Log.h>
22
Andy Hung2490b2d2023-03-09 20:45:36 -080023#include <algorithm>
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +000024#include <inttypes.h>
Eric Laurent6d607012021-07-05 11:54:40 +020025#include <limits.h>
26#include <stdint.h>
27#include <sys/types.h>
28
29#include <android/content/AttributionSourceState.h>
Eric Laurentbe21a942023-11-14 16:05:00 +010030#include <android/sysprop/BluetoothProperties.sysprop.h>
Eric Laurent6d607012021-07-05 11:54:40 +020031#include <audio_utils/fixedfft.h>
Eric Laurentbe21a942023-11-14 16:05:00 +010032#include <com_android_media_audio.h>
Eric Laurent6d607012021-07-05 11:54:40 +020033#include <cutils/bitops.h>
Nikhil Bhanu8f4ea772024-01-31 17:15:52 -080034#include <cutils/properties.h>
Eric Laurent2be8b292021-08-23 09:44:33 -070035#include <hardware/sensors.h>
Eric Laurent8a4259f2021-09-14 16:04:00 +020036#include <media/stagefright/foundation/AHandler.h>
37#include <media/stagefright/foundation/AMessage.h>
Andy Hunga461a002022-05-17 10:36:02 -070038#include <media/MediaMetricsItem.h>
Andy Hung560addd2023-01-30 11:58:44 -080039#include <media/QuaternionUtil.h>
Eric Laurent8a4259f2021-09-14 16:04:00 +020040#include <media/ShmemCompat.h>
Andy Hung41ccf7f2022-12-14 14:25:49 -080041#include <mediautils/SchedulingPolicyService.h>
Eric Laurent6d607012021-07-05 11:54:40 +020042#include <mediautils/ServiceUtilities.h>
43#include <utils/Thread.h>
44
45#include "Spatializer.h"
46
47namespace android {
48
Eric Laurent6d607012021-07-05 11:54:40 +020049using aidl_utils::binderStatusFromStatusT;
Shunkai Yao49bc61f2023-10-10 19:31:10 +000050using aidl_utils::statusTFromBinderStatus;
Eric Laurent6d607012021-07-05 11:54:40 +020051using android::content::AttributionSourceState;
52using binder::Status;
Shunkai Yao2ee06f62024-02-16 22:00:22 +000053using internal::ToString;
Eric Laurent2be8b292021-08-23 09:44:33 -070054using media::HeadTrackingMode;
55using media::Pose3f;
Eric Laurent2be8b292021-08-23 09:44:33 -070056using media::SensorPoseProvider;
Shunkai Yao49bc61f2023-10-10 19:31:10 +000057using media::audio::common::HeadTracking;
58using media::audio::common::Spatialization;
Eric Laurent2be8b292021-08-23 09:44:33 -070059
Eric Laurent2be8b292021-08-23 09:44:33 -070060using namespace std::chrono_literals;
Eric Laurent6d607012021-07-05 11:54:40 +020061
62#define VALUE_OR_RETURN_BINDER_STATUS(x) \
63 ({ auto _tmp = (x); \
64 if (!_tmp.ok()) return aidl_utils::binderStatusFromStatusT(_tmp.error()); \
65 std::move(_tmp.value()); })
66
Andy Hung4e2547c2022-08-29 14:14:58 -070067static audio_channel_mask_t getMaxChannelMask(
68 const std::vector<audio_channel_mask_t>& masks, size_t channelLimit = SIZE_MAX) {
Andy Hunga461a002022-05-17 10:36:02 -070069 uint32_t maxCount = 0;
70 audio_channel_mask_t maxMask = AUDIO_CHANNEL_NONE;
71 for (auto mask : masks) {
72 const size_t count = audio_channel_count_from_out_mask(mask);
Andy Hung4e2547c2022-08-29 14:14:58 -070073 if (count > channelLimit) continue; // ignore masks greater than channelLimit
Andy Hunga461a002022-05-17 10:36:02 -070074 if (count > maxCount) {
75 maxMask = mask;
76 maxCount = count;
77 }
78 }
79 return maxMask;
80}
81
Andy Hung560addd2023-01-30 11:58:44 -080082static std::vector<float> recordFromTranslationRotationVector(
83 const std::vector<float>& trVector) {
84 auto headToStageOpt = Pose3f::fromVector(trVector);
85 if (!headToStageOpt) return {};
86
87 const auto stageToHead = headToStageOpt.value().inverse();
88 const auto stageToHeadTranslation = stageToHead.translation();
Andy Hunga367cb22023-01-30 11:58:44 -080089 constexpr float RAD_TO_DEGREE = 180.f / M_PI;
90 std::vector<float> record{
Andy Hung560addd2023-01-30 11:58:44 -080091 stageToHeadTranslation[0], stageToHeadTranslation[1], stageToHeadTranslation[2],
92 0.f, 0.f, 0.f};
93 media::quaternionToAngles(stageToHead.rotation(), &record[3], &record[4], &record[5]);
94 record[3] *= RAD_TO_DEGREE;
95 record[4] *= RAD_TO_DEGREE;
96 record[5] *= RAD_TO_DEGREE;
Andy Hunga367cb22023-01-30 11:58:44 -080097 return record;
98}
99
Andy Hung2490b2d2023-03-09 20:45:36 -0800100template<typename T>
101static constexpr const T& safe_clamp(const T& value, const T& low, const T& high) {
102 if constexpr (std::is_floating_point_v<T>) {
103 return value != value /* constexpr isnan */
104 ? low : std::clamp(value, low, high);
105 } else /* constexpr */ {
106 return std::clamp(value, low, high);
107 }
108}
109
Eric Laurent6d607012021-07-05 11:54:40 +0200110// ---------------------------------------------------------------------------
111
Eric Laurent8a4259f2021-09-14 16:04:00 +0200112class Spatializer::EngineCallbackHandler : public AHandler {
113public:
114 EngineCallbackHandler(wp<Spatializer> spatializer)
115 : mSpatializer(spatializer) {
116 }
117
118 enum {
119 // Device state callbacks
120 kWhatOnFramesProcessed, // AudioEffect::EVENT_FRAMES_PROCESSED
121 kWhatOnHeadToStagePose, // SpatializerPoseController::Listener::onHeadToStagePose
122 kWhatOnActualModeChange, // SpatializerPoseController::Listener::onActualModeChange
Eric Laurent9c04de92022-07-20 13:49:47 +0200123 kWhatOnLatencyModesChanged, // Spatializer::onSupportedLatencyModesChanged
Eric Laurent8a4259f2021-09-14 16:04:00 +0200124 };
125 static constexpr const char *kNumFramesKey = "numFrames";
126 static constexpr const char *kModeKey = "mode";
127 static constexpr const char *kTranslation0Key = "translation0";
128 static constexpr const char *kTranslation1Key = "translation1";
129 static constexpr const char *kTranslation2Key = "translation2";
130 static constexpr const char *kRotation0Key = "rotation0";
131 static constexpr const char *kRotation1Key = "rotation1";
132 static constexpr const char *kRotation2Key = "rotation2";
Eric Laurent9c04de92022-07-20 13:49:47 +0200133 static constexpr const char *kLatencyModesKey = "latencyModes";
134
135 class LatencyModes : public RefBase {
136 public:
137 LatencyModes(audio_io_handle_t output,
138 const std::vector<audio_latency_mode_t>& latencyModes)
139 : mOutput(output), mLatencyModes(latencyModes) {}
140 ~LatencyModes() = default;
141
142 audio_io_handle_t mOutput;
143 std::vector<audio_latency_mode_t> mLatencyModes;
144 };
Eric Laurent8a4259f2021-09-14 16:04:00 +0200145
146 void onMessageReceived(const sp<AMessage> &msg) override {
Andy Hung41ccf7f2022-12-14 14:25:49 -0800147 // No ALooper method to get the tid so update
148 // Spatializer priority on the first message received.
149 std::call_once(mPrioritySetFlag, [](){
150 const pid_t pid = getpid();
151 const pid_t tid = gettid();
152 (void)requestSpatializerPriority(pid, tid);
153 });
154
Eric Laurent9c04de92022-07-20 13:49:47 +0200155 sp<Spatializer> spatializer = mSpatializer.promote();
156 if (spatializer == nullptr) {
157 ALOGW("%s: Cannot promote spatializer", __func__);
158 return;
159 }
Eric Laurent8a4259f2021-09-14 16:04:00 +0200160 switch (msg->what()) {
161 case kWhatOnFramesProcessed: {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200162 int numFrames;
163 if (!msg->findInt32(kNumFramesKey, &numFrames)) {
164 ALOGE("%s: Cannot find num frames!", __func__);
165 return;
166 }
167 if (numFrames > 0) {
168 spatializer->calculateHeadPose();
169 }
170 } break;
171 case kWhatOnHeadToStagePose: {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200172 std::vector<float> headToStage(sHeadPoseKeys.size());
173 for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
174 if (!msg->findFloat(sHeadPoseKeys[i], &headToStage[i])) {
175 ALOGE("%s: Cannot find kTranslation0Key!", __func__);
176 return;
177 }
178 }
179 spatializer->onHeadToStagePoseMsg(headToStage);
180 } break;
181 case kWhatOnActualModeChange: {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200182 int mode;
Eric Laurent9c04de92022-07-20 13:49:47 +0200183 if (!msg->findInt32(kModeKey, &mode)) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200184 ALOGE("%s: Cannot find actualMode!", __func__);
185 return;
186 }
187 spatializer->onActualModeChangeMsg(static_cast<HeadTrackingMode>(mode));
188 } break;
Eric Laurent9c04de92022-07-20 13:49:47 +0200189
190 case kWhatOnLatencyModesChanged: {
191 sp<RefBase> object;
192 if (!msg->findObject(kLatencyModesKey, &object)) {
193 ALOGE("%s: Cannot find latency modes!", __func__);
194 return;
195 }
196 sp<LatencyModes> latencyModes = static_cast<LatencyModes*>(object.get());
197 spatializer->onSupportedLatencyModesChangedMsg(
198 latencyModes->mOutput, std::move(latencyModes->mLatencyModes));
199 } break;
200
Eric Laurent8a4259f2021-09-14 16:04:00 +0200201 default:
202 LOG_ALWAYS_FATAL("Invalid callback message %d", msg->what());
203 }
204 }
205private:
206 wp<Spatializer> mSpatializer;
Andy Hung41ccf7f2022-12-14 14:25:49 -0800207 std::once_flag mPrioritySetFlag;
Eric Laurent8a4259f2021-09-14 16:04:00 +0200208};
209
210const std::vector<const char *> Spatializer::sHeadPoseKeys = {
211 Spatializer::EngineCallbackHandler::kTranslation0Key,
212 Spatializer::EngineCallbackHandler::kTranslation1Key,
213 Spatializer::EngineCallbackHandler::kTranslation2Key,
214 Spatializer::EngineCallbackHandler::kRotation0Key,
215 Spatializer::EngineCallbackHandler::kRotation1Key,
216 Spatializer::EngineCallbackHandler::kRotation2Key,
217};
218
Eric Laurentbe21a942023-11-14 16:05:00 +0100219// Mapping table between strings read form property bluetooth.core.le.dsa_transport_preference
220// and low latency modes emums.
221//TODO b/273373363: use AIDL enum when available
222const std::map<std::string, audio_latency_mode_t> Spatializer::sStringToLatencyModeMap = {
223 {"le-acl", AUDIO_LATENCY_MODE_LOW},
224 {"iso-sw", AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_SOFTWARE},
225 {"iso-hw", AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_HARDWARE},
226};
227
228void Spatializer::loadOrderedLowLatencyModes() {
229 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
230 return;
231 }
232 auto latencyModesStrs = android::sysprop::BluetoothProperties::dsa_transport_preference();
Andy Hung79eacdb2023-11-30 19:34:24 -0800233 audio_utils::lock_guard lock(mMutex);
Eric Laurentbe21a942023-11-14 16:05:00 +0100234 // First load preferred low latency modes ordered from the property
235 for (auto str : latencyModesStrs) {
236 if (!str.has_value()) continue;
237 if (auto it = sStringToLatencyModeMap.find(str.value());
238 it != sStringToLatencyModeMap.end()) {
239 mOrderedLowLatencyModes.push_back(it->second);
240 }
241 }
242 // Then add unlisted latency modes at the end of the ordered list
243 for (auto it : sStringToLatencyModeMap) {
244 if (std::find(mOrderedLowLatencyModes.begin(), mOrderedLowLatencyModes.end(), it.second)
245 == mOrderedLowLatencyModes.end()) {
246 mOrderedLowLatencyModes.push_back(it.second);
247 }
248 }
249}
250
Eric Laurent8a4259f2021-09-14 16:04:00 +0200251// ---------------------------------------------------------------------------
Shunkai Yao8d6489a2023-04-18 23:14:25 +0000252sp<Spatializer> Spatializer::create(SpatializerPolicyCallback* callback,
253 const sp<EffectsFactoryHalInterface>& effectsFactoryHal) {
Eric Laurent6d607012021-07-05 11:54:40 +0200254 sp<Spatializer> spatializer;
255
Eric Laurent6d607012021-07-05 11:54:40 +0200256 if (effectsFactoryHal == nullptr) {
257 ALOGW("%s failed to create effect factory interface", __func__);
258 return spatializer;
259 }
260
261 std::vector<effect_descriptor_t> descriptors;
Shunkai Yao8d6489a2023-04-18 23:14:25 +0000262 status_t status = effectsFactoryHal->getDescriptors(FX_IID_SPATIALIZER, &descriptors);
Eric Laurent6d607012021-07-05 11:54:40 +0200263 if (status != NO_ERROR) {
264 ALOGW("%s failed to get spatializer descriptor, error %d", __func__, status);
265 return spatializer;
266 }
267 ALOG_ASSERT(!descriptors.empty(),
268 "%s getDescriptors() returned no error but empty list", __func__);
269
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000270 // TODO: get supported spatialization modes from FX engine or descriptor
Eric Laurent6d607012021-07-05 11:54:40 +0200271 sp<EffectHalInterface> effect;
272 status = effectsFactoryHal->createEffect(&descriptors[0].uuid, AUDIO_SESSION_OUTPUT_STAGE,
273 AUDIO_IO_HANDLE_NONE, AUDIO_PORT_HANDLE_NONE, &effect);
Mikhail Naganov89c22e42023-06-14 15:49:59 -0700274 ALOGI("%s FX create status %d effect %p", __func__, status, effect.get());
Eric Laurent6d607012021-07-05 11:54:40 +0200275
276 if (status == NO_ERROR && effect != nullptr) {
277 spatializer = new Spatializer(descriptors[0], callback);
Eric Laurent2be8b292021-08-23 09:44:33 -0700278 if (spatializer->loadEngineConfiguration(effect) != NO_ERROR) {
279 spatializer.clear();
Mikhail Naganov89c22e42023-06-14 15:49:59 -0700280 ALOGW("%s loadEngine error: %d effect %p", __func__, status, effect.get());
Andy Hung4e2547c2022-08-29 14:14:58 -0700281 } else {
Eric Laurentbe21a942023-11-14 16:05:00 +0100282 spatializer->loadOrderedLowLatencyModes();
Mikhail Naganov89c22e42023-06-14 15:49:59 -0700283 spatializer->mLocalLog.log("%s with effect Id %p", __func__, effect.get());
Eric Laurent2be8b292021-08-23 09:44:33 -0700284 }
Eric Laurent6d607012021-07-05 11:54:40 +0200285 }
286
287 return spatializer;
288}
289
Eric Laurent2be8b292021-08-23 09:44:33 -0700290Spatializer::Spatializer(effect_descriptor_t engineDescriptor, SpatializerPolicyCallback* callback)
291 : mEngineDescriptor(engineDescriptor),
292 mPolicyCallback(callback) {
Eric Laurent6d607012021-07-05 11:54:40 +0200293 ALOGV("%s", __func__);
Andy Hung225aef62022-12-06 16:33:20 -0800294 setMinSchedulerPolicy(SCHED_NORMAL, ANDROID_PRIORITY_AUDIO);
Andy Hung58b01b12024-03-26 18:04:29 -0700295 setInheritRt(true);
Eric Laurent6d607012021-07-05 11:54:40 +0200296}
297
Eric Laurent8a4259f2021-09-14 16:04:00 +0200298void Spatializer::onFirstRef() {
299 mLooper = new ALooper;
300 mLooper->setName("Spatializer-looper");
301 mLooper->start(
302 /*runOnCallingThread*/false,
303 /*canCallJava*/ false,
Andy Hungbf1777b2022-11-07 20:09:20 -0800304 PRIORITY_URGENT_AUDIO);
Eric Laurent8a4259f2021-09-14 16:04:00 +0200305
306 mHandler = new EngineCallbackHandler(this);
307 mLooper->registerHandler(mHandler);
308}
309
Eric Laurent6d607012021-07-05 11:54:40 +0200310Spatializer::~Spatializer() {
311 ALOGV("%s", __func__);
Eric Laurent8a4259f2021-09-14 16:04:00 +0200312 if (mLooper != nullptr) {
313 mLooper->stop();
314 mLooper->unregisterHandler(mHandler->id());
315 }
316 mLooper.clear();
317 mHandler.clear();
Eric Laurent6d607012021-07-05 11:54:40 +0200318}
319
Andy Hung8a7ecff2022-08-17 17:27:32 -0700320static std::string channelMaskVectorToString(
321 const std::vector<audio_channel_mask_t>& masks) {
322 std::stringstream ss;
323 for (const auto &mask : masks) {
324 if (ss.tellp() != 0) ss << "|";
325 ss << mask;
326 }
327 return ss.str();
328}
329
Eric Laurent2be8b292021-08-23 09:44:33 -0700330status_t Spatializer::loadEngineConfiguration(sp<EffectHalInterface> effect) {
331 ALOGV("%s", __func__);
332
333 std::vector<bool> supportsHeadTracking;
334 status_t status = getHalParameter<false>(effect, SPATIALIZER_PARAM_HEADTRACKING_SUPPORTED,
335 &supportsHeadTracking);
336 if (status != NO_ERROR) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700337 ALOGW("%s: cannot get SPATIALIZER_PARAM_HEADTRACKING_SUPPORTED", __func__);
Eric Laurent2be8b292021-08-23 09:44:33 -0700338 return status;
339 }
340 mSupportsHeadTracking = supportsHeadTracking[0];
341
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000342 std::vector<Spatialization::Level> spatializationLevels;
Andy Hung119dbdb2022-05-11 19:20:13 -0700343 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_LEVELS,
344 &spatializationLevels);
Eric Laurent2be8b292021-08-23 09:44:33 -0700345 if (status != NO_ERROR) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700346 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_LEVELS", __func__);
Eric Laurent2be8b292021-08-23 09:44:33 -0700347 return status;
348 }
Andy Hung119dbdb2022-05-11 19:20:13 -0700349 bool noneLevelFound = false;
350 bool activeLevelFound = false;
351 for (const auto spatializationLevel : spatializationLevels) {
352 if (!aidl_utils::isValidEnum(spatializationLevel)) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000353 ALOGW("%s: ignoring spatializationLevel:%s", __func__,
354 ToString(spatializationLevel).c_str());
Andy Hung119dbdb2022-05-11 19:20:13 -0700355 continue;
356 }
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000357 if (spatializationLevel == Spatialization::Level::NONE) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700358 noneLevelFound = true;
359 } else {
360 activeLevelFound = true;
361 }
362 // we don't detect duplicates.
363 mLevels.emplace_back(spatializationLevel);
364 }
365 if (!noneLevelFound || !activeLevelFound) {
366 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_LEVELS must include NONE"
367 " and another valid level", __func__);
368 return BAD_VALUE;
369 }
370
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000371 std::vector<Spatialization::Mode> spatializationModes;
Eric Laurent2be8b292021-08-23 09:44:33 -0700372 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES,
Andy Hung119dbdb2022-05-11 19:20:13 -0700373 &spatializationModes);
Eric Laurent2be8b292021-08-23 09:44:33 -0700374 if (status != NO_ERROR) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700375 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES", __func__);
Eric Laurent2be8b292021-08-23 09:44:33 -0700376 return status;
377 }
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000378
Andy Hung119dbdb2022-05-11 19:20:13 -0700379 for (const auto spatializationMode : spatializationModes) {
380 if (!aidl_utils::isValidEnum(spatializationMode)) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000381 ALOGW("%s: ignoring spatializationMode:%s", __func__,
382 ToString(spatializationMode).c_str());
Andy Hung119dbdb2022-05-11 19:20:13 -0700383 continue;
384 }
385 // we don't detect duplicates.
386 mSpatializationModes.emplace_back(spatializationMode);
387 }
388 if (mSpatializationModes.empty()) {
389 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES reports empty", __func__);
390 return BAD_VALUE;
391 }
392
393 std::vector<audio_channel_mask_t> channelMasks;
394 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS,
395 &channelMasks);
396 if (status != NO_ERROR) {
397 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS", __func__);
398 return status;
399 }
400 for (const auto channelMask : channelMasks) {
Nikhil Bhanu8f4ea772024-01-31 17:15:52 -0800401 static const bool stereo_spatialization_enabled =
402 property_get_bool("ro.audio.stereo_spatialization_enabled", false);
Andy Hung481bfe32023-12-18 14:00:29 -0800403 const bool channel_mask_spatialized =
Nikhil Bhanu8f4ea772024-01-31 17:15:52 -0800404 (stereo_spatialization_enabled && com_android_media_audio_stereo_spatialization())
Andy Hung481bfe32023-12-18 14:00:29 -0800405 ? audio_channel_mask_contains_stereo(channelMask)
406 : audio_is_channel_mask_spatialized(channelMask);
407 if (!channel_mask_spatialized) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700408 ALOGW("%s: ignoring channelMask:%#x", __func__, channelMask);
409 continue;
410 }
411 // we don't detect duplicates.
412 mChannelMasks.emplace_back(channelMask);
413 }
414 if (mChannelMasks.empty()) {
415 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS reports empty", __func__);
416 return BAD_VALUE;
417 }
Andy Hunga461a002022-05-17 10:36:02 -0700418
Eric Laurentbe21a942023-11-14 16:05:00 +0100419 if (com::android::media::audio::dsa_over_bt_le_audio()
420 && mSupportsHeadTracking) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000421 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED;
422 std::vector<HeadTracking::ConnectionMode> headtrackingConnectionModes;
Eric Laurentbe21a942023-11-14 16:05:00 +0100423 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_HEADTRACKING_CONNECTION,
424 &headtrackingConnectionModes);
425 if (status == NO_ERROR) {
426 for (const auto htConnectionMode : headtrackingConnectionModes) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000427 if (htConnectionMode < HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED ||
428 htConnectionMode > HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_TUNNEL) {
429 ALOGW("%s: ignoring HT connection mode:%s", __func__,
430 ToString(htConnectionMode).c_str());
Eric Laurentbe21a942023-11-14 16:05:00 +0100431 continue;
432 }
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000433 mSupportedHeadtrackingConnectionModes.insert(htConnectionMode);
Eric Laurentbe21a942023-11-14 16:05:00 +0100434 }
435 ALOGW_IF(mSupportedHeadtrackingConnectionModes.find(
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000436 HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED) ==
437 mSupportedHeadtrackingConnectionModes.end(),
438 "%s: Headtracking FRAMEWORK_PROCESSED not reported", __func__);
Eric Laurentbe21a942023-11-14 16:05:00 +0100439 }
440 }
441
Andy Hunga461a002022-05-17 10:36:02 -0700442 // Currently we expose only RELATIVE_WORLD.
443 // This is a limitation of the head tracking library based on a UX choice.
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000444 mHeadTrackingModes.push_back(HeadTracking::Mode::DISABLED);
Andy Hunga461a002022-05-17 10:36:02 -0700445 if (mSupportsHeadTracking) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000446 mHeadTrackingModes.push_back(HeadTracking::Mode::RELATIVE_WORLD);
Andy Hunga461a002022-05-17 10:36:02 -0700447 }
448 mediametrics::LogItem(mMetricsId)
449 .set(AMEDIAMETRICS_PROP_EVENT, AMEDIAMETRICS_PROP_EVENT_VALUE_CREATE)
Andy Hung8a7ecff2022-08-17 17:27:32 -0700450 .set(AMEDIAMETRICS_PROP_CHANNELMASKS, channelMaskVectorToString(mChannelMasks))
Andy Hunga461a002022-05-17 10:36:02 -0700451 .set(AMEDIAMETRICS_PROP_LEVELS, aidl_utils::enumsToString(mLevels))
452 .set(AMEDIAMETRICS_PROP_MODES, aidl_utils::enumsToString(mSpatializationModes))
453 .set(AMEDIAMETRICS_PROP_HEADTRACKINGMODES, aidl_utils::enumsToString(mHeadTrackingModes))
454 .set(AMEDIAMETRICS_PROP_STATUS, (int32_t)status)
455 .record();
Andy Hung119dbdb2022-05-11 19:20:13 -0700456 return NO_ERROR;
Eric Laurent2be8b292021-08-23 09:44:33 -0700457}
458
Andy Hung8aa43c02022-09-13 18:53:06 -0700459/* static */
460void Spatializer::sendEmptyCreateSpatializerMetricWithStatus(status_t status) {
461 mediametrics::LogItem(kDefaultMetricsId)
462 .set(AMEDIAMETRICS_PROP_EVENT, AMEDIAMETRICS_PROP_EVENT_VALUE_CREATE)
463 .set(AMEDIAMETRICS_PROP_CHANNELMASKS, "")
464 .set(AMEDIAMETRICS_PROP_LEVELS, "")
465 .set(AMEDIAMETRICS_PROP_MODES, "")
466 .set(AMEDIAMETRICS_PROP_HEADTRACKINGMODES, "")
467 .set(AMEDIAMETRICS_PROP_STATUS, (int32_t)status)
468 .record();
469}
470
Eric Laurent2be8b292021-08-23 09:44:33 -0700471/** Gets the channel mask, sampling rate and format set for the spatializer input. */
472audio_config_base_t Spatializer::getAudioInConfig() const {
Andy Hung79eacdb2023-11-30 19:34:24 -0800473 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700474 audio_config_base_t config = AUDIO_CONFIG_BASE_INITIALIZER;
475 // For now use highest supported channel count
Andy Hung4e2547c2022-08-29 14:14:58 -0700476 config.channel_mask = getMaxChannelMask(mChannelMasks, FCC_LIMIT);
Eric Laurent2be8b292021-08-23 09:44:33 -0700477 return config;
478}
479
Eric Laurent6d607012021-07-05 11:54:40 +0200480status_t Spatializer::registerCallback(
481 const sp<media::INativeSpatializerCallback>& callback) {
Andy Hung79eacdb2023-11-30 19:34:24 -0800482 audio_utils::lock_guard lock(mMutex);
Eric Laurent6d607012021-07-05 11:54:40 +0200483 if (callback == nullptr) {
484 return BAD_VALUE;
485 }
486
Eric Laurentd6bee3a2022-08-31 16:07:50 +0200487 if (mSpatializerCallback != nullptr) {
488 if (IInterface::asBinder(callback) == IInterface::asBinder(mSpatializerCallback)) {
489 ALOGW("%s: Registering callback %p again",
490 __func__, mSpatializerCallback.get());
491 return NO_ERROR;
492 }
493 ALOGE("%s: Already one client registered with callback %p",
494 __func__, mSpatializerCallback.get());
495 return INVALID_OPERATION;
496 }
497
Eric Laurent6d607012021-07-05 11:54:40 +0200498 sp<IBinder> binder = IInterface::asBinder(callback);
499 status_t status = binder->linkToDeath(this);
500 if (status == NO_ERROR) {
501 mSpatializerCallback = callback;
502 }
503 ALOGV("%s status %d", __func__, status);
504 return status;
505}
506
507// IBinder::DeathRecipient
508void Spatializer::binderDied(__unused const wp<IBinder> &who) {
509 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800510 audio_utils::lock_guard lock(mMutex);
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000511 mLevel = Spatialization::Level::NONE;
Eric Laurent6d607012021-07-05 11:54:40 +0200512 mSpatializerCallback.clear();
513 }
514 ALOGV("%s", __func__);
515 mPolicyCallback->onCheckSpatializer();
516}
517
518// ISpatializer
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000519Status Spatializer::getSupportedLevels(std::vector<Spatialization::Level> *levels) {
Eric Laurent6d607012021-07-05 11:54:40 +0200520 ALOGV("%s", __func__);
521 if (levels == nullptr) {
522 return binderStatusFromStatusT(BAD_VALUE);
523 }
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000524 // Spatialization::Level::NONE is already required from the effect or we don't load it.
Eric Laurent2be8b292021-08-23 09:44:33 -0700525 levels->insert(levels->end(), mLevels.begin(), mLevels.end());
Eric Laurent6d607012021-07-05 11:54:40 +0200526 return Status::ok();
527}
528
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000529Status Spatializer::setLevel(Spatialization::Level level) {
530 ALOGV("%s level %s", __func__, ToString(level).c_str());
531 mLocalLog.log("%s with %s", __func__, ToString(level).c_str());
532 if (level != Spatialization::Level::NONE
Eric Laurent2be8b292021-08-23 09:44:33 -0700533 && std::find(mLevels.begin(), mLevels.end(), level) == mLevels.end()) {
Eric Laurent6d607012021-07-05 11:54:40 +0200534 return binderStatusFromStatusT(BAD_VALUE);
535 }
536 sp<media::INativeSpatializerCallback> callback;
537 bool levelChanged = false;
538 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800539 audio_utils::lock_guard lock(mMutex);
Eric Laurent6d607012021-07-05 11:54:40 +0200540 levelChanged = mLevel != level;
541 mLevel = level;
542 callback = mSpatializerCallback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700543
544 if (levelChanged && mEngine != nullptr) {
Eric Laurent7ea0d1b2022-04-01 14:23:44 +0200545 checkEngineState_l();
Eric Laurent2be8b292021-08-23 09:44:33 -0700546 }
Eric Laurent9249d342022-03-18 11:55:56 +0100547 checkSensorsState_l();
Eric Laurent6d607012021-07-05 11:54:40 +0200548 }
549
550 if (levelChanged) {
551 mPolicyCallback->onCheckSpatializer();
552 if (callback != nullptr) {
553 callback->onLevelChanged(level);
554 }
555 }
556 return Status::ok();
557}
558
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000559Status Spatializer::getLevel(Spatialization::Level *level) {
Eric Laurent6d607012021-07-05 11:54:40 +0200560 if (level == nullptr) {
561 return binderStatusFromStatusT(BAD_VALUE);
562 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800563 audio_utils::lock_guard lock(mMutex);
Eric Laurent6d607012021-07-05 11:54:40 +0200564 *level = mLevel;
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000565 ALOGV("%s level %s", __func__, ToString(*level).c_str());
Eric Laurent6d607012021-07-05 11:54:40 +0200566 return Status::ok();
567}
568
Eric Laurentc87402b2021-09-17 16:49:42 +0200569Status Spatializer::isHeadTrackingSupported(bool *supports) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000570 ALOGV("%s mSupportsHeadTracking %s", __func__, ToString(mSupportsHeadTracking).c_str());
Eric Laurentc87402b2021-09-17 16:49:42 +0200571 if (supports == nullptr) {
572 return binderStatusFromStatusT(BAD_VALUE);
573 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800574 audio_utils::lock_guard lock(mMutex);
Eric Laurentc87402b2021-09-17 16:49:42 +0200575 *supports = mSupportsHeadTracking;
576 return Status::ok();
577}
578
Eric Laurent6d607012021-07-05 11:54:40 +0200579Status Spatializer::getSupportedHeadTrackingModes(
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000580 std::vector<HeadTracking::Mode>* modes) {
Andy Hung79eacdb2023-11-30 19:34:24 -0800581 audio_utils::lock_guard lock(mMutex);
Eric Laurent6d607012021-07-05 11:54:40 +0200582 ALOGV("%s", __func__);
583 if (modes == nullptr) {
584 return binderStatusFromStatusT(BAD_VALUE);
585 }
Andy Hunga461a002022-05-17 10:36:02 -0700586 modes->insert(modes->end(), mHeadTrackingModes.begin(), mHeadTrackingModes.end());
Eric Laurent6d607012021-07-05 11:54:40 +0200587 return Status::ok();
588}
589
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000590Status Spatializer::setDesiredHeadTrackingMode(HeadTracking::Mode mode) {
591 ALOGV("%s mode %s", __func__, ToString(mode).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700592
593 if (!mSupportsHeadTracking) {
594 return binderStatusFromStatusT(INVALID_OPERATION);
595 }
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000596 mLocalLog.log("%s with %s", __func__, ToString(mode).c_str());
Andy Hung79eacdb2023-11-30 19:34:24 -0800597 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700598 switch (mode) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000599 case HeadTracking::Mode::OTHER:
Eric Laurent2be8b292021-08-23 09:44:33 -0700600 return binderStatusFromStatusT(BAD_VALUE);
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000601 case HeadTracking::Mode::DISABLED:
Eric Laurent2be8b292021-08-23 09:44:33 -0700602 mDesiredHeadTrackingMode = HeadTrackingMode::STATIC;
603 break;
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000604 case HeadTracking::Mode::RELATIVE_WORLD:
Eric Laurent2be8b292021-08-23 09:44:33 -0700605 mDesiredHeadTrackingMode = HeadTrackingMode::WORLD_RELATIVE;
606 break;
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000607 case HeadTracking::Mode::RELATIVE_SCREEN:
Eric Laurent2be8b292021-08-23 09:44:33 -0700608 mDesiredHeadTrackingMode = HeadTrackingMode::SCREEN_RELATIVE;
609 break;
610 }
611
Eric Laurent11094172022-04-05 18:27:42 +0200612 checkPoseController_l();
613 checkSensorsState_l();
Eric Laurent2be8b292021-08-23 09:44:33 -0700614
615 return Status::ok();
616}
617
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000618Status Spatializer::getActualHeadTrackingMode(HeadTracking::Mode *mode) {
Eric Laurent6d607012021-07-05 11:54:40 +0200619 if (mode == nullptr) {
620 return binderStatusFromStatusT(BAD_VALUE);
621 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800622 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700623 *mode = mActualHeadTrackingMode;
Eric Laurent6d607012021-07-05 11:54:40 +0200624 ALOGV("%s mode %d", __func__, (int)*mode);
625 return Status::ok();
626}
627
Ytai Ben-Tsvia16a9df2021-08-05 08:57:06 -0700628Status Spatializer::recenterHeadTracker() {
Eric Laurent780be4a2021-09-16 10:44:24 +0200629 if (!mSupportsHeadTracking) {
630 return binderStatusFromStatusT(INVALID_OPERATION);
631 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800632 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700633 if (mPoseController != nullptr) {
634 mPoseController->recenter();
635 }
Eric Laurent6d607012021-07-05 11:54:40 +0200636 return Status::ok();
637}
638
639Status Spatializer::setGlobalTransform(const std::vector<float>& screenToStage) {
Eric Laurent6d607012021-07-05 11:54:40 +0200640 ALOGV("%s", __func__);
Eric Laurent780be4a2021-09-16 10:44:24 +0200641 if (!mSupportsHeadTracking) {
642 return binderStatusFromStatusT(INVALID_OPERATION);
643 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700644 std::optional<Pose3f> maybePose = Pose3f::fromVector(screenToStage);
645 if (!maybePose.has_value()) {
646 ALOGW("Invalid screenToStage vector.");
647 return binderStatusFromStatusT(BAD_VALUE);
648 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800649 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700650 if (mPoseController != nullptr) {
Andy Hunga367cb22023-01-30 11:58:44 -0800651 mLocalLog.log("%s with screenToStage %s", __func__,
652 media::VectorRecorder::toString<float>(screenToStage).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700653 mPoseController->setScreenToStagePose(maybePose.value());
654 }
Eric Laurent6d607012021-07-05 11:54:40 +0200655 return Status::ok();
656}
657
658Status Spatializer::release() {
659 ALOGV("%s", __func__);
660 bool levelChanged = false;
661 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800662 audio_utils::lock_guard lock(mMutex);
Eric Laurent6d607012021-07-05 11:54:40 +0200663 if (mSpatializerCallback == nullptr) {
664 return binderStatusFromStatusT(INVALID_OPERATION);
665 }
666
667 sp<IBinder> binder = IInterface::asBinder(mSpatializerCallback);
668 binder->unlinkToDeath(this);
669 mSpatializerCallback.clear();
670
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000671 levelChanged = mLevel != Spatialization::Level::NONE;
672 mLevel = Spatialization::Level::NONE;
Eric Laurent6d607012021-07-05 11:54:40 +0200673 }
674
675 if (levelChanged) {
676 mPolicyCallback->onCheckSpatializer();
677 }
678 return Status::ok();
679}
680
Eric Laurent2be8b292021-08-23 09:44:33 -0700681Status Spatializer::setHeadSensor(int sensorHandle) {
682 ALOGV("%s sensorHandle %d", __func__, sensorHandle);
Eric Laurent780be4a2021-09-16 10:44:24 +0200683 if (!mSupportsHeadTracking) {
684 return binderStatusFromStatusT(INVALID_OPERATION);
685 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800686 audio_utils::lock_guard lock(mMutex);
Andy Hungba2a61a2022-05-20 12:00:28 -0700687 if (mHeadSensor != sensorHandle) {
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000688 mLocalLog.log("%s with 0x%08x", __func__, sensorHandle);
Andy Hungba2a61a2022-05-20 12:00:28 -0700689 mHeadSensor = sensorHandle;
690 checkPoseController_l();
691 checkSensorsState_l();
692 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700693 return Status::ok();
694}
695
696Status Spatializer::setScreenSensor(int sensorHandle) {
697 ALOGV("%s sensorHandle %d", __func__, sensorHandle);
Eric Laurent780be4a2021-09-16 10:44:24 +0200698 if (!mSupportsHeadTracking) {
699 return binderStatusFromStatusT(INVALID_OPERATION);
700 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800701 audio_utils::lock_guard lock(mMutex);
Andy Hungba2a61a2022-05-20 12:00:28 -0700702 if (mScreenSensor != sensorHandle) {
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000703 mLocalLog.log("%s with 0x%08x", __func__, sensorHandle);
Andy Hungba2a61a2022-05-20 12:00:28 -0700704 mScreenSensor = sensorHandle;
705 // TODO: consider a new method setHeadAndScreenSensor()
706 // because we generally set both at the same time.
707 // This will avoid duplicated work and recentering.
708 checkSensorsState_l();
709 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700710 return Status::ok();
711}
712
713Status Spatializer::setDisplayOrientation(float physicalToLogicalAngle) {
714 ALOGV("%s physicalToLogicalAngle %f", __func__, physicalToLogicalAngle);
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000715 mLocalLog.log("%s with %f", __func__, physicalToLogicalAngle);
Andy Hung2490b2d2023-03-09 20:45:36 -0800716 const float angle = safe_clamp(physicalToLogicalAngle, 0.f, (float)(2. * M_PI));
717 // It is possible due to numerical inaccuracies to exceed the boundaries of 0 to 2 * M_PI.
718 ALOGI_IF(angle != physicalToLogicalAngle,
719 "%s: clamping %f to %f", __func__, physicalToLogicalAngle, angle);
Andy Hung79eacdb2023-11-30 19:34:24 -0800720 audio_utils::lock_guard lock(mMutex);
Andy Hung2490b2d2023-03-09 20:45:36 -0800721 mDisplayOrientation = angle;
Eric Laurent2be8b292021-08-23 09:44:33 -0700722 if (mPoseController != nullptr) {
Andy Hung2490b2d2023-03-09 20:45:36 -0800723 // This turns on the rate-limiter.
724 mPoseController->setDisplayOrientation(angle);
Eric Laurent2be8b292021-08-23 09:44:33 -0700725 }
Eric Laurent16ddaf42021-09-17 15:00:35 +0200726 if (mEngine != nullptr) {
727 setEffectParameter_l(
Andy Hung2490b2d2023-03-09 20:45:36 -0800728 SPATIALIZER_PARAM_DISPLAY_ORIENTATION, std::vector<float>{angle});
Eric Laurent16ddaf42021-09-17 15:00:35 +0200729 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700730 return Status::ok();
731}
732
733Status Spatializer::setHingeAngle(float hingeAngle) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700734 ALOGV("%s hingeAngle %f", __func__, hingeAngle);
Andy Hung2490b2d2023-03-09 20:45:36 -0800735 mLocalLog.log("%s with %f", __func__, hingeAngle);
736 const float angle = safe_clamp(hingeAngle, 0.f, (float)(2. * M_PI));
737 // It is possible due to numerical inaccuracies to exceed the boundaries of 0 to 2 * M_PI.
738 ALOGI_IF(angle != hingeAngle,
739 "%s: clamping %f to %f", __func__, hingeAngle, angle);
Andy Hung79eacdb2023-11-30 19:34:24 -0800740 audio_utils::lock_guard lock(mMutex);
Andy Hung2490b2d2023-03-09 20:45:36 -0800741 mHingeAngle = angle;
Eric Laurent2be8b292021-08-23 09:44:33 -0700742 if (mEngine != nullptr) {
Andy Hung2490b2d2023-03-09 20:45:36 -0800743 setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE, std::vector<float>{angle});
744 }
745 return Status::ok();
746}
747
748Status Spatializer::setFoldState(bool folded) {
749 ALOGV("%s foldState %d", __func__, (int)folded);
750 mLocalLog.log("%s with %d", __func__, (int)folded);
Andy Hung79eacdb2023-11-30 19:34:24 -0800751 audio_utils::lock_guard lock(mMutex);
Andy Hung2490b2d2023-03-09 20:45:36 -0800752 mFoldedState = folded;
753 if (mEngine != nullptr) {
754 // we don't suppress multiple calls with the same folded state - that's
755 // done at the caller.
756 setEffectParameter_l(SPATIALIZER_PARAM_FOLD_STATE, std::vector<uint8_t>{mFoldedState});
Eric Laurent2be8b292021-08-23 09:44:33 -0700757 }
758 return Status::ok();
759}
760
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000761Status Spatializer::getSupportedModes(std::vector<Spatialization::Mode> *modes) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700762 ALOGV("%s", __func__);
763 if (modes == nullptr) {
764 return binderStatusFromStatusT(BAD_VALUE);
765 }
766 *modes = mSpatializationModes;
767 return Status::ok();
768}
769
Eric Laurent67816e32021-09-16 15:18:40 +0200770Status Spatializer::registerHeadTrackingCallback(
771 const sp<media::ISpatializerHeadTrackingCallback>& callback) {
772 ALOGV("%s callback %p", __func__, callback.get());
Andy Hung79eacdb2023-11-30 19:34:24 -0800773 audio_utils::lock_guard lock(mMutex);
Eric Laurent67816e32021-09-16 15:18:40 +0200774 if (!mSupportsHeadTracking) {
775 return binderStatusFromStatusT(INVALID_OPERATION);
776 }
777 mHeadTrackingCallback = callback;
778 return Status::ok();
779}
780
Eric Laurentc87402b2021-09-17 16:49:42 +0200781Status Spatializer::setParameter(int key, const std::vector<unsigned char>& value) {
782 ALOGV("%s key %d", __func__, key);
Andy Hung79eacdb2023-11-30 19:34:24 -0800783 audio_utils::lock_guard lock(mMutex);
Eric Laurentc87402b2021-09-17 16:49:42 +0200784 status_t status = INVALID_OPERATION;
785 if (mEngine != nullptr) {
786 status = setEffectParameter_l(key, value);
787 }
788 return binderStatusFromStatusT(status);
789}
790
791Status Spatializer::getParameter(int key, std::vector<unsigned char> *value) {
Greg Kaiserf7249f82021-09-21 07:10:12 -0700792 ALOGV("%s key %d value size %d", __func__, key,
793 (value != nullptr ? (int)value->size() : -1));
Eric Laurentc87402b2021-09-17 16:49:42 +0200794 if (value == nullptr) {
George Burgess IV22386222021-09-22 12:09:31 -0700795 return binderStatusFromStatusT(BAD_VALUE);
Eric Laurentc87402b2021-09-17 16:49:42 +0200796 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800797 audio_utils::lock_guard lock(mMutex);
Eric Laurentc87402b2021-09-17 16:49:42 +0200798 status_t status = INVALID_OPERATION;
799 if (mEngine != nullptr) {
800 ALOGV("%s key %d mEngine %p", __func__, key, mEngine.get());
801 status = getEffectParameter_l(key, value);
802 }
803 return binderStatusFromStatusT(status);
804}
805
806Status Spatializer::getOutput(int *output) {
807 ALOGV("%s", __func__);
808 if (output == nullptr) {
809 binderStatusFromStatusT(BAD_VALUE);
810 }
Andy Hung79eacdb2023-11-30 19:34:24 -0800811 audio_utils::lock_guard lock(mMutex);
Eric Laurentc87402b2021-09-17 16:49:42 +0200812 *output = VALUE_OR_RETURN_BINDER_STATUS(legacy2aidl_audio_io_handle_t_int32_t(mOutput));
813 ALOGV("%s got output %d", __func__, *output);
814 return Status::ok();
815}
816
Eric Laurent2be8b292021-08-23 09:44:33 -0700817// SpatializerPoseController::Listener
818void Spatializer::onHeadToStagePose(const Pose3f& headToStage) {
819 ALOGV("%s", __func__);
Eric Laurent780be4a2021-09-16 10:44:24 +0200820 LOG_ALWAYS_FATAL_IF(!mSupportsHeadTracking,
821 "onHeadToStagePose() called with no head tracking support!");
822
Eric Laurent2be8b292021-08-23 09:44:33 -0700823 auto vec = headToStage.toVector();
Eric Laurent8a4259f2021-09-14 16:04:00 +0200824 LOG_ALWAYS_FATAL_IF(vec.size() != sHeadPoseKeys.size(),
825 "%s invalid head to stage vector size %zu", __func__, vec.size());
Eric Laurent8a4259f2021-09-14 16:04:00 +0200826 sp<AMessage> msg =
827 new AMessage(EngineCallbackHandler::kWhatOnHeadToStagePose, mHandler);
828 for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
829 msg->setFloat(sHeadPoseKeys[i], vec[i]);
830 }
831 msg->post();
832}
833
Eric Laurentbdecc052022-10-21 11:28:32 +0200834void Spatializer::resetEngineHeadPose_l() {
835 ALOGV("%s mEngine %p", __func__, mEngine.get());
836 if (mEngine == nullptr) {
837 return;
838 }
839 const std::vector<float> headToStage(6, 0.0);
840 setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, headToStage);
841 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000842 std::vector<HeadTracking::Mode>{HeadTracking::Mode::DISABLED});
Eric Laurentbdecc052022-10-21 11:28:32 +0200843}
844
Eric Laurent8a4259f2021-09-14 16:04:00 +0200845void Spatializer::onHeadToStagePoseMsg(const std::vector<float>& headToStage) {
846 ALOGV("%s", __func__);
Eric Laurent67816e32021-09-16 15:18:40 +0200847 sp<media::ISpatializerHeadTrackingCallback> callback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700848 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800849 audio_utils::lock_guard lock(mMutex);
Eric Laurent67816e32021-09-16 15:18:40 +0200850 callback = mHeadTrackingCallback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700851 if (mEngine != nullptr) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200852 setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, headToStage);
Andy Hung560addd2023-01-30 11:58:44 -0800853 const auto record = recordFromTranslationRotationVector(headToStage);
854 mPoseRecorder.record(record);
855 mPoseDurableRecorder.record(record);
Eric Laurent2be8b292021-08-23 09:44:33 -0700856 }
857 }
858
859 if (callback != nullptr) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200860 callback->onHeadToSoundStagePoseUpdated(headToStage);
Eric Laurent2be8b292021-08-23 09:44:33 -0700861 }
862}
863
864void Spatializer::onActualModeChange(HeadTrackingMode mode) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000865 std::string modeStr = ToString(mode);
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000866 ALOGV("%s(%s)", __func__, modeStr.c_str());
Shunkai Yao56e43f02022-08-30 03:14:50 +0000867 sp<AMessage> msg = new AMessage(EngineCallbackHandler::kWhatOnActualModeChange, mHandler);
Eric Laurent8a4259f2021-09-14 16:04:00 +0200868 msg->setInt32(EngineCallbackHandler::kModeKey, static_cast<int>(mode));
869 msg->post();
870}
871
872void Spatializer::onActualModeChangeMsg(HeadTrackingMode mode) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000873 ALOGV("%s(%s)", __func__, ToString(mode).c_str());
Eric Laurent67816e32021-09-16 15:18:40 +0200874 sp<media::ISpatializerHeadTrackingCallback> callback;
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000875 HeadTracking::Mode spatializerMode;
Eric Laurent2be8b292021-08-23 09:44:33 -0700876 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800877 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700878 if (!mSupportsHeadTracking) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000879 spatializerMode = HeadTracking::Mode::DISABLED;
Eric Laurent2be8b292021-08-23 09:44:33 -0700880 } else {
881 switch (mode) {
882 case HeadTrackingMode::STATIC:
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000883 spatializerMode = HeadTracking::Mode::DISABLED;
Eric Laurent2be8b292021-08-23 09:44:33 -0700884 break;
885 case HeadTrackingMode::WORLD_RELATIVE:
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000886 spatializerMode = HeadTracking::Mode::RELATIVE_WORLD;
Eric Laurent2be8b292021-08-23 09:44:33 -0700887 break;
888 case HeadTrackingMode::SCREEN_RELATIVE:
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000889 spatializerMode = HeadTracking::Mode::RELATIVE_SCREEN;
Eric Laurent2be8b292021-08-23 09:44:33 -0700890 break;
891 default:
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000892 LOG_ALWAYS_FATAL("Unknown mode: %s", ToString(mode).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700893 }
894 }
895 mActualHeadTrackingMode = spatializerMode;
Eric Laurente51f80e2022-04-14 10:20:38 +0200896 if (mEngine != nullptr) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000897 if (spatializerMode == HeadTracking::Mode::DISABLED) {
Eric Laurentbdecc052022-10-21 11:28:32 +0200898 resetEngineHeadPose_l();
899 } else {
900 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
Shunkai Yao49bc61f2023-10-10 19:31:10 +0000901 std::vector<HeadTracking::Mode>{spatializerMode});
Eric Laurentbe21a942023-11-14 16:05:00 +0100902 setEngineHeadtrackingConnectionMode_l();
Eric Laurentbdecc052022-10-21 11:28:32 +0200903 }
Eric Laurent7ea0d1b2022-04-01 14:23:44 +0200904 }
Eric Laurent67816e32021-09-16 15:18:40 +0200905 callback = mHeadTrackingCallback;
Shunkai Yao2ee06f62024-02-16 22:00:22 +0000906 mLocalLog.log("%s: updating mode to %s", __func__, ToString(mode).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700907 }
Eric Laurente51f80e2022-04-14 10:20:38 +0200908 if (callback != nullptr) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700909 callback->onHeadTrackingModeChanged(spatializerMode);
910 }
911}
912
Eric Laurentbe21a942023-11-14 16:05:00 +0100913void Spatializer::setEngineHeadtrackingConnectionMode_l() {
914 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
915 return;
916 }
917 if (mActualHeadTrackingMode != HeadTracking::Mode::DISABLED
918 && !mSupportedHeadtrackingConnectionModes.empty()) {
919 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_CONNECTION,
920 static_cast<uint8_t>(mHeadtrackingConnectionMode),
921 static_cast<uint32_t>(mHeadSensor));
922 }
923}
924
925void Spatializer::sortSupportedLatencyModes_l() {
926 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
927 return;
928 }
929 std::sort(mSupportedLatencyModes.begin(), mSupportedLatencyModes.end(),
930 [this](audio_latency_mode_t x, audio_latency_mode_t y) {
931 auto itX = std::find(mOrderedLowLatencyModes.begin(),
932 mOrderedLowLatencyModes.end(), x);
933 auto itY = std::find(mOrderedLowLatencyModes.begin(),
934 mOrderedLowLatencyModes.end(), y);
935 return itX < itY;
936 });
937}
938
Eric Laurent15903592022-02-24 20:44:36 +0100939status_t Spatializer::attachOutput(audio_io_handle_t output, size_t numActiveTracks) {
Eric Laurent4a872862021-10-11 17:06:47 +0200940 bool outputChanged = false;
941 sp<media::INativeSpatializerCallback> callback;
942
Eric Laurent2be8b292021-08-23 09:44:33 -0700943 {
Andy Hung79eacdb2023-11-30 19:34:24 -0800944 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -0700945 ALOGV("%s output %d mOutput %d", __func__, (int)output, (int)mOutput);
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +0000946 mLocalLog.log("%s with output %d tracks %zu (mOutput %d)", __func__, (int)output,
947 numActiveTracks, (int)mOutput);
Eric Laurent2be8b292021-08-23 09:44:33 -0700948 if (mOutput != AUDIO_IO_HANDLE_NONE) {
949 LOG_ALWAYS_FATAL_IF(mEngine == nullptr, "%s output set without FX engine", __func__);
950 // remove FX instance
951 mEngine->setEnabled(false);
952 mEngine.clear();
Eric Laurent15903592022-02-24 20:44:36 +0100953 mPoseController.reset();
Eric Laurentee398ad2022-05-03 18:19:35 +0200954 AudioSystem::removeSupportedLatencyModesCallback(this);
Eric Laurent2be8b292021-08-23 09:44:33 -0700955 }
Eric Laurentee398ad2022-05-03 18:19:35 +0200956
Eric Laurent2be8b292021-08-23 09:44:33 -0700957 // create FX instance on output
958 AttributionSourceState attributionSource = AttributionSourceState();
959 mEngine = new AudioEffect(attributionSource);
Atneya Nair20e6cc82022-05-17 20:12:37 -0400960 mEngine->set(nullptr /* type */, &mEngineDescriptor.uuid, 0 /* priority */,
961 wp<AudioEffect::IAudioEffectCallback>::fromExisting(this),
962 AUDIO_SESSION_OUTPUT_STAGE, output, {} /* device */, false /* probe */,
963 true /* notifyFramesProcessed */);
Eric Laurent2be8b292021-08-23 09:44:33 -0700964 status_t status = mEngine->initCheck();
965 ALOGV("%s mEngine create status %d", __func__, (int)status);
966 if (status != NO_ERROR) {
967 return status;
968 }
969
Eric Laurent4a872862021-10-11 17:06:47 +0200970 outputChanged = mOutput != output;
Eric Laurent2be8b292021-08-23 09:44:33 -0700971 mOutput = output;
Eric Laurent11094172022-04-05 18:27:42 +0200972 mNumActiveTracks = numActiveTracks;
Eric Laurentee398ad2022-05-03 18:19:35 +0200973 AudioSystem::addSupportedLatencyModesCallback(this);
974
975 std::vector<audio_latency_mode_t> latencyModes;
976 status = AudioSystem::getSupportedLatencyModes(mOutput, &latencyModes);
977 if (status == OK) {
978 mSupportedLatencyModes = latencyModes;
Eric Laurentbe21a942023-11-14 16:05:00 +0100979 sortSupportedLatencyModes_l();
Eric Laurentee398ad2022-05-03 18:19:35 +0200980 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700981
Eric Laurent11094172022-04-05 18:27:42 +0200982 checkEngineState_l();
Eric Laurent780be4a2021-09-16 10:44:24 +0200983 if (mSupportsHeadTracking) {
Eric Laurent11094172022-04-05 18:27:42 +0200984 checkPoseController_l();
Eric Laurent9249d342022-03-18 11:55:56 +0100985 checkSensorsState_l();
Eric Laurent780be4a2021-09-16 10:44:24 +0200986 }
Eric Laurent4a872862021-10-11 17:06:47 +0200987 callback = mSpatializerCallback;
Andy Hung2490b2d2023-03-09 20:45:36 -0800988
989 // Restore common effect state.
990 setEffectParameter_l(SPATIALIZER_PARAM_DISPLAY_ORIENTATION,
991 std::vector<float>{mDisplayOrientation});
992 setEffectParameter_l(SPATIALIZER_PARAM_FOLD_STATE,
993 std::vector<uint8_t>{mFoldedState});
994 setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE,
995 std::vector<float>{mHingeAngle});
Eric Laurent6d607012021-07-05 11:54:40 +0200996 }
Eric Laurent4a872862021-10-11 17:06:47 +0200997
998 if (outputChanged && callback != nullptr) {
999 callback->onOutputChanged(output);
1000 }
1001
Eric Laurent6d607012021-07-05 11:54:40 +02001002 return NO_ERROR;
1003}
1004
1005audio_io_handle_t Spatializer::detachOutput() {
Eric Laurent2be8b292021-08-23 09:44:33 -07001006 audio_io_handle_t output = AUDIO_IO_HANDLE_NONE;
Eric Laurent4a872862021-10-11 17:06:47 +02001007 sp<media::INativeSpatializerCallback> callback;
1008
1009 {
Andy Hung79eacdb2023-11-30 19:34:24 -08001010 audio_utils::lock_guard lock(mMutex);
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001011 mLocalLog.log("%s with output %d tracks %zu", __func__, (int)mOutput, mNumActiveTracks);
Eric Laurent4a872862021-10-11 17:06:47 +02001012 ALOGV("%s mOutput %d", __func__, (int)mOutput);
1013 if (mOutput == AUDIO_IO_HANDLE_NONE) {
1014 return output;
1015 }
1016 // remove FX instance
1017 mEngine->setEnabled(false);
1018 mEngine.clear();
Eric Laurentee398ad2022-05-03 18:19:35 +02001019 AudioSystem::removeSupportedLatencyModesCallback(this);
Eric Laurent4a872862021-10-11 17:06:47 +02001020 output = mOutput;
1021 mOutput = AUDIO_IO_HANDLE_NONE;
1022 mPoseController.reset();
Eric Laurent4a872862021-10-11 17:06:47 +02001023 callback = mSpatializerCallback;
Eric Laurent6d607012021-07-05 11:54:40 +02001024 }
Eric Laurent4a872862021-10-11 17:06:47 +02001025
1026 if (callback != nullptr) {
1027 callback->onOutputChanged(AUDIO_IO_HANDLE_NONE);
1028 }
Eric Laurent6d607012021-07-05 11:54:40 +02001029 return output;
1030}
1031
Eric Laurentee398ad2022-05-03 18:19:35 +02001032void Spatializer::onSupportedLatencyModesChanged(
1033 audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes) {
Eric Laurent9c04de92022-07-20 13:49:47 +02001034 ALOGV("%s output %d num modes %zu", __func__, (int)output, modes.size());
1035 sp<AMessage> msg =
1036 new AMessage(EngineCallbackHandler::kWhatOnLatencyModesChanged, mHandler);
1037 msg->setObject(EngineCallbackHandler::kLatencyModesKey,
1038 sp<EngineCallbackHandler::LatencyModes>::make(output, modes));
1039 msg->post();
1040}
1041
1042void Spatializer::onSupportedLatencyModesChangedMsg(
1043 audio_io_handle_t output, std::vector<audio_latency_mode_t>&& modes) {
Andy Hung79eacdb2023-11-30 19:34:24 -08001044 audio_utils::lock_guard lock(mMutex);
Eric Laurent9c04de92022-07-20 13:49:47 +02001045 ALOGV("%s output %d mOutput %d num modes %zu",
1046 __func__, (int)output, (int)mOutput, modes.size());
Eric Laurentee398ad2022-05-03 18:19:35 +02001047 if (output == mOutput) {
Eric Laurent9c04de92022-07-20 13:49:47 +02001048 mSupportedLatencyModes = std::move(modes);
Eric Laurentbe21a942023-11-14 16:05:00 +01001049 sortSupportedLatencyModes_l();
Eric Laurentee398ad2022-05-03 18:19:35 +02001050 checkSensorsState_l();
1051 }
1052}
1053
Eric Laurent15903592022-02-24 20:44:36 +01001054void Spatializer::updateActiveTracks(size_t numActiveTracks) {
Andy Hung79eacdb2023-11-30 19:34:24 -08001055 audio_utils::lock_guard lock(mMutex);
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001056 if (mNumActiveTracks != numActiveTracks) {
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001057 mLocalLog.log("%s from %zu to %zu", __func__, mNumActiveTracks, numActiveTracks);
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001058 mNumActiveTracks = numActiveTracks;
1059 checkEngineState_l();
1060 checkSensorsState_l();
1061 }
Eric Laurent15903592022-02-24 20:44:36 +01001062}
1063
Eric Laurentbe21a942023-11-14 16:05:00 +01001064audio_latency_mode_t Spatializer::selectHeadtrackingConnectionMode_l() {
1065 if (!com::android::media::audio::dsa_over_bt_le_audio()) {
1066 return AUDIO_LATENCY_MODE_LOW;
1067 }
1068 // mSupportedLatencyModes is ordered according to system preferences loaded in
1069 // mOrderedLowLatencyModes
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001070 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED;
Eric Laurentbe21a942023-11-14 16:05:00 +01001071 audio_latency_mode_t requestedLatencyMode = mSupportedLatencyModes[0];
1072 if (requestedLatencyMode == AUDIO_LATENCY_MODE_DYNAMIC_SPATIAL_AUDIO_HARDWARE) {
1073 if (mSupportedHeadtrackingConnectionModes.find(
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001074 HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_TUNNEL)
Eric Laurentbe21a942023-11-14 16:05:00 +01001075 != mSupportedHeadtrackingConnectionModes.end()) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001076 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_TUNNEL;
Eric Laurentbe21a942023-11-14 16:05:00 +01001077 } else if (mSupportedHeadtrackingConnectionModes.find(
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001078 HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_SW)
Eric Laurentbe21a942023-11-14 16:05:00 +01001079 != mSupportedHeadtrackingConnectionModes.end()) {
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001080 mHeadtrackingConnectionMode = HeadTracking::ConnectionMode::DIRECT_TO_SENSOR_SW;
Eric Laurentbe21a942023-11-14 16:05:00 +01001081 } else {
1082 // if the engine does not support direct reading of IMU data, do not allow
1083 // DYNAMIC_SPATIAL_AUDIO_HARDWARE mode and fallback to next mode
1084 if (mSupportedLatencyModes.size() > 1) {
1085 requestedLatencyMode = mSupportedLatencyModes[1];
1086 } else {
1087 // If only DYNAMIC_SPATIAL_AUDIO_HARDWARE mode is reported by the
1088 // HAL and the engine does not support it, assert as this is a
1089 // product configuration error
1090 LOG_ALWAYS_FATAL("%s: the audio HAL reported only low latency with"
1091 "HW HID tunneling but the spatializer does not support it",
1092 __func__);
1093 }
1094 }
1095 }
1096 return requestedLatencyMode;
1097}
1098
Eric Laurent9249d342022-03-18 11:55:56 +01001099void Spatializer::checkSensorsState_l() {
Eric Laurent82d1f452024-03-07 18:41:00 +01001100 mRequestedLatencyMode = AUDIO_LATENCY_MODE_FREE;
Andy Hungdae53212022-10-21 17:30:30 -07001101 const bool supportsSetLatencyMode = !mSupportedLatencyModes.empty();
Eric Laurentbe21a942023-11-14 16:05:00 +01001102 bool supportsLowLatencyMode;
1103 if (com::android::media::audio::dsa_over_bt_le_audio()) {
1104 // mSupportedLatencyModes is ordered with MODE_FREE always at the end:
1105 // the first entry is never MODE_FREE if at least one low ltency mode is supported.
1106 supportsLowLatencyMode = supportsSetLatencyMode
1107 && mSupportedLatencyModes[0] != AUDIO_LATENCY_MODE_FREE;
1108 } else {
1109 supportsLowLatencyMode = supportsSetLatencyMode && std::find(
Andy Hungdae53212022-10-21 17:30:30 -07001110 mSupportedLatencyModes.begin(), mSupportedLatencyModes.end(),
1111 AUDIO_LATENCY_MODE_LOW) != mSupportedLatencyModes.end();
Eric Laurentbe21a942023-11-14 16:05:00 +01001112 }
Eric Laurentbdecc052022-10-21 11:28:32 +02001113 if (mSupportsHeadTracking) {
1114 if (mPoseController != nullptr) {
Andy Hungdae53212022-10-21 17:30:30 -07001115 // TODO(b/253297301, b/255433067) reenable low latency condition check
1116 // for Head Tracking after Bluetooth HAL supports it correctly.
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001117 if (mNumActiveTracks > 0 && mLevel != Spatialization::Level::NONE
Eric Laurentbe21a942023-11-14 16:05:00 +01001118 && mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
1119 && mHeadSensor != SpatializerPoseController::INVALID_SENSOR) {
1120 if (supportsLowLatencyMode) {
Eric Laurent82d1f452024-03-07 18:41:00 +01001121 mRequestedLatencyMode = selectHeadtrackingConnectionMode_l();
Eric Laurentbe21a942023-11-14 16:05:00 +01001122 }
Eric Laurentbdecc052022-10-21 11:28:32 +02001123 if (mEngine != nullptr) {
1124 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001125 std::vector<HeadTracking::Mode>{mActualHeadTrackingMode});
Eric Laurentbe21a942023-11-14 16:05:00 +01001126 setEngineHeadtrackingConnectionMode_l();
Eric Laurentbdecc052022-10-21 11:28:32 +02001127 }
Eric Laurentbe21a942023-11-14 16:05:00 +01001128 // TODO: b/307588546: configure mPoseController according to selected
1129 // mHeadtrackingConnectionMode
Eric Laurentbdecc052022-10-21 11:28:32 +02001130 mPoseController->setHeadSensor(mHeadSensor);
1131 mPoseController->setScreenSensor(mScreenSensor);
Eric Laurentbdecc052022-10-21 11:28:32 +02001132 } else {
1133 mPoseController->setHeadSensor(SpatializerPoseController::INVALID_SENSOR);
1134 mPoseController->setScreenSensor(SpatializerPoseController::INVALID_SENSOR);
1135 resetEngineHeadPose_l();
1136 }
Eric Laurent15903592022-02-24 20:44:36 +01001137 } else {
Eric Laurentbdecc052022-10-21 11:28:32 +02001138 resetEngineHeadPose_l();
Eric Laurent15903592022-02-24 20:44:36 +01001139 }
1140 }
Andy Hungdae53212022-10-21 17:30:30 -07001141 if (mOutput != AUDIO_IO_HANDLE_NONE && supportsSetLatencyMode) {
Andy Hung4bd53e72022-11-17 17:21:45 -08001142 const status_t status =
Eric Laurent82d1f452024-03-07 18:41:00 +01001143 AudioSystem::setRequestedLatencyMode(mOutput, mRequestedLatencyMode);
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001144 ALOGD("%s: setRequestedLatencyMode for output thread(%d) to %s returned %d", __func__,
Eric Laurent82d1f452024-03-07 18:41:00 +01001145 mOutput, toString(mRequestedLatencyMode).c_str(), status);
Eric Laurentee398ad2022-05-03 18:19:35 +02001146 }
Eric Laurent15903592022-02-24 20:44:36 +01001147}
1148
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001149void Spatializer::checkEngineState_l() {
1150 if (mEngine != nullptr) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001151 if (mLevel != Spatialization::Level::NONE && mNumActiveTracks > 0) {
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001152 mEngine->setEnabled(true);
1153 setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001154 std::vector<Spatialization::Level>{mLevel});
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001155 } else {
1156 setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001157 std::vector<Spatialization::Level>{Spatialization::Level::NONE});
Eric Laurent7ea0d1b2022-04-01 14:23:44 +02001158 mEngine->setEnabled(false);
1159 }
1160 }
1161}
1162
Eric Laurent11094172022-04-05 18:27:42 +02001163void Spatializer::checkPoseController_l() {
1164 bool isControllerNeeded = mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
1165 && mHeadSensor != SpatializerPoseController::INVALID_SENSOR;
1166
1167 if (isControllerNeeded && mPoseController == nullptr) {
1168 mPoseController = std::make_shared<SpatializerPoseController>(
1169 static_cast<SpatializerPoseController::Listener*>(this),
Eric Laurente51f80e2022-04-14 10:20:38 +02001170 10ms, std::nullopt);
Eric Laurent11094172022-04-05 18:27:42 +02001171 LOG_ALWAYS_FATAL_IF(mPoseController == nullptr,
1172 "%s could not allocate pose controller", __func__);
1173 mPoseController->setDisplayOrientation(mDisplayOrientation);
1174 } else if (!isControllerNeeded && mPoseController != nullptr) {
1175 mPoseController.reset();
Eric Laurentbdecc052022-10-21 11:28:32 +02001176 resetEngineHeadPose_l();
Eric Laurent11094172022-04-05 18:27:42 +02001177 }
1178 if (mPoseController != nullptr) {
1179 mPoseController->setDesiredMode(mDesiredHeadTrackingMode);
1180 }
1181}
1182
Eric Laurent2be8b292021-08-23 09:44:33 -07001183void Spatializer::calculateHeadPose() {
1184 ALOGV("%s", __func__);
Andy Hung79eacdb2023-11-30 19:34:24 -08001185 audio_utils::lock_guard lock(mMutex);
Eric Laurent2be8b292021-08-23 09:44:33 -07001186 if (mPoseController != nullptr) {
1187 mPoseController->calculateAsync();
1188 }
1189}
Eric Laurent6d607012021-07-05 11:54:40 +02001190
Atneya Nair20e6cc82022-05-17 20:12:37 -04001191void Spatializer::onFramesProcessed(int32_t framesProcessed) {
Eric Laurent8a4259f2021-09-14 16:04:00 +02001192 sp<AMessage> msg =
1193 new AMessage(EngineCallbackHandler::kWhatOnFramesProcessed, mHandler);
Atneya Nair20e6cc82022-05-17 20:12:37 -04001194 msg->setInt32(EngineCallbackHandler::kNumFramesKey, framesProcessed);
Eric Laurent8a4259f2021-09-14 16:04:00 +02001195 msg->post();
1196}
1197
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001198std::string Spatializer::toString(unsigned level) const {
Andy Hung560addd2023-01-30 11:58:44 -08001199 std::string prefixSpace(level, ' ');
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001200 std::string ss = prefixSpace + "Spatializer:\n";
1201 bool needUnlock = false;
1202
1203 prefixSpace += ' ';
Andy Hung79eacdb2023-11-30 19:34:24 -08001204 if (!mMutex.try_lock()) {
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001205 // dumpsys even try_lock failed, information dump can be useful although may not accurate
1206 ss.append(prefixSpace).append("try_lock failed, dumpsys below maybe INACCURATE!\n");
1207 } else {
1208 needUnlock = true;
1209 }
1210
1211 // Spatializer class information.
1212 // 1. Capabilities (mLevels, mHeadTrackingModes, mSpatializationModes, mChannelMasks, etc)
1213 ss.append(prefixSpace).append("Supported levels: [");
1214 for (auto& level : mLevels) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001215 base::StringAppendF(&ss, " %s", ToString(level).c_str());
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001216 }
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001217 base::StringAppendF(&ss, "], mLevel: %s", ToString(mLevel).c_str());
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001218
1219 base::StringAppendF(&ss, "\n%smHeadTrackingModes: [", prefixSpace.c_str());
1220 for (auto& mode : mHeadTrackingModes) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001221 base::StringAppendF(&ss, " %s", ToString(mode).c_str());
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001222 }
1223 base::StringAppendF(&ss, "], Desired: %s, Actual %s\n",
Shunkai Yao2ee06f62024-02-16 22:00:22 +00001224 ToString(mDesiredHeadTrackingMode).c_str(),
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001225 ToString(mActualHeadTrackingMode).c_str());
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001226
1227 base::StringAppendF(&ss, "%smSpatializationModes: [", prefixSpace.c_str());
1228 for (auto& mode : mSpatializationModes) {
Shunkai Yao49bc61f2023-10-10 19:31:10 +00001229 base::StringAppendF(&ss, " %s", ToString(mode).c_str());
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001230 }
1231 ss += "]\n";
1232
1233 base::StringAppendF(&ss, "%smChannelMasks: ", prefixSpace.c_str());
1234 for (auto& mask : mChannelMasks) {
1235 base::StringAppendF(&ss, "%s", audio_channel_out_mask_to_string(mask));
1236 }
1237 base::StringAppendF(&ss, "\n%smSupportsHeadTracking: %s\n", prefixSpace.c_str(),
1238 mSupportsHeadTracking ? "true" : "false");
1239 // 2. Settings (Output, tracks)
1240 base::StringAppendF(&ss, "%smNumActiveTracks: %zu\n", prefixSpace.c_str(), mNumActiveTracks);
1241 base::StringAppendF(&ss, "%sOutputStreamHandle: %d\n", prefixSpace.c_str(), (int)mOutput);
1242
1243 // 3. Sensors, Effect information.
1244 base::StringAppendF(&ss, "%sHeadSensorHandle: 0x%08x\n", prefixSpace.c_str(), mHeadSensor);
1245 base::StringAppendF(&ss, "%sScreenSensorHandle: 0x%08x\n", prefixSpace.c_str(), mScreenSensor);
1246 base::StringAppendF(&ss, "%sEffectHandle: %p\n", prefixSpace.c_str(), mEngine.get());
1247 base::StringAppendF(&ss, "%sDisplayOrientation: %f\n", prefixSpace.c_str(),
1248 mDisplayOrientation);
1249
Andy Hung481bfe32023-12-18 14:00:29 -08001250 // 4. Show flag or property state.
1251 base::StringAppendF(&ss, "%sStereo Spatialization: %s\n", prefixSpace.c_str(),
1252 com_android_media_audio_stereo_spatialization() ? "true" : "false");
1253
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001254 ss.append(prefixSpace + "CommandLog:\n");
1255 ss += mLocalLog.dumpToString((prefixSpace + " ").c_str(), mMaxLocalLogLine);
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001256
1257 // PostController dump.
1258 if (mPoseController != nullptr) {
Andy Hung560addd2023-01-30 11:58:44 -08001259 ss.append(mPoseController->toString(level + 1))
1260 .append(prefixSpace)
Andy Hunga69459c2023-02-27 13:36:23 -08001261 .append("Pose (active stage-to-head) [tx, ty, tz : pitch, roll, yaw]:\n")
Andy Hung560addd2023-01-30 11:58:44 -08001262 .append(prefixSpace)
1263 .append(" PerMinuteHistory:\n")
Andy Hunga69459c2023-02-27 13:36:23 -08001264 .append(mPoseDurableRecorder.toString(level + 3))
Andy Hung560addd2023-01-30 11:58:44 -08001265 .append(prefixSpace)
1266 .append(" PerSecondHistory:\n")
Andy Hunga69459c2023-02-27 13:36:23 -08001267 .append(mPoseRecorder.toString(level + 3));
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001268 } else {
1269 ss.append(prefixSpace).append("SpatializerPoseController not exist\n");
1270 }
1271
1272 if (needUnlock) {
Andy Hung79eacdb2023-11-30 19:34:24 -08001273 mMutex.unlock();
Shunkai Yaoafc0c2e2022-07-22 18:42:27 +00001274 }
1275 return ss;
1276}
1277
Eric Laurent6d607012021-07-05 11:54:40 +02001278} // namespace android