blob: fbc883fb6749f558380c885cbffc6e90f67264e7 [file] [log] [blame]
Eric Laurent6d607012021-07-05 11:54:40 +02001/*
2**
3** Copyright 2021, The Android Open Source Project
4**
5** Licensed under the Apache License, Version 2.0 (the "License");
6** you may not use this file except in compliance with the License.
7** You may obtain a copy of the License at
8**
9** http://www.apache.org/licenses/LICENSE-2.0
10**
11** Unless required by applicable law or agreed to in writing, software
12** distributed under the License is distributed on an "AS IS" BASIS,
13** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14** See the License for the specific language governing permissions and
15** limitations under the License.
16*/
17
Shunkai Yao59b27bc2022-07-22 18:42:27 +000018#include <string>
Eric Laurent6d607012021-07-05 11:54:40 +020019#define LOG_TAG "Spatializer"
20//#define LOG_NDEBUG 0
21#include <utils/Log.h>
22
Shunkai Yao59b27bc2022-07-22 18:42:27 +000023#include <inttypes.h>
Eric Laurent6d607012021-07-05 11:54:40 +020024#include <limits.h>
25#include <stdint.h>
26#include <sys/types.h>
27
28#include <android/content/AttributionSourceState.h>
29#include <audio_utils/fixedfft.h>
30#include <cutils/bitops.h>
Eric Laurent2be8b292021-08-23 09:44:33 -070031#include <hardware/sensors.h>
Eric Laurent6d607012021-07-05 11:54:40 +020032#include <media/audiohal/EffectsFactoryHalInterface.h>
Eric Laurent8a4259f2021-09-14 16:04:00 +020033#include <media/stagefright/foundation/AHandler.h>
34#include <media/stagefright/foundation/AMessage.h>
Andy Hunga461a002022-05-17 10:36:02 -070035#include <media/MediaMetricsItem.h>
Eric Laurent8a4259f2021-09-14 16:04:00 +020036#include <media/ShmemCompat.h>
Eric Laurent6d607012021-07-05 11:54:40 +020037#include <mediautils/ServiceUtilities.h>
38#include <utils/Thread.h>
39
40#include "Spatializer.h"
41
42namespace android {
43
44using aidl_utils::statusTFromBinderStatus;
45using aidl_utils::binderStatusFromStatusT;
46using android::content::AttributionSourceState;
47using binder::Status;
Eric Laurent2be8b292021-08-23 09:44:33 -070048using media::HeadTrackingMode;
49using media::Pose3f;
Eric Laurent6d607012021-07-05 11:54:40 +020050using media::SpatializationLevel;
Eric Laurent2be8b292021-08-23 09:44:33 -070051using media::SpatializationMode;
Ytai Ben-Tsvia16a9df2021-08-05 08:57:06 -070052using media::SpatializerHeadTrackingMode;
Eric Laurent2be8b292021-08-23 09:44:33 -070053using media::SensorPoseProvider;
54
Eric Laurent2be8b292021-08-23 09:44:33 -070055using namespace std::chrono_literals;
Eric Laurent6d607012021-07-05 11:54:40 +020056
57#define VALUE_OR_RETURN_BINDER_STATUS(x) \
58 ({ auto _tmp = (x); \
59 if (!_tmp.ok()) return aidl_utils::binderStatusFromStatusT(_tmp.error()); \
60 std::move(_tmp.value()); })
61
Andy Hung4e2547c2022-08-29 14:14:58 -070062static audio_channel_mask_t getMaxChannelMask(
63 const std::vector<audio_channel_mask_t>& masks, size_t channelLimit = SIZE_MAX) {
Andy Hunga461a002022-05-17 10:36:02 -070064 uint32_t maxCount = 0;
65 audio_channel_mask_t maxMask = AUDIO_CHANNEL_NONE;
66 for (auto mask : masks) {
67 const size_t count = audio_channel_count_from_out_mask(mask);
Andy Hung4e2547c2022-08-29 14:14:58 -070068 if (count > channelLimit) continue; // ignore masks greater than channelLimit
Andy Hunga461a002022-05-17 10:36:02 -070069 if (count > maxCount) {
70 maxMask = mask;
71 maxCount = count;
72 }
73 }
74 return maxMask;
75}
76
Eric Laurent6d607012021-07-05 11:54:40 +020077// ---------------------------------------------------------------------------
78
Eric Laurent8a4259f2021-09-14 16:04:00 +020079class Spatializer::EngineCallbackHandler : public AHandler {
80public:
81 EngineCallbackHandler(wp<Spatializer> spatializer)
82 : mSpatializer(spatializer) {
83 }
84
85 enum {
86 // Device state callbacks
87 kWhatOnFramesProcessed, // AudioEffect::EVENT_FRAMES_PROCESSED
88 kWhatOnHeadToStagePose, // SpatializerPoseController::Listener::onHeadToStagePose
89 kWhatOnActualModeChange, // SpatializerPoseController::Listener::onActualModeChange
Eric Laurent9c04de92022-07-20 13:49:47 +020090 kWhatOnLatencyModesChanged, // Spatializer::onSupportedLatencyModesChanged
Eric Laurent8a4259f2021-09-14 16:04:00 +020091 };
92 static constexpr const char *kNumFramesKey = "numFrames";
93 static constexpr const char *kModeKey = "mode";
94 static constexpr const char *kTranslation0Key = "translation0";
95 static constexpr const char *kTranslation1Key = "translation1";
96 static constexpr const char *kTranslation2Key = "translation2";
97 static constexpr const char *kRotation0Key = "rotation0";
98 static constexpr const char *kRotation1Key = "rotation1";
99 static constexpr const char *kRotation2Key = "rotation2";
Eric Laurent9c04de92022-07-20 13:49:47 +0200100 static constexpr const char *kLatencyModesKey = "latencyModes";
101
102 class LatencyModes : public RefBase {
103 public:
104 LatencyModes(audio_io_handle_t output,
105 const std::vector<audio_latency_mode_t>& latencyModes)
106 : mOutput(output), mLatencyModes(latencyModes) {}
107 ~LatencyModes() = default;
108
109 audio_io_handle_t mOutput;
110 std::vector<audio_latency_mode_t> mLatencyModes;
111 };
Eric Laurent8a4259f2021-09-14 16:04:00 +0200112
113 void onMessageReceived(const sp<AMessage> &msg) override {
Eric Laurent9c04de92022-07-20 13:49:47 +0200114 sp<Spatializer> spatializer = mSpatializer.promote();
115 if (spatializer == nullptr) {
116 ALOGW("%s: Cannot promote spatializer", __func__);
117 return;
118 }
Eric Laurent8a4259f2021-09-14 16:04:00 +0200119 switch (msg->what()) {
120 case kWhatOnFramesProcessed: {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200121 int numFrames;
122 if (!msg->findInt32(kNumFramesKey, &numFrames)) {
123 ALOGE("%s: Cannot find num frames!", __func__);
124 return;
125 }
126 if (numFrames > 0) {
127 spatializer->calculateHeadPose();
128 }
129 } break;
130 case kWhatOnHeadToStagePose: {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200131 std::vector<float> headToStage(sHeadPoseKeys.size());
132 for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
133 if (!msg->findFloat(sHeadPoseKeys[i], &headToStage[i])) {
134 ALOGE("%s: Cannot find kTranslation0Key!", __func__);
135 return;
136 }
137 }
138 spatializer->onHeadToStagePoseMsg(headToStage);
139 } break;
140 case kWhatOnActualModeChange: {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200141 int mode;
Eric Laurent9c04de92022-07-20 13:49:47 +0200142 if (!msg->findInt32(kModeKey, &mode)) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200143 ALOGE("%s: Cannot find actualMode!", __func__);
144 return;
145 }
146 spatializer->onActualModeChangeMsg(static_cast<HeadTrackingMode>(mode));
147 } break;
Eric Laurent9c04de92022-07-20 13:49:47 +0200148
149 case kWhatOnLatencyModesChanged: {
150 sp<RefBase> object;
151 if (!msg->findObject(kLatencyModesKey, &object)) {
152 ALOGE("%s: Cannot find latency modes!", __func__);
153 return;
154 }
155 sp<LatencyModes> latencyModes = static_cast<LatencyModes*>(object.get());
156 spatializer->onSupportedLatencyModesChangedMsg(
157 latencyModes->mOutput, std::move(latencyModes->mLatencyModes));
158 } break;
159
Eric Laurent8a4259f2021-09-14 16:04:00 +0200160 default:
161 LOG_ALWAYS_FATAL("Invalid callback message %d", msg->what());
162 }
163 }
164private:
165 wp<Spatializer> mSpatializer;
166};
167
168const std::vector<const char *> Spatializer::sHeadPoseKeys = {
169 Spatializer::EngineCallbackHandler::kTranslation0Key,
170 Spatializer::EngineCallbackHandler::kTranslation1Key,
171 Spatializer::EngineCallbackHandler::kTranslation2Key,
172 Spatializer::EngineCallbackHandler::kRotation0Key,
173 Spatializer::EngineCallbackHandler::kRotation1Key,
174 Spatializer::EngineCallbackHandler::kRotation2Key,
175};
176
177// ---------------------------------------------------------------------------
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000178
179// Convert recorded sensor data to string with level indentation.
Shunkai Yao20e23732022-08-25 00:44:04 +0000180std::string Spatializer::HeadToStagePoseRecorder::toString(unsigned level) const {
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000181 std::string prefixSpace(level, ' ');
182 return mPoseRecordLog.dumpToString((prefixSpace + " ").c_str(), Spatializer::mMaxLocalLogLine);
183}
184
185// Compute sensor data, record into local log when it is time.
Shunkai Yao20e23732022-08-25 00:44:04 +0000186void Spatializer::HeadToStagePoseRecorder::record(const std::vector<float>& headToStage) {
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000187 if (headToStage.size() != mPoseVectorSize) return;
188
189 if (mNumOfSampleSinceLastRecord++ == 0) {
190 mFirstSampleTimestamp = std::chrono::steady_clock::now();
191 }
192 // if it's time, do record and reset.
193 if (shouldRecordLog()) {
194 poseSumToAverage();
195 mPoseRecordLog.log(
Shunkai Yao20e23732022-08-25 00:44:04 +0000196 "mean: %s, min: %s, max %s, calculated %d samples in %0.4f second(s)",
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000197 Spatializer::toString<double>(mPoseRadianSum, true /* radianToDegree */).c_str(),
198 Spatializer::toString<float>(mMinPoseAngle, true /* radianToDegree */).c_str(),
199 Spatializer::toString<float>(mMaxPoseAngle, true /* radianToDegree */).c_str(),
Shunkai Yao20e23732022-08-25 00:44:04 +0000200 mNumOfSampleSinceLastRecord, mNumOfSecondsSinceLastRecord.count());
201 resetRecord();
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000202 }
203 // update stream average.
204 for (int i = 0; i < mPoseVectorSize; i++) {
205 mPoseRadianSum[i] += headToStage[i];
206 mMaxPoseAngle[i] = std::max(mMaxPoseAngle[i], headToStage[i]);
207 mMinPoseAngle[i] = std::min(mMinPoseAngle[i], headToStage[i]);
208 }
209 return;
210}
211
212// ---------------------------------------------------------------------------
Eric Laurent6d607012021-07-05 11:54:40 +0200213sp<Spatializer> Spatializer::create(SpatializerPolicyCallback *callback) {
214 sp<Spatializer> spatializer;
215
216 sp<EffectsFactoryHalInterface> effectsFactoryHal = EffectsFactoryHalInterface::create();
217 if (effectsFactoryHal == nullptr) {
218 ALOGW("%s failed to create effect factory interface", __func__);
219 return spatializer;
220 }
221
222 std::vector<effect_descriptor_t> descriptors;
223 status_t status =
Eric Laurent1c5e2e32021-08-18 18:50:28 +0200224 effectsFactoryHal->getDescriptors(FX_IID_SPATIALIZER, &descriptors);
Eric Laurent6d607012021-07-05 11:54:40 +0200225 if (status != NO_ERROR) {
226 ALOGW("%s failed to get spatializer descriptor, error %d", __func__, status);
227 return spatializer;
228 }
229 ALOG_ASSERT(!descriptors.empty(),
230 "%s getDescriptors() returned no error but empty list", __func__);
231
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000232 // TODO: get supported spatialization modes from FX engine or descriptor
Eric Laurent6d607012021-07-05 11:54:40 +0200233 sp<EffectHalInterface> effect;
234 status = effectsFactoryHal->createEffect(&descriptors[0].uuid, AUDIO_SESSION_OUTPUT_STAGE,
235 AUDIO_IO_HANDLE_NONE, AUDIO_PORT_HANDLE_NONE, &effect);
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000236 ALOGI("%s FX create status %d effect ID %" PRId64, __func__, status,
237 effect ? effect->effectId() : 0);
Eric Laurent6d607012021-07-05 11:54:40 +0200238
239 if (status == NO_ERROR && effect != nullptr) {
240 spatializer = new Spatializer(descriptors[0], callback);
Eric Laurent2be8b292021-08-23 09:44:33 -0700241 if (spatializer->loadEngineConfiguration(effect) != NO_ERROR) {
242 spatializer.clear();
Andy Hung4e2547c2022-08-29 14:14:58 -0700243 ALOGW("%s loadEngine error: %d effect Id %" PRId64,
244 __func__, status, effect ? effect->effectId() : 0);
245 } else {
246 spatializer->mLocalLog.log("%s with effect Id %" PRId64, __func__,
247 effect ? effect->effectId() : 0);
Eric Laurent2be8b292021-08-23 09:44:33 -0700248 }
Eric Laurent6d607012021-07-05 11:54:40 +0200249 }
250
251 return spatializer;
252}
253
Eric Laurent2be8b292021-08-23 09:44:33 -0700254Spatializer::Spatializer(effect_descriptor_t engineDescriptor, SpatializerPolicyCallback* callback)
255 : mEngineDescriptor(engineDescriptor),
256 mPolicyCallback(callback) {
Eric Laurent6d607012021-07-05 11:54:40 +0200257 ALOGV("%s", __func__);
258}
259
Eric Laurent8a4259f2021-09-14 16:04:00 +0200260void Spatializer::onFirstRef() {
261 mLooper = new ALooper;
262 mLooper->setName("Spatializer-looper");
263 mLooper->start(
264 /*runOnCallingThread*/false,
265 /*canCallJava*/ false,
266 PRIORITY_AUDIO);
267
268 mHandler = new EngineCallbackHandler(this);
269 mLooper->registerHandler(mHandler);
270}
271
Eric Laurent6d607012021-07-05 11:54:40 +0200272Spatializer::~Spatializer() {
273 ALOGV("%s", __func__);
Eric Laurent8a4259f2021-09-14 16:04:00 +0200274 if (mLooper != nullptr) {
275 mLooper->stop();
276 mLooper->unregisterHandler(mHandler->id());
277 }
278 mLooper.clear();
279 mHandler.clear();
Eric Laurent6d607012021-07-05 11:54:40 +0200280}
281
Andy Hung05874e82022-08-17 17:27:32 -0700282static std::string channelMaskVectorToString(
283 const std::vector<audio_channel_mask_t>& masks) {
284 std::stringstream ss;
285 for (const auto &mask : masks) {
286 if (ss.tellp() != 0) ss << "|";
287 ss << mask;
288 }
289 return ss.str();
290}
291
Eric Laurent2be8b292021-08-23 09:44:33 -0700292status_t Spatializer::loadEngineConfiguration(sp<EffectHalInterface> effect) {
293 ALOGV("%s", __func__);
294
295 std::vector<bool> supportsHeadTracking;
296 status_t status = getHalParameter<false>(effect, SPATIALIZER_PARAM_HEADTRACKING_SUPPORTED,
297 &supportsHeadTracking);
298 if (status != NO_ERROR) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700299 ALOGW("%s: cannot get SPATIALIZER_PARAM_HEADTRACKING_SUPPORTED", __func__);
Eric Laurent2be8b292021-08-23 09:44:33 -0700300 return status;
301 }
302 mSupportsHeadTracking = supportsHeadTracking[0];
303
Andy Hung119dbdb2022-05-11 19:20:13 -0700304 std::vector<media::SpatializationLevel> spatializationLevels;
305 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_LEVELS,
306 &spatializationLevels);
Eric Laurent2be8b292021-08-23 09:44:33 -0700307 if (status != NO_ERROR) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700308 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_LEVELS", __func__);
Eric Laurent2be8b292021-08-23 09:44:33 -0700309 return status;
310 }
Andy Hung119dbdb2022-05-11 19:20:13 -0700311 bool noneLevelFound = false;
312 bool activeLevelFound = false;
313 for (const auto spatializationLevel : spatializationLevels) {
314 if (!aidl_utils::isValidEnum(spatializationLevel)) {
315 ALOGW("%s: ignoring spatializationLevel:%d", __func__, (int)spatializationLevel);
316 continue;
317 }
318 if (spatializationLevel == media::SpatializationLevel::NONE) {
319 noneLevelFound = true;
320 } else {
321 activeLevelFound = true;
322 }
323 // we don't detect duplicates.
324 mLevels.emplace_back(spatializationLevel);
325 }
326 if (!noneLevelFound || !activeLevelFound) {
327 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_LEVELS must include NONE"
328 " and another valid level", __func__);
329 return BAD_VALUE;
330 }
331
332 std::vector<media::SpatializationMode> spatializationModes;
Eric Laurent2be8b292021-08-23 09:44:33 -0700333 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES,
Andy Hung119dbdb2022-05-11 19:20:13 -0700334 &spatializationModes);
Eric Laurent2be8b292021-08-23 09:44:33 -0700335 if (status != NO_ERROR) {
Andy Hung119dbdb2022-05-11 19:20:13 -0700336 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES", __func__);
Eric Laurent2be8b292021-08-23 09:44:33 -0700337 return status;
338 }
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000339
Andy Hung119dbdb2022-05-11 19:20:13 -0700340 for (const auto spatializationMode : spatializationModes) {
341 if (!aidl_utils::isValidEnum(spatializationMode)) {
342 ALOGW("%s: ignoring spatializationMode:%d", __func__, (int)spatializationMode);
343 continue;
344 }
345 // we don't detect duplicates.
346 mSpatializationModes.emplace_back(spatializationMode);
347 }
348 if (mSpatializationModes.empty()) {
349 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES reports empty", __func__);
350 return BAD_VALUE;
351 }
352
353 std::vector<audio_channel_mask_t> channelMasks;
354 status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS,
355 &channelMasks);
356 if (status != NO_ERROR) {
357 ALOGW("%s: cannot get SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS", __func__);
358 return status;
359 }
360 for (const auto channelMask : channelMasks) {
361 if (!audio_is_channel_mask_spatialized(channelMask)) {
362 ALOGW("%s: ignoring channelMask:%#x", __func__, channelMask);
363 continue;
364 }
365 // we don't detect duplicates.
366 mChannelMasks.emplace_back(channelMask);
367 }
368 if (mChannelMasks.empty()) {
369 ALOGW("%s: SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS reports empty", __func__);
370 return BAD_VALUE;
371 }
Andy Hunga461a002022-05-17 10:36:02 -0700372
373 // Currently we expose only RELATIVE_WORLD.
374 // This is a limitation of the head tracking library based on a UX choice.
375 mHeadTrackingModes.push_back(SpatializerHeadTrackingMode::DISABLED);
376 if (mSupportsHeadTracking) {
377 mHeadTrackingModes.push_back(SpatializerHeadTrackingMode::RELATIVE_WORLD);
378 }
379 mediametrics::LogItem(mMetricsId)
380 .set(AMEDIAMETRICS_PROP_EVENT, AMEDIAMETRICS_PROP_EVENT_VALUE_CREATE)
Andy Hung05874e82022-08-17 17:27:32 -0700381 .set(AMEDIAMETRICS_PROP_CHANNELMASKS, channelMaskVectorToString(mChannelMasks))
Andy Hunga461a002022-05-17 10:36:02 -0700382 .set(AMEDIAMETRICS_PROP_LEVELS, aidl_utils::enumsToString(mLevels))
383 .set(AMEDIAMETRICS_PROP_MODES, aidl_utils::enumsToString(mSpatializationModes))
384 .set(AMEDIAMETRICS_PROP_HEADTRACKINGMODES, aidl_utils::enumsToString(mHeadTrackingModes))
385 .set(AMEDIAMETRICS_PROP_STATUS, (int32_t)status)
386 .record();
Andy Hung119dbdb2022-05-11 19:20:13 -0700387 return NO_ERROR;
Eric Laurent2be8b292021-08-23 09:44:33 -0700388}
389
390/** Gets the channel mask, sampling rate and format set for the spatializer input. */
391audio_config_base_t Spatializer::getAudioInConfig() const {
392 std::lock_guard lock(mLock);
393 audio_config_base_t config = AUDIO_CONFIG_BASE_INITIALIZER;
394 // For now use highest supported channel count
Andy Hung4e2547c2022-08-29 14:14:58 -0700395 config.channel_mask = getMaxChannelMask(mChannelMasks, FCC_LIMIT);
Eric Laurent2be8b292021-08-23 09:44:33 -0700396 return config;
397}
398
Eric Laurent6d607012021-07-05 11:54:40 +0200399status_t Spatializer::registerCallback(
400 const sp<media::INativeSpatializerCallback>& callback) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700401 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200402 if (callback == nullptr) {
403 return BAD_VALUE;
404 }
405
406 sp<IBinder> binder = IInterface::asBinder(callback);
407 status_t status = binder->linkToDeath(this);
408 if (status == NO_ERROR) {
409 mSpatializerCallback = callback;
410 }
411 ALOGV("%s status %d", __func__, status);
412 return status;
413}
414
415// IBinder::DeathRecipient
416void Spatializer::binderDied(__unused const wp<IBinder> &who) {
417 {
Eric Laurent2be8b292021-08-23 09:44:33 -0700418 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200419 mLevel = SpatializationLevel::NONE;
420 mSpatializerCallback.clear();
421 }
422 ALOGV("%s", __func__);
423 mPolicyCallback->onCheckSpatializer();
424}
425
426// ISpatializer
427Status Spatializer::getSupportedLevels(std::vector<SpatializationLevel> *levels) {
428 ALOGV("%s", __func__);
429 if (levels == nullptr) {
430 return binderStatusFromStatusT(BAD_VALUE);
431 }
Andy Hunga461a002022-05-17 10:36:02 -0700432 // SpatializationLevel::NONE is already required from the effect or we don't load it.
Eric Laurent2be8b292021-08-23 09:44:33 -0700433 levels->insert(levels->end(), mLevels.begin(), mLevels.end());
Eric Laurent6d607012021-07-05 11:54:40 +0200434 return Status::ok();
435}
436
Eric Laurent2be8b292021-08-23 09:44:33 -0700437Status Spatializer::setLevel(SpatializationLevel level) {
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000438 ALOGV("%s level %s", __func__, media::toString(level).c_str());
439 mLocalLog.log("%s with %s", __func__, media::toString(level).c_str());
Eric Laurent6d607012021-07-05 11:54:40 +0200440 if (level != SpatializationLevel::NONE
Eric Laurent2be8b292021-08-23 09:44:33 -0700441 && std::find(mLevels.begin(), mLevels.end(), level) == mLevels.end()) {
Eric Laurent6d607012021-07-05 11:54:40 +0200442 return binderStatusFromStatusT(BAD_VALUE);
443 }
444 sp<media::INativeSpatializerCallback> callback;
445 bool levelChanged = false;
446 {
Eric Laurent2be8b292021-08-23 09:44:33 -0700447 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200448 levelChanged = mLevel != level;
449 mLevel = level;
450 callback = mSpatializerCallback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700451
452 if (levelChanged && mEngine != nullptr) {
Eric Laurent7ea0d1b2022-04-01 14:23:44 +0200453 checkEngineState_l();
Eric Laurent2be8b292021-08-23 09:44:33 -0700454 }
Eric Laurent9249d342022-03-18 11:55:56 +0100455 checkSensorsState_l();
Eric Laurent6d607012021-07-05 11:54:40 +0200456 }
457
458 if (levelChanged) {
459 mPolicyCallback->onCheckSpatializer();
460 if (callback != nullptr) {
461 callback->onLevelChanged(level);
462 }
463 }
464 return Status::ok();
465}
466
Eric Laurent2be8b292021-08-23 09:44:33 -0700467Status Spatializer::getLevel(SpatializationLevel *level) {
Eric Laurent6d607012021-07-05 11:54:40 +0200468 if (level == nullptr) {
469 return binderStatusFromStatusT(BAD_VALUE);
470 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700471 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200472 *level = mLevel;
473 ALOGV("%s level %d", __func__, (int)*level);
474 return Status::ok();
475}
476
Eric Laurentc87402b2021-09-17 16:49:42 +0200477Status Spatializer::isHeadTrackingSupported(bool *supports) {
478 ALOGV("%s mSupportsHeadTracking %d", __func__, mSupportsHeadTracking);
479 if (supports == nullptr) {
480 return binderStatusFromStatusT(BAD_VALUE);
481 }
482 std::lock_guard lock(mLock);
483 *supports = mSupportsHeadTracking;
484 return Status::ok();
485}
486
Eric Laurent6d607012021-07-05 11:54:40 +0200487Status Spatializer::getSupportedHeadTrackingModes(
Eric Laurent2be8b292021-08-23 09:44:33 -0700488 std::vector<SpatializerHeadTrackingMode>* modes) {
489 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200490 ALOGV("%s", __func__);
491 if (modes == nullptr) {
492 return binderStatusFromStatusT(BAD_VALUE);
493 }
Andy Hunga461a002022-05-17 10:36:02 -0700494 modes->insert(modes->end(), mHeadTrackingModes.begin(), mHeadTrackingModes.end());
Eric Laurent6d607012021-07-05 11:54:40 +0200495 return Status::ok();
496}
497
Eric Laurent2be8b292021-08-23 09:44:33 -0700498Status Spatializer::setDesiredHeadTrackingMode(SpatializerHeadTrackingMode mode) {
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000499 ALOGV("%s mode %s", __func__, media::toString(mode).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700500
501 if (!mSupportsHeadTracking) {
502 return binderStatusFromStatusT(INVALID_OPERATION);
503 }
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000504 mLocalLog.log("%s with %s", __func__, media::toString(mode).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700505 std::lock_guard lock(mLock);
506 switch (mode) {
507 case SpatializerHeadTrackingMode::OTHER:
508 return binderStatusFromStatusT(BAD_VALUE);
509 case SpatializerHeadTrackingMode::DISABLED:
510 mDesiredHeadTrackingMode = HeadTrackingMode::STATIC;
511 break;
512 case SpatializerHeadTrackingMode::RELATIVE_WORLD:
513 mDesiredHeadTrackingMode = HeadTrackingMode::WORLD_RELATIVE;
514 break;
515 case SpatializerHeadTrackingMode::RELATIVE_SCREEN:
516 mDesiredHeadTrackingMode = HeadTrackingMode::SCREEN_RELATIVE;
517 break;
518 }
519
Eric Laurent11094172022-04-05 18:27:42 +0200520 checkPoseController_l();
521 checkSensorsState_l();
Eric Laurent2be8b292021-08-23 09:44:33 -0700522
523 return Status::ok();
524}
525
526Status Spatializer::getActualHeadTrackingMode(SpatializerHeadTrackingMode *mode) {
Eric Laurent6d607012021-07-05 11:54:40 +0200527 if (mode == nullptr) {
528 return binderStatusFromStatusT(BAD_VALUE);
529 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700530 std::lock_guard lock(mLock);
531 *mode = mActualHeadTrackingMode;
Eric Laurent6d607012021-07-05 11:54:40 +0200532 ALOGV("%s mode %d", __func__, (int)*mode);
533 return Status::ok();
534}
535
Ytai Ben-Tsvia16a9df2021-08-05 08:57:06 -0700536Status Spatializer::recenterHeadTracker() {
Eric Laurent780be4a2021-09-16 10:44:24 +0200537 if (!mSupportsHeadTracking) {
538 return binderStatusFromStatusT(INVALID_OPERATION);
539 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700540 std::lock_guard lock(mLock);
541 if (mPoseController != nullptr) {
542 mPoseController->recenter();
543 }
Eric Laurent6d607012021-07-05 11:54:40 +0200544 return Status::ok();
545}
546
547Status Spatializer::setGlobalTransform(const std::vector<float>& screenToStage) {
Eric Laurent6d607012021-07-05 11:54:40 +0200548 ALOGV("%s", __func__);
Eric Laurent780be4a2021-09-16 10:44:24 +0200549 if (!mSupportsHeadTracking) {
550 return binderStatusFromStatusT(INVALID_OPERATION);
551 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700552 std::optional<Pose3f> maybePose = Pose3f::fromVector(screenToStage);
553 if (!maybePose.has_value()) {
554 ALOGW("Invalid screenToStage vector.");
555 return binderStatusFromStatusT(BAD_VALUE);
556 }
557 std::lock_guard lock(mLock);
558 if (mPoseController != nullptr) {
Shunkai Yao20e23732022-08-25 00:44:04 +0000559 mLocalLog.log("%s with screenToStage %s", __func__, toString<float>(screenToStage).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700560 mPoseController->setScreenToStagePose(maybePose.value());
561 }
Eric Laurent6d607012021-07-05 11:54:40 +0200562 return Status::ok();
563}
564
565Status Spatializer::release() {
566 ALOGV("%s", __func__);
567 bool levelChanged = false;
568 {
Eric Laurent2be8b292021-08-23 09:44:33 -0700569 std::lock_guard lock(mLock);
Eric Laurent6d607012021-07-05 11:54:40 +0200570 if (mSpatializerCallback == nullptr) {
571 return binderStatusFromStatusT(INVALID_OPERATION);
572 }
573
574 sp<IBinder> binder = IInterface::asBinder(mSpatializerCallback);
575 binder->unlinkToDeath(this);
576 mSpatializerCallback.clear();
577
578 levelChanged = mLevel != SpatializationLevel::NONE;
579 mLevel = SpatializationLevel::NONE;
580 }
581
582 if (levelChanged) {
583 mPolicyCallback->onCheckSpatializer();
584 }
585 return Status::ok();
586}
587
Eric Laurent2be8b292021-08-23 09:44:33 -0700588Status Spatializer::setHeadSensor(int sensorHandle) {
589 ALOGV("%s sensorHandle %d", __func__, sensorHandle);
Eric Laurent780be4a2021-09-16 10:44:24 +0200590 if (!mSupportsHeadTracking) {
591 return binderStatusFromStatusT(INVALID_OPERATION);
592 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700593 std::lock_guard lock(mLock);
Andy Hungba2a61a2022-05-20 12:00:28 -0700594 if (mHeadSensor != sensorHandle) {
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000595 mLocalLog.log("%s with 0x%08x", __func__, sensorHandle);
Andy Hungba2a61a2022-05-20 12:00:28 -0700596 mHeadSensor = sensorHandle;
597 checkPoseController_l();
598 checkSensorsState_l();
599 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700600 return Status::ok();
601}
602
603Status Spatializer::setScreenSensor(int sensorHandle) {
604 ALOGV("%s sensorHandle %d", __func__, sensorHandle);
Eric Laurent780be4a2021-09-16 10:44:24 +0200605 if (!mSupportsHeadTracking) {
606 return binderStatusFromStatusT(INVALID_OPERATION);
607 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700608 std::lock_guard lock(mLock);
Andy Hungba2a61a2022-05-20 12:00:28 -0700609 if (mScreenSensor != sensorHandle) {
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000610 mLocalLog.log("%s with 0x%08x", __func__, sensorHandle);
Andy Hungba2a61a2022-05-20 12:00:28 -0700611 mScreenSensor = sensorHandle;
612 // TODO: consider a new method setHeadAndScreenSensor()
613 // because we generally set both at the same time.
614 // This will avoid duplicated work and recentering.
615 checkSensorsState_l();
616 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700617 return Status::ok();
618}
619
620Status Spatializer::setDisplayOrientation(float physicalToLogicalAngle) {
621 ALOGV("%s physicalToLogicalAngle %f", __func__, physicalToLogicalAngle);
Eric Laurent780be4a2021-09-16 10:44:24 +0200622 if (!mSupportsHeadTracking) {
623 return binderStatusFromStatusT(INVALID_OPERATION);
624 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700625 std::lock_guard lock(mLock);
626 mDisplayOrientation = physicalToLogicalAngle;
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000627 mLocalLog.log("%s with %f", __func__, physicalToLogicalAngle);
Eric Laurent2be8b292021-08-23 09:44:33 -0700628 if (mPoseController != nullptr) {
629 mPoseController->setDisplayOrientation(mDisplayOrientation);
630 }
Eric Laurent16ddaf42021-09-17 15:00:35 +0200631 if (mEngine != nullptr) {
632 setEffectParameter_l(
633 SPATIALIZER_PARAM_DISPLAY_ORIENTATION, std::vector<float>{physicalToLogicalAngle});
634 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700635 return Status::ok();
636}
637
638Status Spatializer::setHingeAngle(float hingeAngle) {
639 std::lock_guard lock(mLock);
640 ALOGV("%s hingeAngle %f", __func__, hingeAngle);
641 if (mEngine != nullptr) {
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000642 mLocalLog.log("%s with %f", __func__, hingeAngle);
Eric Laurent2be8b292021-08-23 09:44:33 -0700643 setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE, std::vector<float>{hingeAngle});
644 }
645 return Status::ok();
646}
647
648Status Spatializer::getSupportedModes(std::vector<SpatializationMode> *modes) {
649 ALOGV("%s", __func__);
650 if (modes == nullptr) {
651 return binderStatusFromStatusT(BAD_VALUE);
652 }
653 *modes = mSpatializationModes;
654 return Status::ok();
655}
656
Eric Laurent67816e32021-09-16 15:18:40 +0200657Status Spatializer::registerHeadTrackingCallback(
658 const sp<media::ISpatializerHeadTrackingCallback>& callback) {
659 ALOGV("%s callback %p", __func__, callback.get());
660 std::lock_guard lock(mLock);
661 if (!mSupportsHeadTracking) {
662 return binderStatusFromStatusT(INVALID_OPERATION);
663 }
664 mHeadTrackingCallback = callback;
665 return Status::ok();
666}
667
Eric Laurentc87402b2021-09-17 16:49:42 +0200668Status Spatializer::setParameter(int key, const std::vector<unsigned char>& value) {
669 ALOGV("%s key %d", __func__, key);
670 std::lock_guard lock(mLock);
671 status_t status = INVALID_OPERATION;
672 if (mEngine != nullptr) {
673 status = setEffectParameter_l(key, value);
674 }
675 return binderStatusFromStatusT(status);
676}
677
678Status Spatializer::getParameter(int key, std::vector<unsigned char> *value) {
Greg Kaiserf7249f82021-09-21 07:10:12 -0700679 ALOGV("%s key %d value size %d", __func__, key,
680 (value != nullptr ? (int)value->size() : -1));
Eric Laurentc87402b2021-09-17 16:49:42 +0200681 if (value == nullptr) {
George Burgess IV22386222021-09-22 12:09:31 -0700682 return binderStatusFromStatusT(BAD_VALUE);
Eric Laurentc87402b2021-09-17 16:49:42 +0200683 }
684 std::lock_guard lock(mLock);
685 status_t status = INVALID_OPERATION;
686 if (mEngine != nullptr) {
687 ALOGV("%s key %d mEngine %p", __func__, key, mEngine.get());
688 status = getEffectParameter_l(key, value);
689 }
690 return binderStatusFromStatusT(status);
691}
692
693Status Spatializer::getOutput(int *output) {
694 ALOGV("%s", __func__);
695 if (output == nullptr) {
696 binderStatusFromStatusT(BAD_VALUE);
697 }
698 std::lock_guard lock(mLock);
699 *output = VALUE_OR_RETURN_BINDER_STATUS(legacy2aidl_audio_io_handle_t_int32_t(mOutput));
700 ALOGV("%s got output %d", __func__, *output);
701 return Status::ok();
702}
703
Eric Laurent2be8b292021-08-23 09:44:33 -0700704// SpatializerPoseController::Listener
705void Spatializer::onHeadToStagePose(const Pose3f& headToStage) {
706 ALOGV("%s", __func__);
Eric Laurent780be4a2021-09-16 10:44:24 +0200707 LOG_ALWAYS_FATAL_IF(!mSupportsHeadTracking,
708 "onHeadToStagePose() called with no head tracking support!");
709
Eric Laurent2be8b292021-08-23 09:44:33 -0700710 auto vec = headToStage.toVector();
Eric Laurent8a4259f2021-09-14 16:04:00 +0200711 LOG_ALWAYS_FATAL_IF(vec.size() != sHeadPoseKeys.size(),
712 "%s invalid head to stage vector size %zu", __func__, vec.size());
Eric Laurent8a4259f2021-09-14 16:04:00 +0200713 sp<AMessage> msg =
714 new AMessage(EngineCallbackHandler::kWhatOnHeadToStagePose, mHandler);
715 for (size_t i = 0 ; i < sHeadPoseKeys.size(); i++) {
716 msg->setFloat(sHeadPoseKeys[i], vec[i]);
717 }
718 msg->post();
719}
720
721void Spatializer::onHeadToStagePoseMsg(const std::vector<float>& headToStage) {
722 ALOGV("%s", __func__);
Eric Laurent67816e32021-09-16 15:18:40 +0200723 sp<media::ISpatializerHeadTrackingCallback> callback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700724 {
725 std::lock_guard lock(mLock);
Eric Laurent67816e32021-09-16 15:18:40 +0200726 callback = mHeadTrackingCallback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700727 if (mEngine != nullptr) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200728 setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, headToStage);
Shunkai Yao20e23732022-08-25 00:44:04 +0000729 mPoseRecorder.record(headToStage);
730 mPoseDurableRecorder.record(headToStage);
Eric Laurent2be8b292021-08-23 09:44:33 -0700731 }
732 }
733
734 if (callback != nullptr) {
Eric Laurent8a4259f2021-09-14 16:04:00 +0200735 callback->onHeadToSoundStagePoseUpdated(headToStage);
Eric Laurent2be8b292021-08-23 09:44:33 -0700736 }
737}
738
739void Spatializer::onActualModeChange(HeadTrackingMode mode) {
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000740 std::string modeStr = SpatializerPoseController::toString(mode);
741 ALOGV("%s(%s)", __func__, modeStr.c_str());
742 mLocalLog.log("%s with %s", __func__, modeStr.c_str());
Eric Laurent8a4259f2021-09-14 16:04:00 +0200743 sp<AMessage> msg =
744 new AMessage(EngineCallbackHandler::kWhatOnActualModeChange, mHandler);
745 msg->setInt32(EngineCallbackHandler::kModeKey, static_cast<int>(mode));
746 msg->post();
747}
748
749void Spatializer::onActualModeChangeMsg(HeadTrackingMode mode) {
750 ALOGV("%s(%d)", __func__, (int) mode);
Eric Laurent67816e32021-09-16 15:18:40 +0200751 sp<media::ISpatializerHeadTrackingCallback> callback;
Eric Laurent2be8b292021-08-23 09:44:33 -0700752 SpatializerHeadTrackingMode spatializerMode;
753 {
754 std::lock_guard lock(mLock);
755 if (!mSupportsHeadTracking) {
756 spatializerMode = SpatializerHeadTrackingMode::DISABLED;
757 } else {
758 switch (mode) {
759 case HeadTrackingMode::STATIC:
760 spatializerMode = SpatializerHeadTrackingMode::DISABLED;
761 break;
762 case HeadTrackingMode::WORLD_RELATIVE:
763 spatializerMode = SpatializerHeadTrackingMode::RELATIVE_WORLD;
764 break;
765 case HeadTrackingMode::SCREEN_RELATIVE:
766 spatializerMode = SpatializerHeadTrackingMode::RELATIVE_SCREEN;
767 break;
768 default:
769 LOG_ALWAYS_FATAL("Unknown mode: %d", mode);
770 }
771 }
772 mActualHeadTrackingMode = spatializerMode;
Eric Laurente51f80e2022-04-14 10:20:38 +0200773 if (mEngine != nullptr) {
Eric Laurent7ea0d1b2022-04-01 14:23:44 +0200774 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
775 std::vector<SpatializerHeadTrackingMode>{spatializerMode});
776 }
Eric Laurent67816e32021-09-16 15:18:40 +0200777 callback = mHeadTrackingCallback;
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000778 mLocalLog.log("%s: %s, spatializerMode %s", __func__,
779 SpatializerPoseController::toString(mode).c_str(),
780 media::toString(spatializerMode).c_str());
Eric Laurent2be8b292021-08-23 09:44:33 -0700781 }
Eric Laurente51f80e2022-04-14 10:20:38 +0200782 if (callback != nullptr) {
Eric Laurent2be8b292021-08-23 09:44:33 -0700783 callback->onHeadTrackingModeChanged(spatializerMode);
784 }
785}
786
Eric Laurent15903592022-02-24 20:44:36 +0100787status_t Spatializer::attachOutput(audio_io_handle_t output, size_t numActiveTracks) {
Eric Laurent4a872862021-10-11 17:06:47 +0200788 bool outputChanged = false;
789 sp<media::INativeSpatializerCallback> callback;
790
Eric Laurent2be8b292021-08-23 09:44:33 -0700791 {
792 std::lock_guard lock(mLock);
793 ALOGV("%s output %d mOutput %d", __func__, (int)output, (int)mOutput);
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000794 mLocalLog.log("%s with output %d tracks %zu (mOutput %d)", __func__, (int)output,
795 numActiveTracks, (int)mOutput);
Eric Laurent2be8b292021-08-23 09:44:33 -0700796 if (mOutput != AUDIO_IO_HANDLE_NONE) {
797 LOG_ALWAYS_FATAL_IF(mEngine == nullptr, "%s output set without FX engine", __func__);
798 // remove FX instance
799 mEngine->setEnabled(false);
800 mEngine.clear();
Eric Laurent15903592022-02-24 20:44:36 +0100801 mPoseController.reset();
Eric Laurentee398ad2022-05-03 18:19:35 +0200802 AudioSystem::removeSupportedLatencyModesCallback(this);
Eric Laurent2be8b292021-08-23 09:44:33 -0700803 }
Eric Laurentee398ad2022-05-03 18:19:35 +0200804
Eric Laurent2be8b292021-08-23 09:44:33 -0700805 // create FX instance on output
806 AttributionSourceState attributionSource = AttributionSourceState();
807 mEngine = new AudioEffect(attributionSource);
808 mEngine->set(nullptr, &mEngineDescriptor.uuid, 0, Spatializer::engineCallback /* cbf */,
809 this /* user */, AUDIO_SESSION_OUTPUT_STAGE, output, {} /* device */,
810 false /* probe */, true /* notifyFramesProcessed */);
811 status_t status = mEngine->initCheck();
812 ALOGV("%s mEngine create status %d", __func__, (int)status);
813 if (status != NO_ERROR) {
814 return status;
815 }
816
Eric Laurent4a872862021-10-11 17:06:47 +0200817 outputChanged = mOutput != output;
Eric Laurent2be8b292021-08-23 09:44:33 -0700818 mOutput = output;
Eric Laurent11094172022-04-05 18:27:42 +0200819 mNumActiveTracks = numActiveTracks;
Eric Laurentee398ad2022-05-03 18:19:35 +0200820 AudioSystem::addSupportedLatencyModesCallback(this);
821
822 std::vector<audio_latency_mode_t> latencyModes;
823 status = AudioSystem::getSupportedLatencyModes(mOutput, &latencyModes);
824 if (status == OK) {
825 mSupportedLatencyModes = latencyModes;
826 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700827
Eric Laurent11094172022-04-05 18:27:42 +0200828 checkEngineState_l();
Eric Laurent780be4a2021-09-16 10:44:24 +0200829 if (mSupportsHeadTracking) {
Eric Laurent11094172022-04-05 18:27:42 +0200830 checkPoseController_l();
Eric Laurent9249d342022-03-18 11:55:56 +0100831 checkSensorsState_l();
Eric Laurent780be4a2021-09-16 10:44:24 +0200832 }
Eric Laurent4a872862021-10-11 17:06:47 +0200833 callback = mSpatializerCallback;
Eric Laurent6d607012021-07-05 11:54:40 +0200834 }
Eric Laurent4a872862021-10-11 17:06:47 +0200835
836 if (outputChanged && callback != nullptr) {
837 callback->onOutputChanged(output);
838 }
839
Eric Laurent6d607012021-07-05 11:54:40 +0200840 return NO_ERROR;
841}
842
843audio_io_handle_t Spatializer::detachOutput() {
Eric Laurent2be8b292021-08-23 09:44:33 -0700844 audio_io_handle_t output = AUDIO_IO_HANDLE_NONE;
Eric Laurent4a872862021-10-11 17:06:47 +0200845 sp<media::INativeSpatializerCallback> callback;
846
847 {
848 std::lock_guard lock(mLock);
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000849 mLocalLog.log("%s with output %d tracks %zu", __func__, (int)mOutput, mNumActiveTracks);
Eric Laurent4a872862021-10-11 17:06:47 +0200850 ALOGV("%s mOutput %d", __func__, (int)mOutput);
851 if (mOutput == AUDIO_IO_HANDLE_NONE) {
852 return output;
853 }
854 // remove FX instance
855 mEngine->setEnabled(false);
856 mEngine.clear();
Eric Laurentee398ad2022-05-03 18:19:35 +0200857 AudioSystem::removeSupportedLatencyModesCallback(this);
Eric Laurent4a872862021-10-11 17:06:47 +0200858 output = mOutput;
859 mOutput = AUDIO_IO_HANDLE_NONE;
860 mPoseController.reset();
Eric Laurent4a872862021-10-11 17:06:47 +0200861 callback = mSpatializerCallback;
Eric Laurent6d607012021-07-05 11:54:40 +0200862 }
Eric Laurent4a872862021-10-11 17:06:47 +0200863
864 if (callback != nullptr) {
865 callback->onOutputChanged(AUDIO_IO_HANDLE_NONE);
866 }
Eric Laurent6d607012021-07-05 11:54:40 +0200867 return output;
868}
869
Eric Laurentee398ad2022-05-03 18:19:35 +0200870void Spatializer::onSupportedLatencyModesChanged(
871 audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes) {
Eric Laurent9c04de92022-07-20 13:49:47 +0200872 ALOGV("%s output %d num modes %zu", __func__, (int)output, modes.size());
873 sp<AMessage> msg =
874 new AMessage(EngineCallbackHandler::kWhatOnLatencyModesChanged, mHandler);
875 msg->setObject(EngineCallbackHandler::kLatencyModesKey,
876 sp<EngineCallbackHandler::LatencyModes>::make(output, modes));
877 msg->post();
878}
879
880void Spatializer::onSupportedLatencyModesChangedMsg(
881 audio_io_handle_t output, std::vector<audio_latency_mode_t>&& modes) {
Eric Laurentee398ad2022-05-03 18:19:35 +0200882 std::lock_guard lock(mLock);
Eric Laurent9c04de92022-07-20 13:49:47 +0200883 ALOGV("%s output %d mOutput %d num modes %zu",
884 __func__, (int)output, (int)mOutput, modes.size());
Eric Laurentee398ad2022-05-03 18:19:35 +0200885 if (output == mOutput) {
Eric Laurent9c04de92022-07-20 13:49:47 +0200886 mSupportedLatencyModes = std::move(modes);
Eric Laurentee398ad2022-05-03 18:19:35 +0200887 checkSensorsState_l();
888 }
889}
890
Eric Laurent15903592022-02-24 20:44:36 +0100891void Spatializer::updateActiveTracks(size_t numActiveTracks) {
892 std::lock_guard lock(mLock);
Eric Laurent7ea0d1b2022-04-01 14:23:44 +0200893 if (mNumActiveTracks != numActiveTracks) {
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000894 mLocalLog.log("%s from %zu to %zu", __func__, mNumActiveTracks, numActiveTracks);
Eric Laurent7ea0d1b2022-04-01 14:23:44 +0200895 mNumActiveTracks = numActiveTracks;
896 checkEngineState_l();
897 checkSensorsState_l();
898 }
Eric Laurent15903592022-02-24 20:44:36 +0100899}
900
Eric Laurent9249d342022-03-18 11:55:56 +0100901void Spatializer::checkSensorsState_l() {
Eric Laurentee398ad2022-05-03 18:19:35 +0200902 audio_latency_mode_t requestedLatencyMode = AUDIO_LATENCY_MODE_FREE;
903 bool lowLatencySupported = mSupportedLatencyModes.empty()
904 || (std::find(mSupportedLatencyModes.begin(), mSupportedLatencyModes.end(),
905 AUDIO_LATENCY_MODE_LOW) != mSupportedLatencyModes.end());
Eric Laurent9bcefc62022-07-08 13:35:46 +0200906 if (mSupportsHeadTracking && mPoseController != nullptr) {
907 if (lowLatencySupported && mNumActiveTracks > 0 && mLevel != SpatializationLevel::NONE
Eric Laurent15903592022-02-24 20:44:36 +0100908 && mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
909 && mHeadSensor != SpatializerPoseController::INVALID_SENSOR) {
910 mPoseController->setHeadSensor(mHeadSensor);
Eric Laurent9249d342022-03-18 11:55:56 +0100911 mPoseController->setScreenSensor(mScreenSensor);
Eric Laurentee398ad2022-05-03 18:19:35 +0200912 requestedLatencyMode = AUDIO_LATENCY_MODE_LOW;
Eric Laurent15903592022-02-24 20:44:36 +0100913 } else {
914 mPoseController->setHeadSensor(SpatializerPoseController::INVALID_SENSOR);
Eric Laurent9249d342022-03-18 11:55:56 +0100915 mPoseController->setScreenSensor(SpatializerPoseController::INVALID_SENSOR);
Eric Laurent15903592022-02-24 20:44:36 +0100916 }
917 }
Eric Laurentee398ad2022-05-03 18:19:35 +0200918 if (mOutput != AUDIO_IO_HANDLE_NONE) {
919 AudioSystem::setRequestedLatencyMode(mOutput, requestedLatencyMode);
920 }
Eric Laurent15903592022-02-24 20:44:36 +0100921}
922
Eric Laurent7ea0d1b2022-04-01 14:23:44 +0200923void Spatializer::checkEngineState_l() {
924 if (mEngine != nullptr) {
925 if (mLevel != SpatializationLevel::NONE && mNumActiveTracks > 0) {
926 mEngine->setEnabled(true);
927 setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
928 std::vector<SpatializationLevel>{mLevel});
929 setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
930 std::vector<SpatializerHeadTrackingMode>{mActualHeadTrackingMode});
931 } else {
932 setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
933 std::vector<SpatializationLevel>{SpatializationLevel::NONE});
934 mEngine->setEnabled(false);
935 }
936 }
937}
938
Eric Laurent11094172022-04-05 18:27:42 +0200939void Spatializer::checkPoseController_l() {
940 bool isControllerNeeded = mDesiredHeadTrackingMode != HeadTrackingMode::STATIC
941 && mHeadSensor != SpatializerPoseController::INVALID_SENSOR;
942
943 if (isControllerNeeded && mPoseController == nullptr) {
944 mPoseController = std::make_shared<SpatializerPoseController>(
945 static_cast<SpatializerPoseController::Listener*>(this),
Eric Laurente51f80e2022-04-14 10:20:38 +0200946 10ms, std::nullopt);
Eric Laurent11094172022-04-05 18:27:42 +0200947 LOG_ALWAYS_FATAL_IF(mPoseController == nullptr,
948 "%s could not allocate pose controller", __func__);
949 mPoseController->setDisplayOrientation(mDisplayOrientation);
950 } else if (!isControllerNeeded && mPoseController != nullptr) {
951 mPoseController.reset();
952 }
953 if (mPoseController != nullptr) {
954 mPoseController->setDesiredMode(mDesiredHeadTrackingMode);
955 }
956}
957
Eric Laurent2be8b292021-08-23 09:44:33 -0700958void Spatializer::calculateHeadPose() {
959 ALOGV("%s", __func__);
960 std::lock_guard lock(mLock);
961 if (mPoseController != nullptr) {
962 mPoseController->calculateAsync();
963 }
964}
Eric Laurent6d607012021-07-05 11:54:40 +0200965
Eric Laurent2be8b292021-08-23 09:44:33 -0700966void Spatializer::engineCallback(int32_t event, void *user, void *info) {
Eric Laurent6d607012021-07-05 11:54:40 +0200967 if (user == nullptr) {
968 return;
969 }
Eric Laurent2be8b292021-08-23 09:44:33 -0700970 Spatializer* const me = reinterpret_cast<Spatializer *>(user);
Eric Laurent6d607012021-07-05 11:54:40 +0200971 switch (event) {
972 case AudioEffect::EVENT_FRAMES_PROCESSED: {
Eric Laurent2be8b292021-08-23 09:44:33 -0700973 int frames = info == nullptr ? 0 : *(int*)info;
Eric Laurent9249d342022-03-18 11:55:56 +0100974 ALOGV("%s frames processed %d for me %p", __func__, frames, me);
Eric Laurent8a4259f2021-09-14 16:04:00 +0200975 me->postFramesProcessedMsg(frames);
Eric Laurent2be8b292021-08-23 09:44:33 -0700976 } break;
Eric Laurent6d607012021-07-05 11:54:40 +0200977 default:
Eric Laurent9249d342022-03-18 11:55:56 +0100978 ALOGV("%s event %d", __func__, event);
Eric Laurent6d607012021-07-05 11:54:40 +0200979 break;
980 }
981}
982
Eric Laurent8a4259f2021-09-14 16:04:00 +0200983void Spatializer::postFramesProcessedMsg(int frames) {
984 sp<AMessage> msg =
985 new AMessage(EngineCallbackHandler::kWhatOnFramesProcessed, mHandler);
986 msg->setInt32(EngineCallbackHandler::kNumFramesKey, frames);
987 msg->post();
988}
989
Shunkai Yao59b27bc2022-07-22 18:42:27 +0000990std::string Spatializer::toString(unsigned level) const {
991 std::string prefixSpace;
992 prefixSpace.append(level, ' ');
993 std::string ss = prefixSpace + "Spatializer:\n";
994 bool needUnlock = false;
995
996 prefixSpace += ' ';
997 if (!mLock.try_lock()) {
998 // dumpsys even try_lock failed, information dump can be useful although may not accurate
999 ss.append(prefixSpace).append("try_lock failed, dumpsys below maybe INACCURATE!\n");
1000 } else {
1001 needUnlock = true;
1002 }
1003
1004 // Spatializer class information.
1005 // 1. Capabilities (mLevels, mHeadTrackingModes, mSpatializationModes, mChannelMasks, etc)
1006 ss.append(prefixSpace).append("Supported levels: [");
1007 for (auto& level : mLevels) {
1008 base::StringAppendF(&ss, " %s", media::toString(level).c_str());
1009 }
1010 base::StringAppendF(&ss, "], mLevel: %s", media::toString(mLevel).c_str());
1011
1012 base::StringAppendF(&ss, "\n%smHeadTrackingModes: [", prefixSpace.c_str());
1013 for (auto& mode : mHeadTrackingModes) {
1014 base::StringAppendF(&ss, " %s", media::toString(mode).c_str());
1015 }
1016 base::StringAppendF(&ss, "], Desired: %s, Actual %s\n",
1017 SpatializerPoseController::toString(mDesiredHeadTrackingMode).c_str(),
1018 media::toString(mActualHeadTrackingMode).c_str());
1019
1020 base::StringAppendF(&ss, "%smSpatializationModes: [", prefixSpace.c_str());
1021 for (auto& mode : mSpatializationModes) {
1022 base::StringAppendF(&ss, " %s", media::toString(mode).c_str());
1023 }
1024 ss += "]\n";
1025
1026 base::StringAppendF(&ss, "%smChannelMasks: ", prefixSpace.c_str());
1027 for (auto& mask : mChannelMasks) {
1028 base::StringAppendF(&ss, "%s", audio_channel_out_mask_to_string(mask));
1029 }
1030 base::StringAppendF(&ss, "\n%smSupportsHeadTracking: %s\n", prefixSpace.c_str(),
1031 mSupportsHeadTracking ? "true" : "false");
1032 // 2. Settings (Output, tracks)
1033 base::StringAppendF(&ss, "%smNumActiveTracks: %zu\n", prefixSpace.c_str(), mNumActiveTracks);
1034 base::StringAppendF(&ss, "%sOutputStreamHandle: %d\n", prefixSpace.c_str(), (int)mOutput);
1035
1036 // 3. Sensors, Effect information.
1037 base::StringAppendF(&ss, "%sHeadSensorHandle: 0x%08x\n", prefixSpace.c_str(), mHeadSensor);
1038 base::StringAppendF(&ss, "%sScreenSensorHandle: 0x%08x\n", prefixSpace.c_str(), mScreenSensor);
1039 base::StringAppendF(&ss, "%sEffectHandle: %p\n", prefixSpace.c_str(), mEngine.get());
1040 base::StringAppendF(&ss, "%sDisplayOrientation: %f\n", prefixSpace.c_str(),
1041 mDisplayOrientation);
1042
1043 ss.append(prefixSpace + "CommandLog:\n");
1044 ss += mLocalLog.dumpToString((prefixSpace + " ").c_str(), mMaxLocalLogLine);
Shunkai Yao59b27bc2022-07-22 18:42:27 +00001045
1046 // PostController dump.
1047 if (mPoseController != nullptr) {
1048 ss += mPoseController->toString(level + 1);
Shunkai Yao20e23732022-08-25 00:44:04 +00001049 ss.append(prefixSpace +
1050 "Sensor data format - [rx, ry, rz, vx, vy, vz] (units-degree, "
1051 "r-transform, v-angular velocity, x-pitch, y-roll, z-yaw):\n");
1052 ss.append(prefixSpace + "PerMinuteHistory:\n");
1053 ss += mPoseDurableRecorder.toString(level + 1);
1054 ss.append(prefixSpace + "PerSecondHistory:\n");
1055 ss += mPoseRecorder.toString(level + 1);
Shunkai Yao59b27bc2022-07-22 18:42:27 +00001056 } else {
1057 ss.append(prefixSpace).append("SpatializerPoseController not exist\n");
1058 }
1059
1060 if (needUnlock) {
1061 mLock.unlock();
1062 }
1063 return ss;
1064}
1065
Eric Laurent6d607012021-07-05 11:54:40 +02001066} // namespace android