AudioFlinger: compute MEL values on framework level
The MelReporter is responsible for starting the MEL calculation for
different audio streams. For now we only print the values in dumpsys.
Test: dumpsys media.audio_flinger
Bug: 252776298
Change-Id: Ic5757bac23844358cb4c886b3eaf2fd2e9ffbf40
diff --git a/services/audioflinger/Android.bp b/services/audioflinger/Android.bp
index 2549f7c..27ac212 100644
--- a/services/audioflinger/Android.bp
+++ b/services/audioflinger/Android.bp
@@ -43,6 +43,7 @@
"FastThread.cpp",
"FastThreadDumpState.cpp",
"FastThreadState.cpp",
+ "MelReporter.cpp",
"NBAIO_Tee.cpp",
"PatchCommandThread.cpp",
"PatchPanel.cpp",
@@ -100,6 +101,7 @@
"libaaudio_headers",
"libaudioclient_headers",
"libaudiohal_headers",
+ "libaudioutils_headers",
"libmedia_headers",
],
diff --git a/services/audioflinger/AudioFlinger.cpp b/services/audioflinger/AudioFlinger.cpp
index f2dd600..6cb9913 100644
--- a/services/audioflinger/AudioFlinger.cpp
+++ b/services/audioflinger/AudioFlinger.cpp
@@ -325,6 +325,7 @@
mPatchPanel(this),
mPatchCommandThread(sp<PatchCommandThread>::make()),
mDeviceEffectManager(sp<DeviceEffectManager>::make(*this)),
+ mMelReporter(sp<MelReporter>::make(*this)),
mSystemReady(false)
{
// Move the audio session unique ID generator start base as time passes to limit risk of
@@ -876,6 +877,9 @@
mDeviceEffectManager->dump(fd);
+ std::string melOutput = mMelReporter->dump();
+ write(fd, melOutput.c_str(), melOutput.size());
+
// dump external setParameters
auto dumpLogger = [fd](SimpleLog& logger, const char* name) {
dprintf(fd, "\n%s setParameters:\n", name);
diff --git a/services/audioflinger/AudioFlinger.h b/services/audioflinger/AudioFlinger.h
index 00ca355..6a94164 100644
--- a/services/audioflinger/AudioFlinger.h
+++ b/services/audioflinger/AudioFlinger.h
@@ -82,6 +82,8 @@
#include <audio_utils/clock.h>
#include <audio_utils/FdToString.h>
#include <audio_utils/LinearMap.h>
+#include <audio_utils/MelAggregator.h>
+#include <audio_utils/MelProcessor.h>
#include <audio_utils/SimpleLog.h>
#include <audio_utils/TimestampVerifier.h>
@@ -632,12 +634,14 @@
#include "PatchPanel.h"
-#include "Effects.h"
-
#include "PatchCommandThread.h"
+#include "Effects.h"
+
#include "DeviceEffectManager.h"
+#include "MelReporter.h"
+
// Find io handle by session id.
// Preference is given to an io handle with a matching effect chain to session id.
// If none found, AUDIO_IO_HANDLE_NONE is returned.
@@ -1016,6 +1020,7 @@
const sp<PatchCommandThread> mPatchCommandThread;
sp<DeviceEffectManager> mDeviceEffectManager;
+ sp<MelReporter> mMelReporter;
bool mSystemReady;
std::atomic_bool mAudioPolicyReady{};
diff --git a/services/audioflinger/MelReporter.cpp b/services/audioflinger/MelReporter.cpp
new file mode 100644
index 0000000..9a579dd
--- /dev/null
+++ b/services/audioflinger/MelReporter.cpp
@@ -0,0 +1,127 @@
+/*
+**
+** Copyright 2022, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+
+// #define LOG_NDEBUG 0
+#define LOG_TAG "AudioFlinger::MelReporter"
+
+#include <cinttypes>
+#include <utils/Log.h>
+#include <android-base/stringprintf.h>
+#include <audio_utils/power.h>
+
+#include "AudioFlinger.h"
+
+namespace android {
+
+bool AudioFlinger::MelReporter::shouldComputeMelForDeviceType(audio_devices_t device) {
+ switch (device) {
+ case AUDIO_DEVICE_OUT_WIRED_HEADSET:
+ case AUDIO_DEVICE_OUT_WIRED_HEADPHONE:
+ case AUDIO_DEVICE_OUT_BLUETOOTH_A2DP:
+ case AUDIO_DEVICE_OUT_BLUETOOTH_A2DP_HEADPHONES:
+ case AUDIO_DEVICE_OUT_HEARING_AID:
+ case AUDIO_DEVICE_OUT_USB_HEADSET:
+ case AUDIO_DEVICE_OUT_BLE_HEADSET:
+ case AUDIO_DEVICE_OUT_BLE_BROADCAST:
+ return true;
+ default:
+ return false;
+ }
+}
+
+void AudioFlinger::MelReporter::onCreateAudioPatch(audio_patch_handle_t handle,
+ const PatchPanel::Patch& patch) {
+ ALOGV("%s: handle %d mHalHandle %d device sink %08x",
+ __func__, handle, patch.mHalHandle,
+ patch.mAudioPatch.num_sinks > 0 ? patch.mAudioPatch.sinks[0].ext.device.type : 0);
+ if (patch.mAudioPatch.num_sources == 0
+ || patch.mAudioPatch.sources[0].type != AUDIO_PORT_TYPE_MIX) {
+ ALOGW("%s: patch does not contain any mix sources", __func__);
+ return;
+ }
+
+ audio_io_handle_t streamHandle = patch.mAudioPatch.sources[0].ext.mix.handle;
+ ActiveMelPatch newPatch;
+ newPatch.streamHandle = streamHandle;
+ for (int i = 0; i < patch.mAudioPatch.num_sinks; ++ i) {
+ if (patch.mAudioPatch.sinks[i].type == AUDIO_PORT_TYPE_DEVICE
+ && shouldComputeMelForDeviceType(patch.mAudioPatch.sinks[i].ext.device.type)) {
+ audio_port_handle_t deviceId = patch.mAudioPatch.sinks[i].id;
+ newPatch.deviceHandles.push_back(deviceId);
+
+ // Start the MEL calculation in the PlaybackThread
+ std::lock_guard _lAf(mAudioFlinger.mLock);
+ auto thread = mAudioFlinger.checkPlaybackThread_l(streamHandle);
+ if (thread != nullptr) {
+ thread->startMelComputation(mMelAggregator.getOrCreateCallbackForDevice(
+ deviceId,
+ newPatch.streamHandle));
+ }
+ }
+ }
+
+ std::lock_guard _l(mLock);
+ mActiveMelPatches[patch.mAudioPatch.id] = newPatch;
+}
+
+void AudioFlinger::MelReporter::onReleaseAudioPatch(audio_patch_handle_t handle) {
+ ALOGV("%s", __func__);
+
+ ActiveMelPatch melPatch;
+ {
+ std::lock_guard _l(mLock);
+
+ auto patchIt = mActiveMelPatches.find(handle);
+ if (patchIt == mActiveMelPatches.end()) {
+ ALOGW(
+ "%s patch does not contain any mix sources with active MEL calculation",
+ __func__);
+ return;
+ }
+
+ melPatch = patchIt->second;
+ mActiveMelPatches.erase(patchIt);
+ }
+
+ // Stop MEL calculation for the PlaybackThread
+ std::lock_guard _lAf(mAudioFlinger.mLock);
+ auto thread = mAudioFlinger.checkPlaybackThread_l(melPatch.streamHandle);
+ if (thread != nullptr) {
+ thread->stopMelComputation();
+ }
+ mMelAggregator.removeStreamCallback(melPatch.streamHandle);
+}
+
+std::string AudioFlinger::MelReporter::dump() {
+ std::lock_guard _l(mLock);
+ std::string output;
+
+ base::StringAppendF(&output, "\nMel Reporter:\n");
+ mMelAggregator.foreach([&output](const audio_utils::MelRecord& melRecord) {
+ base::StringAppendF(&output, "Continuous MELs for portId=%d, ", melRecord.portId);
+ base::StringAppendF(&output, "starting at timestamp %" PRId64 ": ", melRecord.timestamp);
+
+ for (const auto& mel : melRecord.mels) {
+ base::StringAppendF(&output, "%d ", mel);
+ }
+ base::StringAppendF(&output, "\n");
+ });
+
+ return output;
+}
+
+} // namespace android
diff --git a/services/audioflinger/MelReporter.h b/services/audioflinger/MelReporter.h
new file mode 100644
index 0000000..905a4cd
--- /dev/null
+++ b/services/audioflinger/MelReporter.h
@@ -0,0 +1,71 @@
+/*
+**
+** Copyright 2022, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+
+#ifndef INCLUDING_FROM_AUDIOFLINGER_H
+ #error This header file should only be included from AudioFlinger.h
+#endif
+
+#include <unordered_map>
+#include <mutex>
+
+constexpr static int kMaxTimestampDeltaInSec = 120;
+
+/**
+ * Class for listening to new patches and starting the MEL computation. MelReporter is
+ * concealed within AudioFlinger, their lifetimes are the same.
+ */
+class MelReporter : public PatchCommandThread::PatchCommandListener {
+public:
+ explicit MelReporter(AudioFlinger& audioFlinger)
+ : mAudioFlinger(audioFlinger),
+ mMelAggregator(kMaxTimestampDeltaInSec) {}
+
+ void onFirstRef() override {
+ mAudioFlinger.mPatchCommandThread->addListener(this);
+ }
+
+ /** Returns true if we should compute MEL for the given device. */
+ static bool shouldComputeMelForDeviceType(audio_devices_t device);
+
+ // For now only support internal MelReporting
+ [[nodiscard]] bool isHalReportingEnabled() const { return false; }
+
+ std::string dump();
+
+ // PatchCommandListener methods
+ void onCreateAudioPatch(audio_patch_handle_t handle,
+ const PatchPanel::Patch& patch) override;
+ void onReleaseAudioPatch(audio_patch_handle_t handle) override;
+
+private:
+ AudioFlinger& mAudioFlinger; // does not own the object
+
+ audio_utils::MelAggregator mMelAggregator;
+
+ struct ActiveMelPatch {
+ audio_io_handle_t streamHandle{AUDIO_IO_HANDLE_NONE};
+ std::vector<audio_port_handle_t> deviceHandles;
+ };
+
+ /**
+ * Lock for protecting the active mel patches. Do not mix with the AudioFlinger lock.
+ * Locking order AudioFlinger::mLock -> PatchCommandThread::mLock -> MelReporter::mLock.
+ */
+ std::mutex mLock;
+ std::unordered_map<audio_patch_handle_t, ActiveMelPatch>
+ mActiveMelPatches GUARDED_BY(AudioFlinger::MelReporter::mLock);
+};
diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp
index 60ba0f3..3766709 100644
--- a/services/audioflinger/Threads.cpp
+++ b/services/audioflinger/Threads.cpp
@@ -17,7 +17,7 @@
#define LOG_TAG "AudioFlinger"
-//#define LOG_NDEBUG 0
+// #define LOG_NDEBUG 0
#define ATRACE_TAG ATRACE_TAG_AUDIO
#include "Configuration.h"
@@ -44,6 +44,7 @@
#include <private/media/AudioTrackShared.h>
#include <private/android_filesystem_config.h>
#include <audio_utils/Balance.h>
+#include <audio_utils/MelProcessor.h>
#include <audio_utils/Metadata.h>
#include <audio_utils/channels.h>
#include <audio_utils/mono_blend.h>
@@ -3333,6 +3334,7 @@
}
ssize_t framesWritten = mNormalSink->write((char *)mSinkBuffer + offset, count);
ATRACE_END();
+
if (framesWritten > 0) {
bytesWritten = framesWritten * mFrameSize;
#ifdef TEE_SINK
@@ -3341,6 +3343,11 @@
} else {
bytesWritten = framesWritten;
}
+
+ auto processor = mMelProcessor.load();
+ if (processor) {
+ processor->process((char *)mSinkBuffer + offset, bytesWritten);
+ }
// otherwise use the HAL / AudioStreamOut directly
} else {
// Direct output and offload threads
@@ -3377,6 +3384,21 @@
return bytesWritten;
}
+void AudioFlinger::PlaybackThread::startMelComputation(const sp<
+ audio_utils::MelProcessor::MelCallback>& callback)
+{
+ ALOGV("%s: creating new mel processor for thread %d", __func__, id());
+ mMelProcessor = sp<audio_utils::MelProcessor>::make(mSampleRate,
+ mChannelCount,
+ mFormat,
+ callback);
+}
+
+void AudioFlinger::PlaybackThread::stopMelComputation() {
+ ALOGV("%s: stopping mel processor for thread %d", __func__, id());
+ mMelProcessor = nullptr;
+}
+
void AudioFlinger::PlaybackThread::threadLoop_drain()
{
bool supportsDrain = false;
diff --git a/services/audioflinger/Threads.h b/services/audioflinger/Threads.h
index c509d73..a7028ee 100644
--- a/services/audioflinger/Threads.h
+++ b/services/audioflinger/Threads.h
@@ -1091,6 +1091,10 @@
return INVALID_OPERATION;
}
+ void startMelComputation(const sp
+ <audio_utils::MelProcessor::MelCallback>& callback);
+ void stopMelComputation();
+
protected:
// updated by readOutputParameters_l()
size_t mNormalFrameCount; // normal mixer and effects
@@ -1190,6 +1194,8 @@
audio_channel_mask_t mMixerChannelMask = AUDIO_CHANNEL_NONE;
private:
+ mediautils::atomic_sp<audio_utils::MelProcessor> mMelProcessor;
+
// mMasterMute is in both PlaybackThread and in AudioFlinger. When a
// PlaybackThread needs to find out if master-muted, it checks it's local
// copy rather than the one in AudioFlinger. This optimization saves a lock.