Add system service for music recognition.
The client submits a RecognitionRequest via MusicRecognitionManager. System server opens an audio stream based on the request and sends it to a MusicRecognitionService (which is exposed by a system app). The result is passed back through system server to the original caller.
Test: tracked in b/169662646
Bug: 169403302
Change-Id: I4c7fd9d9d72ddd5678867fd037cab6198bff2c2d
diff --git a/services/Android.bp b/services/Android.bp
index ef52c2a..d3577ef 100644
--- a/services/Android.bp
+++ b/services/Android.bp
@@ -28,6 +28,7 @@
":services.coverage-sources",
":services.devicepolicy-sources",
":services.midi-sources",
+ ":services.musicsearch-sources",
":services.net-sources",
":services.print-sources",
":services.profcollect-sources",
@@ -71,6 +72,7 @@
"services.coverage",
"services.devicepolicy",
"services.midi",
+ "services.musicsearch",
"services.net",
"services.people",
"services.print",
diff --git a/services/java/com/android/server/SystemServer.java b/services/java/com/android/server/SystemServer.java
index 97ae505..0b35ba4 100644
--- a/services/java/com/android/server/SystemServer.java
+++ b/services/java/com/android/server/SystemServer.java
@@ -278,6 +278,8 @@
"com.android.server.autofill.AutofillManagerService";
private static final String CONTENT_CAPTURE_MANAGER_SERVICE_CLASS =
"com.android.server.contentcapture.ContentCaptureManagerService";
+ private static final String MUSIC_RECOGNITION_MANAGER_SERVICE_CLASS =
+ "com.android.server.musicrecognition.MusicRecognitionManagerService";
private static final String SYSTEM_CAPTIONS_MANAGER_SERVICE_CLASS =
"com.android.server.systemcaptions.SystemCaptionsManagerService";
private static final String TIME_ZONE_RULES_MANAGER_SERVICE_CLASS =
@@ -1402,6 +1404,17 @@
t.traceEnd();
}
+ if (deviceHasConfigString(context,
+ R.string.config_defaultMusicRecognitionService)) {
+ t.traceBegin("StartMusicRecognitionManagerService");
+ mSystemServiceManager.startService(MUSIC_RECOGNITION_MANAGER_SERVICE_CLASS);
+ t.traceEnd();
+ } else {
+ Slog.d(TAG,
+ "MusicRecognitionManagerService not defined by OEM or disabled by flag");
+ }
+
+
startContentCaptureService(context, t);
startAttentionService(context, t);
diff --git a/services/musicrecognition/Android.bp b/services/musicrecognition/Android.bp
new file mode 100644
index 0000000..39b5bb6
--- /dev/null
+++ b/services/musicrecognition/Android.bp
@@ -0,0 +1,13 @@
+filegroup {
+ name: "services.musicsearch-sources",
+ srcs: ["java/**/*.java"],
+ path: "java",
+ visibility: ["//frameworks/base/services"],
+}
+
+java_library_static {
+ name: "services.musicsearch",
+ defaults: ["services_defaults"],
+ srcs: [":services.musicsearch-sources"],
+ libs: ["services.core", "app-compat-annotations"],
+}
\ No newline at end of file
diff --git a/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerPerUserService.java b/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerPerUserService.java
new file mode 100644
index 0000000..e258ef0
--- /dev/null
+++ b/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerPerUserService.java
@@ -0,0 +1,300 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.musicrecognition;
+
+import static android.media.musicrecognition.MusicRecognitionManager.RECOGNITION_FAILED_SERVICE_KILLED;
+import static android.media.musicrecognition.MusicRecognitionManager.RECOGNITION_FAILED_SERVICE_UNAVAILABLE;
+import static android.media.musicrecognition.MusicRecognitionManager.RecognitionFailureCode;
+
+import android.Manifest;
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.app.AppGlobals;
+import android.content.ComponentName;
+import android.content.pm.PackageManager;
+import android.content.pm.ServiceInfo;
+import android.media.AudioRecord;
+import android.media.MediaMetadata;
+import android.media.musicrecognition.IMusicRecognitionManagerCallback;
+import android.media.musicrecognition.IMusicRecognitionServiceCallback;
+import android.media.musicrecognition.RecognitionRequest;
+import android.os.Bundle;
+import android.os.IBinder;
+import android.os.ParcelFileDescriptor;
+import android.os.RemoteException;
+import android.util.Pair;
+import android.util.Slog;
+
+import com.android.internal.annotations.GuardedBy;
+import com.android.server.infra.AbstractPerUserSystemService;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Handles per-user requests received by
+ * {@link MusicRecognitionManagerService}. Opens an audio stream from the
+ * dsp and writes it into a pipe to {@link RemoteMusicRecognitionService}.
+ */
+public final class MusicRecognitionManagerPerUserService extends
+ AbstractPerUserSystemService<MusicRecognitionManagerPerUserService,
+ MusicRecognitionManagerService>
+ implements RemoteMusicRecognitionService.Callbacks {
+
+ private static final String TAG = MusicRecognitionManagerPerUserService.class.getSimpleName();
+ // Number of bytes per sample of audio (which is a short).
+ private static final int BYTES_PER_SAMPLE = 2;
+ private static final int MAX_STREAMING_SECONDS = 24;
+
+ @Nullable
+ @GuardedBy("mLock")
+ private RemoteMusicRecognitionService mRemoteService;
+
+ private MusicRecognitionServiceCallback mRemoteServiceCallback =
+ new MusicRecognitionServiceCallback();
+ private IMusicRecognitionManagerCallback mCallback;
+
+ MusicRecognitionManagerPerUserService(
+ @NonNull MusicRecognitionManagerService primary,
+ @NonNull Object lock, int userId) {
+ super(primary, lock, userId);
+ }
+
+ @NonNull
+ @GuardedBy("mLock")
+ @Override
+ protected ServiceInfo newServiceInfoLocked(@NonNull ComponentName serviceComponent)
+ throws PackageManager.NameNotFoundException {
+ ServiceInfo si;
+ try {
+ si = AppGlobals.getPackageManager().getServiceInfo(serviceComponent,
+ PackageManager.GET_META_DATA, mUserId);
+ } catch (RemoteException e) {
+ throw new PackageManager.NameNotFoundException(
+ "Could not get service for " + serviceComponent);
+ }
+ if (!Manifest.permission.BIND_MUSIC_RECOGNITION_SERVICE.equals(si.permission)) {
+ Slog.w(TAG, "MusicRecognitionService from '" + si.packageName
+ + "' does not require permission "
+ + Manifest.permission.BIND_MUSIC_RECOGNITION_SERVICE);
+ throw new SecurityException("Service does not require permission "
+ + Manifest.permission.BIND_MUSIC_RECOGNITION_SERVICE);
+ }
+ // TODO(b/158194857): check process which owns the service has RECORD_AUDIO permission. How?
+ return si;
+ }
+
+ @GuardedBy("mLock")
+ @Nullable
+ private RemoteMusicRecognitionService ensureRemoteServiceLocked() {
+ if (mRemoteService == null) {
+ final String serviceName = getComponentNameLocked();
+ if (serviceName == null) {
+ if (mMaster.verbose) {
+ Slog.v(TAG, "ensureRemoteServiceLocked(): not set");
+ }
+ return null;
+ }
+ ComponentName serviceComponent = ComponentName.unflattenFromString(serviceName);
+
+ mRemoteService = new RemoteMusicRecognitionService(getContext(),
+ serviceComponent, mUserId, this,
+ mRemoteServiceCallback, mMaster.isBindInstantServiceAllowed(), mMaster.verbose);
+ }
+
+ return mRemoteService;
+ }
+
+ /**
+ * Read audio from the given capture session using an AudioRecord and writes it to a
+ * ParcelFileDescriptor.
+ */
+ @GuardedBy("mLock")
+ public void beginRecognitionLocked(
+ @NonNull RecognitionRequest recognitionRequest,
+ @NonNull IBinder callback) {
+ int maxAudioLengthSeconds = Math.min(recognitionRequest.getMaxAudioLengthSeconds(),
+ MAX_STREAMING_SECONDS);
+ mCallback = IMusicRecognitionManagerCallback.Stub.asInterface(callback);
+ AudioRecord audioRecord = createAudioRecord(recognitionRequest, maxAudioLengthSeconds);
+
+ mRemoteService = ensureRemoteServiceLocked();
+ if (mRemoteService == null) {
+ try {
+ mCallback.onRecognitionFailed(
+ RECOGNITION_FAILED_SERVICE_UNAVAILABLE);
+ } catch (RemoteException e) {
+ // Ignored.
+ }
+ return;
+ }
+
+ Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe();
+ if (clientPipe == null) {
+ try {
+ mCallback.onAudioStreamClosed();
+ } catch (RemoteException ignored) {
+ // Ignored.
+ }
+ return;
+ }
+ ParcelFileDescriptor audioSink = clientPipe.second;
+ ParcelFileDescriptor clientRead = clientPipe.first;
+
+ mMaster.mExecutorService.execute(() -> {
+ try (OutputStream fos =
+ new ParcelFileDescriptor.AutoCloseOutputStream(audioSink)) {
+ int halfSecondBufferSize =
+ audioRecord.getBufferSizeInFrames() / maxAudioLengthSeconds;
+ byte[] byteBuffer = new byte[halfSecondBufferSize];
+ int bytesRead = 0;
+ int totalBytesRead = 0;
+ int ignoreBytes =
+ recognitionRequest.getIgnoreBeginningFrames() * BYTES_PER_SAMPLE;
+ audioRecord.startRecording();
+ while (bytesRead >= 0 && totalBytesRead
+ < audioRecord.getBufferSizeInFrames() * BYTES_PER_SAMPLE) {
+ bytesRead = audioRecord.read(byteBuffer, 0, byteBuffer.length);
+ if (bytesRead > 0) {
+ totalBytesRead += bytesRead;
+ // If we are ignoring the first x bytes, update that counter.
+ if (ignoreBytes > 0) {
+ ignoreBytes -= bytesRead;
+ // If we've dipped negative, we've skipped through all ignored bytes
+ // and then some. Write out the bytes we shouldn't have skipped.
+ if (ignoreBytes < 0) {
+ fos.write(byteBuffer, bytesRead + ignoreBytes, -ignoreBytes);
+ }
+ } else {
+ fos.write(byteBuffer);
+ }
+ }
+ }
+ Slog.i(TAG, String.format("Streamed %s bytes from audio record", totalBytesRead));
+ } catch (IOException e) {
+ Slog.e(TAG, "Audio streaming stopped.", e);
+ } finally {
+ audioRecord.release();
+ try {
+ mCallback.onAudioStreamClosed();
+ } catch (RemoteException ignored) {
+ // Ignored.
+ }
+
+ }
+ });
+ // Send the pipe down to the lookup service while we write to it asynchronously.
+ mRemoteService.writeAudioToPipe(clientRead, recognitionRequest.getAudioFormat());
+ }
+
+ /**
+ * Callback invoked by {@link android.service.musicrecognition.MusicRecognitionService} to pass
+ * back the music search result.
+ */
+ private final class MusicRecognitionServiceCallback extends
+ IMusicRecognitionServiceCallback.Stub {
+ @Override
+ public void onRecognitionSucceeded(MediaMetadata result, Bundle extras) {
+ try {
+ sanitizeBundle(extras);
+ mCallback.onRecognitionSucceeded(result, extras);
+ } catch (RemoteException ignored) {
+ // Ignored.
+ }
+ }
+
+ @Override
+ public void onRecognitionFailed(@RecognitionFailureCode int failureCode) {
+ try {
+ mCallback.onRecognitionFailed(failureCode);
+ } catch (RemoteException ignored) {
+ // Ignored.
+ }
+ }
+ }
+
+ @Override
+ public void onServiceDied(@NonNull RemoteMusicRecognitionService service) {
+ try {
+ mCallback.onRecognitionFailed(RECOGNITION_FAILED_SERVICE_KILLED);
+ } catch (RemoteException e) {
+ // Ignored.
+ }
+ Slog.w(TAG, "remote service died: " + service);
+ }
+
+ /** Establishes an audio stream from the DSP audio source. */
+ private static AudioRecord createAudioRecord(
+ @NonNull RecognitionRequest recognitionRequest,
+ int maxAudioLengthSeconds) {
+ int sampleRate = recognitionRequest.getAudioFormat().getSampleRate();
+ int bufferSize = getBufferSizeInBytes(sampleRate, maxAudioLengthSeconds);
+ return new AudioRecord(recognitionRequest.getAudioAttributes(),
+ recognitionRequest.getAudioFormat(), bufferSize,
+ recognitionRequest.getCaptureSession());
+ }
+
+ /**
+ * Returns the number of bytes required to store {@code bufferLengthSeconds} of audio sampled at
+ * {@code sampleRate} Hz, using the format returned by DSP audio capture.
+ */
+ private static int getBufferSizeInBytes(int sampleRate, int bufferLengthSeconds) {
+ return BYTES_PER_SAMPLE * sampleRate * bufferLengthSeconds;
+ }
+
+ private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() {
+ ParcelFileDescriptor[] fileDescriptors;
+ try {
+ fileDescriptors = ParcelFileDescriptor.createPipe();
+ } catch (IOException e) {
+ Slog.e(TAG, "Failed to create audio stream pipe", e);
+ return null;
+ }
+
+ if (fileDescriptors.length != 2) {
+ Slog.e(TAG, "Failed to create audio stream pipe, "
+ + "unexpected number of file descriptors");
+ return null;
+ }
+
+ if (!fileDescriptors[0].getFileDescriptor().valid()
+ || !fileDescriptors[1].getFileDescriptor().valid()) {
+ Slog.e(TAG, "Failed to create audio stream pipe, didn't "
+ + "receive a pair of valid file descriptors.");
+ return null;
+ }
+
+ return Pair.create(fileDescriptors[0], fileDescriptors[1]);
+ }
+
+ /** Removes remote objects from the bundle. */
+ private static void sanitizeBundle(@Nullable Bundle bundle) {
+ if (bundle == null) {
+ return;
+ }
+
+ for (String key : bundle.keySet()) {
+ Object o = bundle.get(key);
+
+ if (o instanceof Bundle) {
+ sanitizeBundle((Bundle) o);
+ } else if (o instanceof IBinder || o instanceof ParcelFileDescriptor) {
+ bundle.remove(key);
+ }
+ }
+ }
+}
diff --git a/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerService.java b/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerService.java
new file mode 100644
index 0000000..b4cb337
--- /dev/null
+++ b/services/musicrecognition/java/com/android/server/musicrecognition/MusicRecognitionManagerService.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.musicrecognition;
+
+import static android.content.PermissionChecker.PERMISSION_GRANTED;
+import static android.media.musicrecognition.MusicRecognitionManager.RECOGNITION_FAILED_SERVICE_UNAVAILABLE;
+
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.content.Context;
+import android.media.musicrecognition.IMusicRecognitionManager;
+import android.media.musicrecognition.IMusicRecognitionManagerCallback;
+import android.media.musicrecognition.RecognitionRequest;
+import android.os.Binder;
+import android.os.IBinder;
+import android.os.RemoteException;
+import android.os.UserHandle;
+
+import com.android.server.infra.AbstractMasterSystemService;
+import com.android.server.infra.FrameworkResourcesServiceNameResolver;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Service which allows a DSP audio event to be securely streamed to a designated {@link
+ * MusicRecognitionService}.
+ */
+public class MusicRecognitionManagerService extends
+ AbstractMasterSystemService<MusicRecognitionManagerService,
+ MusicRecognitionManagerPerUserService> {
+
+ private static final String TAG = MusicRecognitionManagerService.class.getSimpleName();
+
+ private MusicRecognitionManagerStub mMusicRecognitionManagerStub;
+ final ExecutorService mExecutorService = Executors.newCachedThreadPool();
+
+ /**
+ * Initializes the system service.
+ *
+ * Subclasses must define a single argument constructor that accepts the context
+ * and passes it to super.
+ *
+ * @param context The system server context.
+ */
+ public MusicRecognitionManagerService(@NonNull Context context) {
+ super(context, new FrameworkResourcesServiceNameResolver(context,
+ com.android.internal.R.string.config_defaultMusicRecognitionService),
+ /** disallowProperty */null);
+ }
+
+ @Nullable
+ @Override
+ protected MusicRecognitionManagerPerUserService newServiceLocked(int resolvedUserId,
+ boolean disabled) {
+ return new MusicRecognitionManagerPerUserService(this, mLock, resolvedUserId);
+ }
+
+ @Override
+ public void onStart() {
+ mMusicRecognitionManagerStub = new MusicRecognitionManagerStub();
+ publishBinderService(Context.MUSIC_RECOGNITION_SERVICE, mMusicRecognitionManagerStub);
+ }
+
+ private void enforceCaller(String func) {
+ Context ctx = getContext();
+ if (ctx.checkCallingPermission(android.Manifest.permission.MANAGE_MUSIC_RECOGNITION)
+ == PERMISSION_GRANTED) {
+ return;
+ }
+
+ String msg = "Permission Denial: " + func + " from pid="
+ + Binder.getCallingPid()
+ + ", uid=" + Binder.getCallingUid()
+ + " doesn't hold " + android.Manifest.permission.MANAGE_MUSIC_RECOGNITION;
+ throw new SecurityException(msg);
+ }
+
+ final class MusicRecognitionManagerStub extends IMusicRecognitionManager.Stub {
+ @Override
+ public void beginRecognition(
+ @NonNull RecognitionRequest recognitionRequest,
+ @NonNull IBinder callback) {
+ enforceCaller("beginRecognition");
+
+ synchronized (mLock) {
+ final MusicRecognitionManagerPerUserService service = getServiceForUserLocked(
+ UserHandle.getCallingUserId());
+ if (service != null) {
+ service.beginRecognitionLocked(recognitionRequest, callback);
+ } else {
+ try {
+ IMusicRecognitionManagerCallback.Stub.asInterface(callback)
+ .onRecognitionFailed(RECOGNITION_FAILED_SERVICE_UNAVAILABLE);
+ } catch (RemoteException e) {
+ // ignored.
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/services/musicrecognition/java/com/android/server/musicrecognition/RemoteMusicRecognitionService.java b/services/musicrecognition/java/com/android/server/musicrecognition/RemoteMusicRecognitionService.java
new file mode 100644
index 0000000..4814a82
--- /dev/null
+++ b/services/musicrecognition/java/com/android/server/musicrecognition/RemoteMusicRecognitionService.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.musicrecognition;
+
+import android.annotation.NonNull;
+import android.content.ComponentName;
+import android.content.Context;
+import android.media.AudioFormat;
+import android.media.musicrecognition.IMusicRecognitionService;
+import android.media.musicrecognition.IMusicRecognitionServiceCallback;
+import android.media.musicrecognition.MusicRecognitionService;
+import android.os.IBinder;
+import android.os.ParcelFileDescriptor;
+import android.text.format.DateUtils;
+
+import com.android.internal.infra.AbstractMultiplePendingRequestsRemoteService;
+
+/** Remote connection to an instance of {@link MusicRecognitionService}. */
+public class RemoteMusicRecognitionService extends
+ AbstractMultiplePendingRequestsRemoteService<RemoteMusicRecognitionService,
+ IMusicRecognitionService> {
+
+ // Maximum time allotted for the remote service to return a result. Up to 24s of audio plus
+ // time to fingerprint and make rpcs.
+ private static final long TIMEOUT_IDLE_BIND_MILLIS = 40 * DateUtils.SECOND_IN_MILLIS;
+
+ // Allows the remote service to send back a result.
+ private final IMusicRecognitionServiceCallback mServerCallback;
+
+ public RemoteMusicRecognitionService(Context context, ComponentName serviceName,
+ int userId, MusicRecognitionManagerPerUserService perUserService,
+ IMusicRecognitionServiceCallback callback,
+ boolean bindInstantServiceAllowed, boolean verbose) {
+ super(context, MusicRecognitionService.ACTION_MUSIC_SEARCH_LOOKUP, serviceName, userId,
+ perUserService,
+ context.getMainThreadHandler(),
+ // Prevents the service from having its permissions stripped while in background.
+ Context.BIND_INCLUDE_CAPABILITIES | (bindInstantServiceAllowed
+ ? Context.BIND_ALLOW_INSTANT : 0), verbose,
+ /* initialCapacity= */ 1);
+ mServerCallback = callback;
+ }
+
+ @NonNull
+ @Override
+ protected IMusicRecognitionService getServiceInterface(@NonNull IBinder service) {
+ return IMusicRecognitionService.Stub.asInterface(service);
+ }
+
+ @Override
+ protected long getTimeoutIdleBindMillis() {
+ return TIMEOUT_IDLE_BIND_MILLIS;
+ }
+
+ /**
+ * Required, but empty since we don't need to notify the callback implementation of the request
+ * results.
+ */
+ interface Callbacks extends VultureCallback<RemoteMusicRecognitionService> {}
+
+ /**
+ * Sends the given descriptor to the app's {@link MusicRecognitionService} to read the
+ * audio.
+ */
+ public void writeAudioToPipe(@NonNull ParcelFileDescriptor fd,
+ @NonNull AudioFormat audioFormat) {
+ scheduleAsyncRequest(
+ binder -> binder.onAudioStreamStarted(fd, audioFormat, mServerCallback));
+ }
+}