Redirect all speech recognition traffic through system server.
Test: atest CtsVoiceRecognitionTestCases
Bug: 176578753
Change-Id: I5783257b76fa21c2a6f1d2e589fb843b93753350
diff --git a/core/api/current.txt b/core/api/current.txt
index 9ef3eb4..1e17f2b 100644
--- a/core/api/current.txt
+++ b/core/api/current.txt
@@ -38726,7 +38726,9 @@
field public static final int ERROR_NO_MATCH = 7; // 0x7
field public static final int ERROR_RECOGNIZER_BUSY = 8; // 0x8
field public static final int ERROR_SERVER = 4; // 0x4
+ field public static final int ERROR_SERVER_DISCONNECTED = 11; // 0xb
field public static final int ERROR_SPEECH_TIMEOUT = 6; // 0x6
+ field public static final int ERROR_TOO_MANY_REQUESTS = 10; // 0xa
field public static final String RESULTS_RECOGNITION = "results_recognition";
}
diff --git a/core/java/android/speech/IRecognitionService.aidl b/core/java/android/speech/IRecognitionService.aidl
index f91e122..cc1cded 100644
--- a/core/java/android/speech/IRecognitionService.aidl
+++ b/core/java/android/speech/IRecognitionService.aidl
@@ -43,7 +43,7 @@
* @param featureId The feature in the package
*/
void startListening(in Intent recognizerIntent, in IRecognitionListener listener,
- String packageName, String featureId);
+ String packageName, String featureId, int callingUid);
/**
* Stops listening for speech. Speech captured so far will be recognized as
@@ -62,6 +62,7 @@
* @param listener to receive callbacks, note that this must be non-null
* @param packageName the package name calling this API
* @param featureId The feature in the package
+ * @param isShutdown Whether the cancellation is caused by a client calling #shutdown
*/
- void cancel(in IRecognitionListener listener, String packageName, String featureId);
+ void cancel(in IRecognitionListener listener, String packageName, String featureId, boolean isShutdown);
}
diff --git a/core/java/android/speech/IRecognitionServiceManager.aidl b/core/java/android/speech/IRecognitionServiceManager.aidl
index 7158ba2..8e5292d 100644
--- a/core/java/android/speech/IRecognitionServiceManager.aidl
+++ b/core/java/android/speech/IRecognitionServiceManager.aidl
@@ -16,6 +16,8 @@
package android.speech;
+import android.content.ComponentName;
+
import android.speech.IRecognitionServiceManagerCallback;
/**
@@ -23,6 +25,10 @@
*
* {@hide}
*/
-interface IRecognitionServiceManager {
- void createSession(in IRecognitionServiceManagerCallback callback);
+oneway interface IRecognitionServiceManager {
+ void createSession(
+ in ComponentName componentName,
+ in IBinder clientToken,
+ boolean onDevice,
+ in IRecognitionServiceManagerCallback callback);
}
diff --git a/core/java/android/speech/IRecognitionServiceManagerCallback.aidl b/core/java/android/speech/IRecognitionServiceManagerCallback.aidl
index d760810..26afdaa 100644
--- a/core/java/android/speech/IRecognitionServiceManagerCallback.aidl
+++ b/core/java/android/speech/IRecognitionServiceManagerCallback.aidl
@@ -25,5 +25,5 @@
*/
oneway interface IRecognitionServiceManagerCallback {
void onSuccess(in IRecognitionService service);
- void onError();
+ void onError(int errorCode);
}
diff --git a/core/java/android/speech/RecognitionService.java b/core/java/android/speech/RecognitionService.java
index c97dbfe..fd584f1 100644
--- a/core/java/android/speech/RecognitionService.java
+++ b/core/java/android/speech/RecognitionService.java
@@ -105,17 +105,6 @@
int callingUid) {
if (mCurrentCallback == null) {
if (DBG) Log.d(TAG, "created new mCurrentCallback, listener = " + listener.asBinder());
- try {
- listener.asBinder().linkToDeath(new IBinder.DeathRecipient() {
- @Override
- public void binderDied() {
- mHandler.sendMessage(mHandler.obtainMessage(MSG_CANCEL, listener));
- }
- }, 0);
- } catch (RemoteException re) {
- Log.e(TAG, "dead listener on startListening");
- return;
- }
mCurrentCallback = new Callback(listener, callingUid);
RecognitionService.this.onStartListening(intent, mCurrentCallback);
} else {
@@ -352,7 +341,6 @@
* Return the Linux uid assigned to the process that sent you the current transaction that
* is being processed. This is obtained from {@link Binder#getCallingUid()}.
*/
- // TODO(b/176578753): need to make sure this is fixed when proxied through system.
public int getCallingUid() {
return mCallingUid;
}
@@ -368,7 +356,7 @@
@Override
public void startListening(Intent recognizerIntent, IRecognitionListener listener,
- String packageName, String featureId) {
+ String packageName, String featureId, int callingUid) {
Preconditions.checkNotNull(packageName);
if (DBG) Log.d(TAG, "startListening called by:" + listener.asBinder());
@@ -377,7 +365,7 @@
packageName, featureId)) {
service.mHandler.sendMessage(Message.obtain(service.mHandler,
MSG_START_LISTENING, service.new StartListeningArgs(
- recognizerIntent, listener, Binder.getCallingUid())));
+ recognizerIntent, listener, callingUid)));
}
}
@@ -397,7 +385,7 @@
@Override
public void cancel(IRecognitionListener listener, String packageName,
- String featureId) {
+ String featureId, boolean isShutdown) {
Preconditions.checkNotNull(packageName);
if (DBG) Log.d(TAG, "cancel called by:" + listener.asBinder());
diff --git a/core/java/android/speech/SpeechRecognizer.java b/core/java/android/speech/SpeechRecognizer.java
index de879c6..850f997 100644
--- a/core/java/android/speech/SpeechRecognizer.java
+++ b/core/java/android/speech/SpeechRecognizer.java
@@ -20,8 +20,8 @@
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
-import android.content.ServiceConnection;
import android.content.pm.ResolveInfo;
+import android.os.Binder;
import android.os.Bundle;
import android.os.Handler;
import android.os.IBinder;
@@ -32,10 +32,9 @@
import android.provider.Settings;
import android.text.TextUtils;
import android.util.Log;
+import android.util.Slog;
-import java.util.LinkedList;
import java.util.List;
-import java.util.Queue;
/**
* This class provides access to the speech recognition service. This service allows access to the
@@ -107,6 +106,12 @@
/** Insufficient permissions */
public static final int ERROR_INSUFFICIENT_PERMISSIONS = 9;
+ /** Too many requests from the same client. */
+ public static final int ERROR_TOO_MANY_REQUESTS = 10;
+
+ /** Server has been disconnected, e.g. because the app has crashed. */
+ public static final int ERROR_SERVER_DISCONNECTED = 11;
+
/** action codes */
private final static int MSG_START = 1;
private final static int MSG_STOP = 2;
@@ -116,9 +121,6 @@
/** The actual RecognitionService endpoint */
private IRecognitionService mService;
- /** The connection to the actual service */
- private Connection mConnection;
-
/** Context with which the manager was created */
private final Context mContext;
@@ -151,15 +153,11 @@
}
};
- /**
- * Temporary queue, saving the messages until the connection will be established, afterwards,
- * only mHandler will receive the messages
- */
- private final Queue<Message> mPendingTasks = new LinkedList<Message>();
-
/** The Listener that will receive all the callbacks */
private final InternalListener mListener = new InternalListener();
+ private final IBinder mClientToken = new Binder();
+
/**
* The right way to create a {@code SpeechRecognizer} is by using
* {@link #createSpeechRecognizer} static factory method
@@ -181,30 +179,6 @@
}
/**
- * Basic ServiceConnection that records the mService variable. Additionally, on creation it
- * invokes the {@link IRecognitionService#startListening(Intent, IRecognitionListener)}.
- */
- private class Connection implements ServiceConnection {
-
- public void onServiceConnected(final ComponentName name, final IBinder service) {
- // always done on the application main thread, so no need to send message to mHandler
- mService = IRecognitionService.Stub.asInterface(service);
- if (DBG) Log.d(TAG, "onServiceConnected - Success");
- while (!mPendingTasks.isEmpty()) {
- mHandler.sendMessage(mPendingTasks.poll());
- }
- }
-
- public void onServiceDisconnected(final ComponentName name) {
- // always done on the application main thread, so no need to send message to mHandler
- mService = null;
- mConnection = null;
- mPendingTasks.clear();
- if (DBG) Log.d(TAG, "onServiceDisconnected - Success");
- }
- }
-
- /**
* Checks whether a speech recognition service is available on the system. If this method
* returns {@code false}, {@link SpeechRecognizer#createSpeechRecognizer(Context)} will
* fail.
@@ -303,87 +277,52 @@
throw new IllegalArgumentException("intent must not be null");
}
checkIsCalledFromMainThread();
- if (mConnection == null) { // first time connection
- // TODO(b/176578753): both flows should go through system service.
- if (mOnDevice) {
- connectToSystemService();
- } else {
- connectToService();
+
+ if (DBG) {
+ Slog.i(TAG, "#startListening called");
+ if (mService == null) {
+ Slog.i(TAG, "Connection is not established yet");
}
}
- putMessage(Message.obtain(mHandler, MSG_START, recognizerIntent));
- }
- private void connectToSystemService() {
- mManagerService = IRecognitionServiceManager.Stub.asInterface(
- ServiceManager.getService(Context.SPEECH_RECOGNITION_SERVICE));
-
- if (mManagerService == null) {
- mListener.onError(ERROR_CLIENT);
- return;
- }
-
- try {
- // TODO(b/176578753): this has to supply information on whether to use on-device impl.
- mManagerService.createSession(new IRecognitionServiceManagerCallback.Stub(){
- @Override
- public void onSuccess(IRecognitionService service) throws RemoteException {
- mService = service;
- }
-
- @Override
- public void onError() throws RemoteException {
- Log.e(TAG, "Bind to system recognition service failed");
- mListener.onError(ERROR_CLIENT);
- }
- });
- } catch (RemoteException e) {
- e.rethrowFromSystemServer();
- }
- }
-
- private void connectToService() {
- mConnection = new Connection();
-
- Intent serviceIntent = new Intent(RecognitionService.SERVICE_INTERFACE);
-
- if (mServiceComponent == null) {
- String serviceComponent = Settings.Secure.getString(mContext.getContentResolver(),
- Settings.Secure.VOICE_RECOGNITION_SERVICE);
-
- if (TextUtils.isEmpty(serviceComponent)) {
- Log.e(TAG, "no selected voice recognition service");
- mListener.onError(ERROR_CLIENT);
- return;
- }
-
- serviceIntent.setComponent(
- ComponentName.unflattenFromString(serviceComponent));
+ if (mService == null) {
+ // First time connection: first establish a connection, then dispatch #startListening.
+ connectToSystemService(
+ () -> putMessage(Message.obtain(mHandler, MSG_START, recognizerIntent)));
} else {
- serviceIntent.setComponent(mServiceComponent);
- }
- if (!mContext.bindService(serviceIntent, mConnection,
- Context.BIND_AUTO_CREATE | Context.BIND_INCLUDE_CAPABILITIES)) {
- Log.e(TAG, "bind to recognition service failed");
- mConnection = null;
- mService = null;
- mListener.onError(ERROR_CLIENT);
- return;
+ putMessage(Message.obtain(mHandler, MSG_START, recognizerIntent));
}
}
/**
* Stops listening for speech. Speech captured so far will be recognized as if the user had
- * stopped speaking at this point. Note that in the default case, this does not need to be
- * called, as the speech endpointer will automatically stop the recognizer listening when it
- * determines speech has completed. However, you can manipulate endpointer parameters directly
- * using the intent extras defined in {@link RecognizerIntent}, in which case you may sometimes
- * want to manually call this method to stop listening sooner. Please note that
+ * stopped speaking at this point.
+ *
+ * <p>Note that in the default case, this does not need to be called, as the speech endpointer
+ * will automatically stop the recognizer listening when it determines speech has completed.
+ * However, you can manipulate endpointer parameters directly using the intent extras defined in
+ * {@link RecognizerIntent}, in which case you may sometimes want to manually call this method
+ * to stop listening sooner.
+ *
+ * <p>Upon invocation clients must wait until {@link RecognitionListener#onResults} or
+ * {@link RecognitionListener#onError} are invoked before calling
+ * {@link SpeechRecognizer#startListening} again. Otherwise such an attempt would be rejected by
+ * recognition service.
+ *
+ * <p>Please note that
* {@link #setRecognitionListener(RecognitionListener)} should be called beforehand, otherwise
* no notifications will be received.
*/
public void stopListening() {
checkIsCalledFromMainThread();
+
+ if (DBG) {
+ Slog.i(TAG, "#stopListening called");
+ if (mService == null) {
+ Slog.i(TAG, "Connection is not established yet");
+ }
+ }
+
putMessage(Message.obtain(mHandler, MSG_STOP));
}
@@ -405,11 +344,7 @@
}
private void putMessage(Message msg) {
- if (mService == null) {
- mPendingTasks.offer(msg);
- } else {
- mHandler.sendMessage(msg);
- }
+ mHandler.sendMessage(msg);
}
/** sends the actual message to the service */
@@ -419,7 +354,7 @@
}
try {
mService.startListening(recognizerIntent, mListener, mContext.getOpPackageName(),
- mContext.getAttributionTag());
+ mContext.getAttributionTag(), android.os.Process.myUid());
if (DBG) Log.d(TAG, "service start listening command succeded");
} catch (final RemoteException e) {
Log.e(TAG, "startListening() failed", e);
@@ -448,7 +383,11 @@
return;
}
try {
- mService.cancel(mListener, mContext.getOpPackageName(), mContext.getAttributionTag());
+ mService.cancel(
+ mListener,
+ mContext.getOpPackageName(),
+ mContext.getAttributionTag(),
+ false /* isShutdown */);
if (DBG) Log.d(TAG, "service cancel command succeded");
} catch (final RemoteException e) {
Log.e(TAG, "cancel() failed", e);
@@ -471,28 +410,94 @@
mListener.mInternalListener = listener;
}
- /**
- * Destroys the {@code SpeechRecognizer} object.
- */
+ /** Destroys the {@code SpeechRecognizer} object. */
public void destroy() {
if (mService != null) {
try {
mService.cancel(mListener, mContext.getOpPackageName(),
- mContext.getAttributionTag());
+ mContext.getAttributionTag(), true /* isShutdown */);
} catch (final RemoteException e) {
// Not important
}
}
- if (mConnection != null) {
- mContext.unbindService(mConnection);
- }
- mPendingTasks.clear();
mService = null;
- mConnection = null;
mListener.mInternalListener = null;
}
+ /** Establishes a connection to system server proxy and initializes the session. */
+ private void connectToSystemService(Runnable onSuccess) {
+ mManagerService = IRecognitionServiceManager.Stub.asInterface(
+ ServiceManager.getService(Context.SPEECH_RECOGNITION_SERVICE));
+
+ if (mManagerService == null) {
+ mListener.onError(ERROR_CLIENT);
+ return;
+ }
+
+ ComponentName componentName = getSpeechRecognizerComponentName();
+
+ if (!mOnDevice && componentName == null) {
+ mListener.onError(ERROR_CLIENT);
+ return;
+ }
+
+ try {
+ mManagerService.createSession(
+ componentName,
+ mClientToken,
+ mOnDevice,
+ new IRecognitionServiceManagerCallback.Stub(){
+ @Override
+ public void onSuccess(IRecognitionService service) throws RemoteException {
+ mService = service;
+ onSuccess.run();
+ }
+
+ @Override
+ public void onError(int errorCode) throws RemoteException {
+ Log.e(TAG, "Bind to system recognition service failed");
+ mListener.onError(errorCode);
+ }
+ });
+ } catch (RemoteException e) {
+ e.rethrowFromSystemServer();
+ }
+ }
+
+ /**
+ * Returns the component name to be used for establishing a connection, based on the parameters
+ * used during initialization.
+ *
+ * <p>Note the 3 different scenarios:
+ * <ol>
+ * <li>On-device speech recognizer which is determined by the manufacturer and not
+ * changeable by the user
+ * <li>Default user-selected speech recognizer as specified by
+ * {@code Settings.Secure.VOICE_RECOGNITION_SERVICE}
+ * <li>Custom speech recognizer supplied by the client.
+ */
+ private ComponentName getSpeechRecognizerComponentName() {
+ if (mOnDevice) {
+ return null;
+ }
+
+ if (mServiceComponent != null) {
+ return mServiceComponent;
+ }
+
+ String serviceComponent = Settings.Secure.getString(mContext.getContentResolver(),
+ Settings.Secure.VOICE_RECOGNITION_SERVICE);
+
+ if (TextUtils.isEmpty(serviceComponent)) {
+ Log.e(TAG, "no selected voice recognition service");
+ mListener.onError(ERROR_CLIENT);
+ return null;
+ }
+
+ return ComponentName.unflattenFromString(serviceComponent);
+ }
+
/**
* Internal wrapper of IRecognitionListener which will propagate the results to
* RecognitionListener
diff --git a/services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java b/services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java
index 96248c3..0974537 100644
--- a/services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java
+++ b/services/core/java/com/android/server/speech/RemoteSpeechRecognitionService.java
@@ -18,22 +18,65 @@
import static com.android.internal.infra.AbstractRemoteService.PERMANENT_BOUND_TIMEOUT_MS;
+import android.annotation.Nullable;
+import android.app.AppOpsManager;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
+import android.os.Binder;
+import android.os.Bundle;
import android.os.RemoteException;
import android.speech.IRecognitionListener;
import android.speech.IRecognitionService;
import android.speech.RecognitionService;
+import android.speech.SpeechRecognizer;
+import android.util.Log;
import android.util.Slog;
+import com.android.internal.annotations.GuardedBy;
import com.android.internal.infra.ServiceConnector;
final class RemoteSpeechRecognitionService extends ServiceConnector.Impl<IRecognitionService> {
private static final String TAG = RemoteSpeechRecognitionService.class.getSimpleName();
- private static final boolean DEBUG = true;
+ private static final boolean DEBUG = false;
- RemoteSpeechRecognitionService(Context context, ComponentName serviceName, int userId) {
+ private static final String APP_OP_MESSAGE = "Recording audio for speech recognition";
+ private static final String RECORD_AUDIO_APP_OP =
+ AppOpsManager.permissionToOp(android.Manifest.permission.RECORD_AUDIO);
+
+ private final Object mLock = new Object();
+
+ private boolean mConnected = false;
+
+ @Nullable
+ private IRecognitionListener mListener;
+
+ @Nullable
+ @GuardedBy("mLock")
+ private String mPackageName;
+
+ @Nullable
+ @GuardedBy("mLock")
+ private String mFeatureId;
+
+ @Nullable
+ @GuardedBy("mLock")
+ private DelegatingListener mDelegatingListener;
+
+ // Makes sure we can block startListening() if session is still in progress.
+ @GuardedBy("mLock")
+ private boolean mSessionInProgress = false;
+
+ // Makes sure we call startProxyOp / finishProxyOp at right times and only once per session.
+ @GuardedBy("mLock")
+ private boolean mRecordingInProgress = false;
+
+ private final int mCallingUid;
+ private final AppOpsManager mAppOpsManager;
+ private final ComponentName mComponentName;
+
+ RemoteSpeechRecognitionService(
+ Context context, ComponentName serviceName, int userId, int callingUid) {
super(context,
new Intent(RecognitionService.SERVICE_INTERFACE).setComponent(serviceName),
Context.BIND_AUTO_CREATE
@@ -43,46 +86,197 @@
userId,
IRecognitionService.Stub::asInterface);
+ mCallingUid = callingUid;
+ mAppOpsManager = mContext.getSystemService(AppOpsManager.class);
+ mComponentName = serviceName;
+
if (DEBUG) {
Slog.i(TAG, "Bound to recognition service at: " + serviceName.flattenToString());
}
}
- void startListening(Intent recognizerIntent, IRecognitionListener listener, String packageName,
- String featureId) throws RemoteException {
- if (DEBUG) {
- Slog.i(TAG, "#startListening for package: " + packageName + ", feature=" + featureId);
- }
- run(service -> service.startListening(recognizerIntent, listener, packageName, featureId));
+ ComponentName getServiceComponentName() {
+ return mComponentName;
}
- void stopListening(IRecognitionListener listener, String packageName, String featureId)
- throws RemoteException {
+ void startListening(Intent recognizerIntent, IRecognitionListener listener, String packageName,
+ String featureId) {
+ if (DEBUG) {
+ Slog.i(TAG, String.format("#startListening for package: %s, feature=%s, callingUid=%d",
+ packageName, featureId, mCallingUid));
+ }
+
+ if (listener == null) {
+ Log.w(TAG, "#startListening called with no preceding #setListening - ignoring");
+ return;
+ }
+
+ if (!mConnected) {
+ tryRespondWithError(listener, SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
+ return;
+ }
+
+ synchronized (mLock) {
+ if (mSessionInProgress) {
+ Slog.i(TAG, "#startListening called while listening is in progress.");
+ tryRespondWithError(listener, SpeechRecognizer.ERROR_RECOGNIZER_BUSY);
+ return;
+ }
+
+ if (startProxyOp(packageName, featureId) != AppOpsManager.MODE_ALLOWED) {
+ tryRespondWithError(listener, SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS);
+ return;
+ }
+ mSessionInProgress = true;
+ mRecordingInProgress = true;
+
+ mListener = listener;
+ mDelegatingListener = new DelegatingListener(listener, () -> {
+ // To be invoked in terminal calls of the callback: results() or error()
+ if (DEBUG) {
+ Slog.i(TAG, "Recognition session complete");
+ }
+
+ synchronized (mLock) {
+ resetStateLocked();
+ }
+ });
+ mPackageName = packageName;
+ mFeatureId = featureId;
+
+ run(service ->
+ service.startListening(
+ recognizerIntent,
+ mDelegatingListener,
+ packageName,
+ featureId,
+ mCallingUid));
+ }
+ }
+
+ void stopListening(
+ IRecognitionListener listener, String packageName, String featureId) {
if (DEBUG) {
Slog.i(TAG, "#stopListening for package: " + packageName + ", feature=" + featureId);
}
- run(service -> service.stopListening(listener, packageName, featureId));
+
+ if (!mConnected) {
+ tryRespondWithError(listener, SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
+ return;
+ }
+
+ synchronized (mLock) {
+ if (mListener == null) {
+ Log.w(TAG, "#stopListening called with no preceding #startListening - ignoring");
+ tryRespondWithError(listener, SpeechRecognizer.ERROR_CLIENT);
+ return;
+ }
+
+ if (mListener.asBinder() != listener.asBinder()) {
+ Log.w(TAG, "#stopListening called with an unexpected listener");
+ tryRespondWithError(listener, SpeechRecognizer.ERROR_CLIENT);
+ return;
+ }
+
+ if (!mRecordingInProgress) {
+ Slog.i(TAG, "#stopListening called while listening isn't in progress, ignoring.");
+ return;
+ }
+ mRecordingInProgress = false;
+
+ finishProxyOp(packageName, featureId);
+
+ run(service -> service.stopListening(mDelegatingListener, packageName, featureId));
+ }
}
- void cancel(IRecognitionListener listener, String packageName, String featureId)
- throws RemoteException {
+ void cancel(
+ IRecognitionListener listener,
+ String packageName,
+ String featureId,
+ boolean isShutdown) {
if (DEBUG) {
Slog.i(TAG, "#cancel for package: " + packageName + ", feature=" + featureId);
}
- run(service -> service.cancel(listener, packageName, featureId));
+
+ if (!mConnected) {
+ tryRespondWithError(listener, SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
+ }
+
+ synchronized (mLock) {
+ if (mListener == null) {
+ if (DEBUG) {
+ Log.w(TAG, "#cancel called with no preceding #startListening - ignoring");
+ }
+ return;
+ }
+
+ if (mListener.asBinder() != listener.asBinder()) {
+ Log.w(TAG, "#cancel called with an unexpected listener");
+ tryRespondWithError(listener, SpeechRecognizer.ERROR_CLIENT);
+ return;
+ }
+
+ // Temporary reference to allow for resetting the hard link mDelegatingListener to null.
+ IRecognitionListener delegatingListener = mDelegatingListener;
+
+ run(service -> service.cancel(delegatingListener, packageName, featureId, isShutdown));
+
+ if (mRecordingInProgress) {
+ finishProxyOp(packageName, featureId);
+ }
+ mRecordingInProgress = false;
+ mSessionInProgress = false;
+
+ mDelegatingListener = null;
+ mListener = null;
+
+ // Schedule to unbind after cancel is delivered.
+ if (isShutdown) {
+ run(service -> unbind());
+ }
+ }
+ }
+
+ void shutdown() {
+ synchronized (mLock) {
+ if (this.mListener == null) {
+ if (DEBUG) {
+ Slog.i(TAG, "Package died, but session wasn't initialized. "
+ + "Not invoking #cancel");
+ }
+ return;
+ }
+ }
+
+ cancel(mListener, mPackageName, mFeatureId, true /* isShutdown */);
}
@Override // from ServiceConnector.Impl
protected void onServiceConnectionStatusChanged(
IRecognitionService service, boolean connected) {
- if (!DEBUG) {
- return;
+ mConnected = connected;
+
+ if (DEBUG) {
+ if (connected) {
+ Slog.i(TAG, "Connected to speech recognition service");
+ } else {
+ Slog.w(TAG, "Disconnected from speech recognition service");
+ }
}
- if (connected) {
- Slog.i(TAG, "Connected to ASR service");
- } else {
- Slog.w(TAG, "Disconnected from ASR service");
+ synchronized (mLock) {
+ if (!connected) {
+ if (mListener == null) {
+ Slog.i(TAG, "Connection to speech recognition service lost, but no "
+ + "#startListening has been invoked yet.");
+ return;
+ }
+
+ tryRespondWithError(mListener, SpeechRecognizer.ERROR_SERVER_DISCONNECTED);
+
+ resetStateLocked();
+ }
}
}
@@ -90,4 +284,119 @@
protected long getAutoDisconnectTimeoutMs() {
return PERMANENT_BOUND_TIMEOUT_MS;
}
+
+ private void resetStateLocked() {
+ if (mRecordingInProgress && mPackageName != null && mFeatureId != null) {
+ finishProxyOp(mPackageName, mFeatureId);
+ }
+
+ mListener = null;
+ mDelegatingListener = null;
+ mSessionInProgress = false;
+ mRecordingInProgress = false;
+ }
+
+ private int startProxyOp(String packageName, String featureId) {
+ final long identity = Binder.clearCallingIdentity();
+ try {
+ return mAppOpsManager.startProxyOp(
+ RECORD_AUDIO_APP_OP,
+ mCallingUid,
+ packageName,
+ featureId,
+ APP_OP_MESSAGE);
+ } finally {
+ Binder.restoreCallingIdentity(identity);
+ }
+ }
+
+ private void finishProxyOp(String packageName, String featureId) {
+ final long identity = Binder.clearCallingIdentity();
+ try {
+ mAppOpsManager.finishProxyOp(
+ RECORD_AUDIO_APP_OP, mCallingUid, packageName, featureId);
+ } finally {
+ Binder.restoreCallingIdentity(identity);
+ }
+ }
+
+ private static void tryRespondWithError(IRecognitionListener listener, int errorCode) {
+ if (DEBUG) {
+ Slog.i(TAG, "Responding with error " + errorCode);
+ }
+
+ try {
+ if (listener != null) {
+ listener.onError(errorCode);
+ }
+ } catch (RemoteException e) {
+ Slog.w(TAG,
+ String.format("Failed to respond with an error %d to the client", errorCode),
+ e);
+ }
+ }
+
+ private static class DelegatingListener extends IRecognitionListener.Stub {
+
+ private final IRecognitionListener mRemoteListener;
+ private final Runnable mOnSessionComplete;
+
+ DelegatingListener(IRecognitionListener listener, Runnable onSessionComplete) {
+ mRemoteListener = listener;
+ mOnSessionComplete = onSessionComplete;
+ }
+
+ @Override
+ public void onReadyForSpeech(Bundle params) throws RemoteException {
+ mRemoteListener.onReadyForSpeech(params);
+ }
+
+ @Override
+ public void onBeginningOfSpeech() throws RemoteException {
+ mRemoteListener.onBeginningOfSpeech();
+ }
+
+ @Override
+ public void onRmsChanged(float rmsdB) throws RemoteException {
+ mRemoteListener.onRmsChanged(rmsdB);
+ }
+
+ @Override
+ public void onBufferReceived(byte[] buffer) throws RemoteException {
+ mRemoteListener.onBufferReceived(buffer);
+ }
+
+ @Override
+ public void onEndOfSpeech() throws RemoteException {
+ mRemoteListener.onEndOfSpeech();
+ }
+
+ @Override
+ public void onError(int error) throws RemoteException {
+ if (DEBUG) {
+ Slog.i(TAG, String.format("Error %d during recognition session", error));
+ }
+ mOnSessionComplete.run();
+ mRemoteListener.onError(error);
+ }
+
+ @Override
+ public void onResults(Bundle results) throws RemoteException {
+ if (DEBUG) {
+ Slog.i(TAG, "#onResults invoked for a recognition session");
+ }
+ mOnSessionComplete.run();
+ mRemoteListener.onResults(results);
+ }
+
+ @Override
+ public void onPartialResults(Bundle results) throws RemoteException {
+ mRemoteListener.onPartialResults(results);
+ }
+
+ @Override
+ public void onEvent(int eventType, Bundle params) throws RemoteException {
+ mRemoteListener.onEvent(eventType, params);
+ }
+ }
}
diff --git a/services/core/java/com/android/server/speech/SpeechRecognitionManagerService.java b/services/core/java/com/android/server/speech/SpeechRecognitionManagerService.java
index 592ba9e..dbe7354 100644
--- a/services/core/java/com/android/server/speech/SpeechRecognitionManagerService.java
+++ b/services/core/java/com/android/server/speech/SpeechRecognitionManagerService.java
@@ -18,7 +18,9 @@
import android.annotation.NonNull;
import android.annotation.UserIdInt;
+import android.content.ComponentName;
import android.content.Context;
+import android.os.IBinder;
import android.os.UserHandle;
import android.speech.IRecognitionServiceManager;
import android.speech.IRecognitionServiceManagerCallback;
@@ -42,6 +44,7 @@
public SpeechRecognitionManagerService(@NonNull Context context) {
super(context,
+ // TODO(b/176578753): think if we want to favor the particular service here.
new FrameworkResourcesServiceNameResolver(
context,
R.string.config_defaultOnDeviceSpeechRecognitionService),
@@ -63,11 +66,15 @@
final class SpeechRecognitionManagerServiceStub extends IRecognitionServiceManager.Stub {
@Override
- public void createSession(IRecognitionServiceManagerCallback callback) {
+ public void createSession(
+ ComponentName componentName,
+ IBinder clientToken,
+ boolean onDevice,
+ IRecognitionServiceManagerCallback callback) {
int userId = UserHandle.getCallingUserId();
synchronized (mLock) {
SpeechRecognitionManagerServiceImpl service = getServiceForUserLocked(userId);
- service.createSessionLocked(callback);
+ service.createSessionLocked(componentName, clientToken, onDevice, callback);
}
}
}
diff --git a/services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java b/services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java
index bcaf174..2656a3d 100644
--- a/services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java
+++ b/services/core/java/com/android/server/speech/SpeechRecognitionManagerServiceImpl.java
@@ -24,30 +24,44 @@
import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.pm.ServiceInfo;
+import android.os.Binder;
+import android.os.IBinder;
import android.os.RemoteException;
import android.speech.IRecognitionListener;
import android.speech.IRecognitionService;
import android.speech.IRecognitionServiceManagerCallback;
+import android.speech.SpeechRecognizer;
import android.util.Slog;
import com.android.internal.annotations.GuardedBy;
import com.android.server.infra.AbstractPerUserSystemService;
+import com.google.android.collect.Sets;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
final class SpeechRecognitionManagerServiceImpl extends
AbstractPerUserSystemService<SpeechRecognitionManagerServiceImpl,
SpeechRecognitionManagerService> {
-
private static final String TAG = SpeechRecognitionManagerServiceImpl.class.getSimpleName();
+ private static final int MAX_CONCURRENT_CONNECTIONS_BY_CLIENT = 10;
+
+ private final Object mLock = new Object();
+
+ @NonNull
@GuardedBy("mLock")
- @Nullable
- private RemoteSpeechRecognitionService mRemoteService;
+ private final Map<Integer, Set<RemoteSpeechRecognitionService>> mRemoteServicesByUid =
+ new HashMap<>();
SpeechRecognitionManagerServiceImpl(
@NonNull SpeechRecognitionManagerService master,
@NonNull Object lock, @UserIdInt int userId, boolean disabled) {
super(master, lock, userId);
- updateRemoteServiceLocked();
}
@GuardedBy("mLock")
@@ -67,92 +81,196 @@
@Override // from PerUserSystemService
protected boolean updateLocked(boolean disabled) {
final boolean enabledChanged = super.updateLocked(disabled);
- updateRemoteServiceLocked();
return enabledChanged;
}
- /**
- * Updates the reference to the remote service.
- */
- @GuardedBy("mLock")
- private void updateRemoteServiceLocked() {
- if (mRemoteService != null) {
- if (mMaster.debug) {
- Slog.d(TAG, "updateRemoteService(): destroying old remote service");
- }
- mRemoteService.unbind();
- mRemoteService = null;
+ void createSessionLocked(
+ ComponentName componentName,
+ IBinder clientToken,
+ boolean onDevice,
+ IRecognitionServiceManagerCallback callback) {
+ if (mMaster.debug) {
+ Slog.i(TAG, String.format("#createSessionLocked, component=%s, onDevice=%s",
+ componentName, onDevice));
}
- }
- void createSessionLocked(IRecognitionServiceManagerCallback callback) {
- // TODO(b/176578753): check clients have record audio permission.
- // TODO(b/176578753): verify caller package is the one supplied
+ ComponentName serviceComponent = componentName;
+ if (onDevice) {
+ serviceComponent = getOnDeviceComponentNameLocked();
+ }
- RemoteSpeechRecognitionService service = ensureRemoteServiceLocked();
+ if (serviceComponent == null) {
+ tryRespondWithError(callback, SpeechRecognizer.ERROR_CLIENT);
+ return;
+ }
+
+ final int creatorCallingUid = Binder.getCallingUid();
+ Set<String> creatorPackageNames =
+ Sets.newArraySet(
+ getContext().getPackageManager().getPackagesForUid(creatorCallingUid));
+
+ RemoteSpeechRecognitionService service = createService(creatorCallingUid, serviceComponent);
if (service == null) {
- tryRespondWithError(callback);
+ tryRespondWithError(callback, SpeechRecognizer.ERROR_TOO_MANY_REQUESTS);
return;
}
+ IBinder.DeathRecipient deathRecipient =
+ () -> handleClientDeath(creatorCallingUid, service, true /* invoke #cancel */);
+
+ try {
+ clientToken.linkToDeath(deathRecipient, 0);
+ } catch (RemoteException e) {
+ // RemoteException == binder already died, schedule disconnect anyway.
+ handleClientDeath(creatorCallingUid, service, true /* invoke #cancel */);
+ }
+
service.connect().thenAccept(binderService -> {
if (binderService != null) {
try {
callback.onSuccess(new IRecognitionService.Stub() {
@Override
- public void startListening(Intent recognizerIntent,
+ public void startListening(
+ Intent recognizerIntent,
IRecognitionListener listener,
- String packageName, String featureId) throws RemoteException {
+ String packageName,
+ String featureId,
+ int callingUid) throws RemoteException {
+ verifyCallerIdentity(
+ creatorCallingUid, packageName, creatorPackageNames, listener);
+ if (callingUid != creatorCallingUid) {
+ listener.onError(SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS);
+ return;
+ }
+
service.startListening(
recognizerIntent, listener, packageName, featureId);
}
@Override
- public void stopListening(IRecognitionListener listener,
+ public void stopListening(
+ IRecognitionListener listener,
String packageName,
String featureId) throws RemoteException {
+ verifyCallerIdentity(
+ creatorCallingUid, packageName, creatorPackageNames, listener);
+
service.stopListening(listener, packageName, featureId);
}
@Override
- public void cancel(IRecognitionListener listener,
+ public void cancel(
+ IRecognitionListener listener,
String packageName,
- String featureId) throws RemoteException {
- service.cancel(listener, packageName, featureId);
+ String featureId,
+ boolean isShutdown) throws RemoteException {
+ verifyCallerIdentity(
+ creatorCallingUid, packageName, creatorPackageNames, listener);
+
+ service.cancel(listener, packageName, featureId, isShutdown);
+
+ if (isShutdown) {
+ handleClientDeath(
+ creatorCallingUid,
+ service,
+ false /* invoke #cancel */);
+ clientToken.unlinkToDeath(deathRecipient, 0);
+ }
}
});
} catch (RemoteException e) {
Slog.e(TAG, "Error creating a speech recognition session", e);
- tryRespondWithError(callback);
+ tryRespondWithError(callback, SpeechRecognizer.ERROR_CLIENT);
}
} else {
- tryRespondWithError(callback);
+ tryRespondWithError(callback, SpeechRecognizer.ERROR_CLIENT);
}
});
}
- @GuardedBy("mLock")
- @Nullable
- private RemoteSpeechRecognitionService ensureRemoteServiceLocked() {
- if (mRemoteService == null) {
- final String serviceName = getComponentNameLocked();
- if (serviceName == null) {
- if (mMaster.verbose) {
- Slog.v(TAG, "ensureRemoteServiceLocked(): no service component name.");
- }
- return null;
- }
- final ComponentName serviceComponent = ComponentName.unflattenFromString(serviceName);
- mRemoteService =
- new RemoteSpeechRecognitionService(getContext(), serviceComponent, mUserId);
+ private void verifyCallerIdentity(
+ int creatorCallingUid,
+ String packageName,
+ Set<String> creatorPackageNames,
+ IRecognitionListener listener) throws RemoteException {
+ if (creatorCallingUid != Binder.getCallingUid()
+ || !creatorPackageNames.contains(packageName)) {
+ listener.onError(SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS);
}
- return mRemoteService;
}
- private static void tryRespondWithError(IRecognitionServiceManagerCallback callback) {
+ private void handleClientDeath(
+ int callingUid,
+ RemoteSpeechRecognitionService service, boolean invokeCancel) {
+ if (invokeCancel) {
+ service.shutdown();
+ }
+ removeService(callingUid, service);
+ }
+
+ @GuardedBy("mLock")
+ @Nullable
+ private ComponentName getOnDeviceComponentNameLocked() {
+ final String serviceName = getComponentNameLocked();
+ if (serviceName == null) {
+ if (mMaster.verbose) {
+ Slog.v(TAG, "ensureRemoteServiceLocked(): no service component name.");
+ }
+ return null;
+ }
+ return ComponentName.unflattenFromString(serviceName);
+ }
+
+ private RemoteSpeechRecognitionService createService(
+ int callingUid, ComponentName serviceComponent) {
+ synchronized (mLock) {
+ Set<RemoteSpeechRecognitionService> servicesForClient =
+ mRemoteServicesByUid.get(callingUid);
+
+ if (servicesForClient != null
+ && servicesForClient.size() >= MAX_CONCURRENT_CONNECTIONS_BY_CLIENT) {
+ return null;
+ }
+
+ if (servicesForClient != null) {
+ Optional<RemoteSpeechRecognitionService> existingService =
+ servicesForClient
+ .stream()
+ .filter(service ->
+ service.getServiceComponentName().equals(serviceComponent))
+ .findFirst();
+ if (existingService.isPresent()) {
+ return existingService.get();
+ }
+ }
+
+ RemoteSpeechRecognitionService service =
+ new RemoteSpeechRecognitionService(
+ getContext(), serviceComponent, getUserId(), callingUid);
+
+ Set<RemoteSpeechRecognitionService> valuesByCaller =
+ mRemoteServicesByUid.computeIfAbsent(callingUid, key -> new HashSet<>());
+ valuesByCaller.add(service);
+
+ return service;
+ }
+ }
+
+ private void removeService(int callingUid, RemoteSpeechRecognitionService service) {
+ synchronized (mLock) {
+ Set<RemoteSpeechRecognitionService> valuesByCaller =
+ mRemoteServicesByUid.get(callingUid);
+ if (valuesByCaller != null) {
+ valuesByCaller.remove(service);
+ }
+ }
+ }
+
+ private static void tryRespondWithError(IRecognitionServiceManagerCallback callback,
+ int errorCode) {
try {
- callback.onError();
+ callback.onError(errorCode);
} catch (RemoteException e) {
Slog.w(TAG, "Failed to respond with error");
}