Fix many accessibility issues.
Note: this is NOT fully a11y friendly, but much better than what we hade
before. This change supports screen readers, switches, and voice
control.
This change is full of dirty hacks and fragile assumptions about ttyd.
And below is why:
TerminalView is special in two aspects. First, all the UI elements of
terminal is rendered on a WebView, not using conventional Android
widgets. It is even different from conventional custom views because
WebView is (and has to be) responsible for the generation of
AccessibilityNodeInfos and we can only hook into the process.
Second, the web UI itself looks very different from
conventional web UIs with buttons an texts. Notably, each line of the
terminal is modeled as a <div> tag in a list. The cursor is implemented
as a tiny <textarea>. WebView and Android accessibility service are not
quite ready to handle such an exotic web UI nicely.
Below is major changes done in this change.
* AccessibilityDelegate is attached to (the parent) of TerminalView so
that we can filter out some unnecessary accessibility events such as
excessively long screen readings.
* A new AccessibilityNodeInfoProvider is put after the WebView's
original AccessibilityNodeInfoProvider. The new provider amends
AccessibilityNodeInfos created by the the original provider with the
understanding about ttyd.
* Listen on AccessibilityStateChanges not TouchExplorationStateChanges,
because the latter is enabled only for screen readers. To account for
switch controls, we need to listen to the global enablement of the
accessibility service.
Bug: 376205512
Bug: 376196669 (partially)
Bug: 376827536 (partially)
Bug: 376203872
Bug: 376824356
Bug: 376827479
Test: follow the test scenarios in the bugs
Change-Id: Ia36ef738165683b80387305f58a4f0c224d4d1f1
diff --git a/android/TerminalApp/java/com/android/virtualization/terminal/MainActivity.java b/android/TerminalApp/java/com/android/virtualization/terminal/MainActivity.java
index eb0e7e2..09c8f99 100644
--- a/android/TerminalApp/java/com/android/virtualization/terminal/MainActivity.java
+++ b/android/TerminalApp/java/com/android/virtualization/terminal/MainActivity.java
@@ -41,6 +41,7 @@
import android.view.MenuItem;
import android.view.View;
import android.view.accessibility.AccessibilityManager;
+import android.view.accessibility.AccessibilityManager.AccessibilityStateChangeListener;
import android.webkit.ClientCertRequest;
import android.webkit.SslErrorHandler;
import android.webkit.WebChromeClient;
@@ -71,10 +72,8 @@
import java.security.cert.X509Certificate;
public class MainActivity extends BaseActivity
- implements VmLauncherServices.VmLauncherServiceCallback,
- AccessibilityManager.TouchExplorationStateChangeListener {
-
- private static final String TAG = "VmTerminalApp";
+ implements VmLauncherServices.VmLauncherServiceCallback, AccessibilityStateChangeListener {
+ static final String TAG = "VmTerminalApp";
private static final String VM_ADDR = "192.168.0.2";
private static final int TTYD_PORT = 7681;
private static final int REQUEST_CODE_INSTALLER = 0x33;
@@ -118,7 +117,7 @@
mWebView.setWebChromeClient(new WebChromeClient());
mAccessibilityManager = getSystemService(AccessibilityManager.class);
- mAccessibilityManager.addTouchExplorationStateChangeListener(this);
+ mAccessibilityManager.addAccessibilityStateChangeListener(this);
readClientCertificate();
connectToTerminalService();
@@ -170,7 +169,7 @@
+ "&fontWeightBold="
+ (FontStyle.FONT_WEIGHT_BOLD + config.fontWeightAdjustment)
+ "&screenReaderMode="
- + mAccessibilityManager.isTouchExplorationEnabled()
+ + mAccessibilityManager.isEnabled()
+ "&titleFixed="
+ getString(R.string.app_name);
@@ -366,7 +365,7 @@
@Override
protected void onDestroy() {
- getSystemService(AccessibilityManager.class).removeTouchExplorationStateChangeListener(this);
+ getSystemService(AccessibilityManager.class).removeAccessibilityStateChangeListener(this);
VmLauncherServices.stopVmLauncherService(this);
super.onDestroy();
}
@@ -410,7 +409,7 @@
}
@Override
- public void onTouchExplorationStateChanged(boolean enabled) {
+ public void onAccessibilityStateChanged(boolean enabled) {
connectToTerminalService();
}
diff --git a/android/TerminalApp/java/com/android/virtualization/terminal/TerminalView.java b/android/TerminalApp/java/com/android/virtualization/terminal/TerminalView.java
index d40ae8f..aabcae7 100644
--- a/android/TerminalApp/java/com/android/virtualization/terminal/TerminalView.java
+++ b/android/TerminalApp/java/com/android/virtualization/terminal/TerminalView.java
@@ -15,12 +15,250 @@
*/
package com.android.virtualization.terminal;
+import static com.android.virtualization.terminal.MainActivity.TAG;
+
import android.content.Context;
+import android.graphics.Rect;
+import android.os.Bundle;
+import android.text.TextUtils;
import android.util.AttributeSet;
+import android.util.Log;
+import android.view.View;
+import android.view.View.AccessibilityDelegate;
+import android.view.ViewGroup;
+import android.view.accessibility.AccessibilityEvent;
+import android.view.accessibility.AccessibilityManager;
+import android.view.accessibility.AccessibilityManager.AccessibilityStateChangeListener;
+import android.view.accessibility.AccessibilityManager.TouchExplorationStateChangeListener;
+import android.view.accessibility.AccessibilityNodeInfo;
+import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;
+import android.view.accessibility.AccessibilityNodeProvider;
import android.webkit.WebView;
-public class TerminalView extends WebView {
+import java.util.List;
+
+public class TerminalView extends WebView
+ implements AccessibilityStateChangeListener, TouchExplorationStateChangeListener {
+ // Maximum length of texts the talk back announcements can be. This value is somewhat
+ // arbitrarily set. We may want to adjust this in the future.
+ private static final int TEXT_TOO_LONG_TO_ANNOUNCE = 200;
+
+ private final AccessibilityManager mA11yManager;
+
public TerminalView(Context context, AttributeSet attrs) {
super(context, attrs);
+
+ mA11yManager = context.getSystemService(AccessibilityManager.class);
+ mA11yManager.addTouchExplorationStateChangeListener(this);
+ mA11yManager.addAccessibilityStateChangeListener(this);
+ adjustToA11yStateChange();
+ }
+
+ @Override
+ public void onAccessibilityStateChanged(boolean enabled) {
+ Log.d(TAG, "accessibility " + enabled);
+ adjustToA11yStateChange();
+ }
+
+ @Override
+ public void onTouchExplorationStateChanged(boolean enabled) {
+ Log.d(TAG, "touch exploration " + enabled);
+ adjustToA11yStateChange();
+ }
+
+ private void adjustToA11yStateChange() {
+ if (!mA11yManager.isEnabled()) {
+ setFocusable(true);
+ return;
+ }
+
+ // When accessibility is on, the webview itself doesn't have to be focusable. The (virtual)
+ // edittext will be focusable to accept inputs. However, the webview has to be focusable for
+ // an accessibility purpose so that users can read the contents in it or scroll the view.
+ setFocusable(false);
+ setFocusableInTouchMode(true);
+ }
+
+ // AccessibilityEvents for WebView are sent directly from WebContentsAccessibilityImpl to the
+ // parent of WebView, without going through WebView. So, there's no WebView methods we can
+ // override to intercept the event handling process. To work around this, we attach an
+ // AccessibilityDelegate to the parent view where the events are sent to. And to guarantee that
+ // the parent view exists, wait until the WebView is attached to the window by when the parent
+ // must exist.
+ private final AccessibilityDelegate mA11yEventFilter =
+ new AccessibilityDelegate() {
+ @Override
+ public boolean onRequestSendAccessibilityEvent(
+ ViewGroup host, View child, AccessibilityEvent e) {
+ // We filter only the a11y events from the WebView
+ if (child != TerminalView.this) {
+ return super.onRequestSendAccessibilityEvent(host, child, e);
+ }
+ final int eventType = e.getEventType();
+ switch (e.getEventType()) {
+ // Skip reading texts that are too long. Right now, ttyd emits entire
+ // text on the terminal to the live region, which is very annoying to
+ // screen reader users.
+ case AccessibilityEvent.TYPE_ANNOUNCEMENT:
+ CharSequence text = e.getText().get(0); // there always is a text
+ if (text.length() >= TEXT_TOO_LONG_TO_ANNOUNCE) {
+ Log.i(TAG, "Announcement skipped because it's too long: " + text);
+ return false;
+ }
+ break;
+ }
+ return super.onRequestSendAccessibilityEvent(host, child, e);
+ }
+ };
+
+ @Override
+ protected void onAttachedToWindow() {
+ super.onAttachedToWindow();
+ if (mA11yManager.isEnabled()) {
+ View parent = (View) getParent();
+ parent.setAccessibilityDelegate(mA11yEventFilter);
+ }
+ }
+
+ private final AccessibilityNodeProvider mA11yNodeProvider =
+ new AccessibilityNodeProvider() {
+
+ /** Returns the original NodeProvider that WebView implements. */
+ private AccessibilityNodeProvider getParent() {
+ return TerminalView.super.getAccessibilityNodeProvider();
+ }
+
+ /** Convenience method for reading a string resource. */
+ private String getString(int resId) {
+ return TerminalView.this.getContext().getResources().getString(resId);
+ }
+
+ /** Checks if NodeInfo renders an empty line in the terminal. */
+ private boolean isEmptyLine(AccessibilityNodeInfo info) {
+ final CharSequence text = info.getText();
+ // Node with no text is not consiered a line. ttyd emits at least one character,
+ // which usually is NBSP.
+ if (text == null) {
+ return false;
+ }
+ for (int i = 0; i < text.length(); i++) {
+ char c = text.charAt(i);
+ // Note: don't use Characters.isWhitespace as it doesn't recognize NBSP as a
+ // whitespace.
+ if (!TextUtils.isWhitespace(c)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public AccessibilityNodeInfo createAccessibilityNodeInfo(int id) {
+ AccessibilityNodeInfo info = getParent().createAccessibilityNodeInfo(id);
+ if (info == null) {
+ return null;
+ }
+
+ final String className = info.getClassName().toString();
+
+ // By default all views except the cursor is not click-able. Other views are
+ // read-only. This ensures that user is not navigated to non-clickable elements
+ // when using switches.
+ if (!"android.widget.EditText".equals(className)) {
+ info.removeAction(AccessibilityAction.ACTION_CLICK);
+ }
+
+ switch (className) {
+ case "android.webkit.WebView":
+ // There are two NodeInfo objects of class name WebView. The one is the
+ // real WebView whose ID is View.NO_ID as it's at the root of the
+ // virtual view hierarchy. The second one is a virtual view for the
+ // iframe. The latter one's text is set to the command that we give to
+ // ttyd, which is "login -f droid ...". This is an impl detail which
+ // doesn't have to be announced. Replace the text with "Terminal
+ // display".
+ if (id != View.NO_ID) {
+ info.setText(null);
+ info.setContentDescription(getString(R.string.terminal_display));
+ }
+
+ // These two lines below are to prevent this WebView element from being
+ // fousable by the screen reader, while allowing any other element in
+ // the WebView to be focusable by the reader. In our case, the EditText
+ // is a117_focusable.
+ info.setScreenReaderFocusable(false);
+ info.addAction(AccessibilityAction.ACTION_ACCESSIBILITY_FOCUS);
+ break;
+ case "android.view.View":
+ // Empty line was announced as "space" (via the NBSP character).
+ // Localize the spoken text.
+ if (isEmptyLine(info)) {
+ info.setContentDescription(getString(R.string.empty_line));
+ }
+ break;
+ case "android.widget.TextView":
+ // There are several TextViews in the terminal, and one of them is an
+ // invisible TextView which seems to be from the <div
+ // class="live-region"> tag. Interestingly, its text is often populated
+ // with the entire text on the screen. Silence this by forcibly setting
+ // the text to null. Note that this TextView is identified by having a
+ // zero width. This certainly is not elegant, but I couldn't find other
+ // options.
+ Rect rect = new Rect();
+ info.getBoundsInScreen(rect);
+ if (rect.width() == 0) {
+ info.setText(null);
+ }
+ info.setScreenReaderFocusable(false);
+ break;
+ case "android.widget.EditText":
+ // This EditText is for the <textarea> accepting user input; the cursor.
+ // ttyd name it as "Terminal input" but it's not i18n'ed. Override it
+ // here for better i18n.
+ info.setText(null);
+ info.setHintText(null);
+ info.setContentDescription(getString(R.string.terminal_input));
+ info.setScreenReaderFocusable(true);
+ info.addAction(AccessibilityAction.ACTION_FOCUS);
+ break;
+ }
+ return info;
+ }
+
+ @Override
+ public boolean performAction(int id, int action, Bundle arguments) {
+ return getParent().performAction(id, action, arguments);
+ }
+
+ @Override
+ public void addExtraDataToAccessibilityNodeInfo(
+ int virtualViewId,
+ AccessibilityNodeInfo info,
+ String extraDataKey,
+ Bundle arguments) {
+ getParent()
+ .addExtraDataToAccessibilityNodeInfo(
+ virtualViewId, info, extraDataKey, arguments);
+ }
+
+ @Override
+ public List<AccessibilityNodeInfo> findAccessibilityNodeInfosByText(
+ String text, int virtualViewId) {
+ return getParent().findAccessibilityNodeInfosByText(text, virtualViewId);
+ }
+
+ @Override
+ public AccessibilityNodeInfo findFocus(int focus) {
+ return getParent().findFocus(focus);
+ }
+ };
+
+ @Override
+ public AccessibilityNodeProvider getAccessibilityNodeProvider() {
+ AccessibilityNodeProvider p = super.getAccessibilityNodeProvider();
+ if (p != null && mA11yManager.isEnabled()) {
+ return mA11yNodeProvider;
+ }
+ return p;
}
}
diff --git a/android/TerminalApp/res/values/strings.xml b/android/TerminalApp/res/values/strings.xml
index d498286..68551a0 100644
--- a/android/TerminalApp/res/values/strings.xml
+++ b/android/TerminalApp/res/values/strings.xml
@@ -20,6 +20,13 @@
<!-- Application name of this terminal app shown in the launcher. This app provides computer terminal to connect to virtual machine. [CHAR LIMIT=16] -->
<string name="app_name">Terminal</string>
+ <!-- Description of the entire terminal display showing texts. This is read by talkback. -->
+ <string name="terminal_display">Terminal display</string>
+ <!-- Description of the edit box accepting user input. This is read by talkback. -->
+ <string name="terminal_input">Cursor</string>
+ <!-- Description of an empty line in the terminal. This is read by talkback. -->
+ <string name="empty_line">Empty line</string>
+
<!-- Installer activity title [CHAR LIMIT=none] -->
<string name="installer_title_text">Install Linux terminal</string>
<!-- Installer activity description format [CHAR LIMIT=none] -->