Make mirror window accessible (2/2)
To support accessibility, we added a11y actions to move
the mirror window and change its scale. We also provide
content description, state description to deliver the
information of the mirror window.
Finally, we update state desceription to send window content change
when changing the scale.
Test: manually test with voice access
atest WindowMagnificationControllerTest
Bug: 143852371
Change-Id: Ia27c5bee39e732609ac1d8e2c7566524e6d7c64c
diff --git a/packages/SystemUI/res/values/ids.xml b/packages/SystemUI/res/values/ids.xml
index d815681..f2bb490 100644
--- a/packages/SystemUI/res/values/ids.xml
+++ b/packages/SystemUI/res/values/ids.xml
@@ -175,5 +175,13 @@
<!-- Accessibility actions for PIP -->
<item type="id" name="action_pip_resize" />
+
+ <!-- Accessibility actions for window magnification. -->
+ <item type="id" name="accessibility_action_zoom_in"/>
+ <item type="id" name="accessibility_action_zoom_out"/>
+ <item type="id" name="accessibility_action_move_left"/>
+ <item type="id" name="accessibility_action_move_right"/>
+ <item type="id" name="accessibility_action_move_up"/>
+ <item type="id" name="accessibility_action_move_down"/>
</resources>
diff --git a/packages/SystemUI/res/values/strings.xml b/packages/SystemUI/res/values/strings.xml
index f693149..4ba757f 100644
--- a/packages/SystemUI/res/values/strings.xml
+++ b/packages/SystemUI/res/values/strings.xml
@@ -2666,6 +2666,18 @@
<string name="magnification_window_title">Magnification Window</string>
<!-- Title for Magnification Controls Window [CHAR LIMIT=NONE] -->
<string name="magnification_controls_title">Magnification Window Controls</string>
+ <!-- Action in accessibility menu to zoom in content of the magnification window. [CHAR LIMIT=30] -->
+ <string name="accessibility_control_zoom_in">Zoom in</string>
+ <!-- Action in accessibility menu to zoom out content of the magnification window. [CHAR LIMIT=30] -->
+ <string name="accessibility_control_zoom_out">Zoom out</string>
+ <!-- Action in accessibility menu to move the magnification window up. [CHAR LIMIT=30] -->
+ <string name="accessibility_control_move_up">Move up</string>
+ <!-- Action in accessibility menu to move the magnification window down. [CHAR LIMIT=30] -->
+ <string name="accessibility_control_move_down">Move down</string>
+ <!-- Action in accessibility menu to move the magnification window left. [CHAR LIMIT=30] -->
+ <string name="accessibility_control_move_left">Move left</string>
+ <!-- Action in accessibility menu to move the magnification window right. [CHAR LIMIT=30] -->
+ <string name="accessibility_control_move_right">Move right</string>
<!-- Device Controls strings -->
<!-- Device Controls empty state, title [CHAR LIMIT=30] -->
diff --git a/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationController.java b/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationController.java
index 7140956..3832ff30 100644
--- a/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationController.java
+++ b/packages/SystemUI/src/com/android/systemui/accessibility/WindowMagnificationController.java
@@ -30,9 +30,11 @@
import android.graphics.Rect;
import android.graphics.RectF;
import android.graphics.Region;
+import android.os.Bundle;
import android.os.Handler;
import android.os.RemoteException;
import android.util.Log;
+import android.util.Range;
import android.view.Choreographer;
import android.view.Display;
import android.view.Gravity;
@@ -47,12 +49,17 @@
import android.view.View;
import android.view.WindowManager;
import android.view.WindowManagerGlobal;
+import android.view.accessibility.AccessibilityNodeInfo;
+import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.graphics.SfVsyncFrameCallbackProvider;
import com.android.systemui.R;
import com.android.systemui.shared.system.WindowManagerWrapper;
+import java.text.NumberFormat;
+import java.util.Locale;
+
/**
* Class to handle adding and removing a window magnification.
*/
@@ -60,6 +67,11 @@
MirrorWindowControl.MirrorWindowDelegate {
private static final String TAG = "WindowMagnificationController";
+ // Delay to avoid updating state description too frequently.
+ private static final int UPDATE_STATE_DESCRIPTION_DELAY_MS = 100;
+ // It should be consistent with the value defined in WindowMagnificationGestureHandler.
+ private static final Range<Float> A11Y_ACTION_SCALE_RANGE = new Range<>(2.0f, 8.0f);
+ private static final float A11Y_CHANGE_SCALE_DIFFERENCE = 1.0f;
private final Context mContext;
private final Resources mResources;
private final Handler mHandler;
@@ -95,6 +107,7 @@
private final View.OnLayoutChangeListener mMirrorViewLayoutChangeListener;
private final View.OnLayoutChangeListener mMirrorSurfaceViewLayoutChangeListener;
private final Runnable mMirrorViewRunnable;
+ private final Runnable mUpdateStateDescriptionRunnable;
private View mMirrorView;
private SurfaceView mMirrorSurfaceView;
private int mMirrorSurfaceMargin;
@@ -106,6 +119,8 @@
private final SfVsyncFrameCallbackProvider mSfVsyncFrameProvider;
private Choreographer.FrameCallback mMirrorViewGeometryVsyncCallback;
+ private Locale mLocale;
+ private NumberFormat mPercentFormat;
@Nullable
private MirrorWindowControl mMirrorWindowControl;
@@ -164,6 +179,11 @@
mWindowMagnifierCallback.onSourceBoundsChanged(mDisplayId, mSourceBounds);
}
};
+ mUpdateStateDescriptionRunnable = () -> {
+ if (isWindowVisible()) {
+ mMirrorView.setStateDescription(formatStateDescription(mScale));
+ }
+ };
}
private void updateDimensions() {
@@ -292,12 +312,13 @@
| View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY
| View.SYSTEM_UI_FLAG_HIDE_NAVIGATION);
mMirrorView.addOnLayoutChangeListener(mMirrorViewLayoutChangeListener);
+ mMirrorView.setAccessibilityDelegate(new MirrorWindowA11yDelegate());
+
mWm.addView(mMirrorView, params);
SurfaceHolder holder = mMirrorSurfaceView.getHolder();
holder.addCallback(this);
holder.setFormat(PixelFormat.RGBA_8888);
-
addDragTouchListeners();
}
@@ -526,6 +547,7 @@
final float offsetY = Float.isNaN(centerY) ? 0
: centerY - mMagnificationFrame.exactCenterY();
mScale = Float.isNaN(scale) ? mScale : scale;
+
setMagnificationFrameBoundary();
updateMagnificationFramePosition((int) offsetX, (int) offsetY);
if (!isWindowVisible()) {
@@ -546,6 +568,8 @@
return;
}
enableWindowMagnification(scale, Float.NaN, Float.NaN);
+ mHandler.removeCallbacks(mUpdateStateDescriptionRunnable);
+ mHandler.postDelayed(mUpdateStateDescriptionRunnable, UPDATE_STATE_DESCRIPTION_DELAY_MS);
}
/**
@@ -596,4 +620,78 @@
private boolean isWindowVisible() {
return mMirrorView != null;
}
+
+ private CharSequence formatStateDescription(float scale) {
+ // Cache the locale-appropriate NumberFormat. Configuration locale is guaranteed
+ // non-null, so the first time this is called we will always get the appropriate
+ // NumberFormat, then never regenerate it unless the locale changes on the fly.
+ final Locale curLocale = mContext.getResources().getConfiguration().getLocales().get(0);
+ if (!curLocale.equals(mLocale)) {
+ mLocale = curLocale;
+ mPercentFormat = NumberFormat.getPercentInstance(curLocale);
+ }
+ return mPercentFormat.format(scale);
+ }
+
+ private class MirrorWindowA11yDelegate extends View.AccessibilityDelegate {
+
+ @Override
+ public void onInitializeAccessibilityNodeInfo(View host, AccessibilityNodeInfo info) {
+ super.onInitializeAccessibilityNodeInfo(host, info);
+ info.addAction(
+ new AccessibilityAction(R.id.accessibility_action_zoom_in,
+ mContext.getString(R.string.accessibility_control_zoom_in)));
+ info.addAction(new AccessibilityAction(R.id.accessibility_action_zoom_out,
+ mContext.getString(R.string.accessibility_control_zoom_out)));
+ info.addAction(new AccessibilityAction(R.id.accessibility_action_move_up,
+ mContext.getString(R.string.accessibility_control_move_up)));
+ info.addAction(new AccessibilityAction(R.id.accessibility_action_move_down,
+ mContext.getString(R.string.accessibility_control_move_down)));
+ info.addAction(new AccessibilityAction(R.id.accessibility_action_move_left,
+ mContext.getString(R.string.accessibility_control_move_left)));
+ info.addAction(new AccessibilityAction(R.id.accessibility_action_move_right,
+ mContext.getString(R.string.accessibility_control_move_right)));
+
+ info.setContentDescription(mContext.getString(R.string.magnification_window_title));
+ info.setStateDescription(formatStateDescription(getScale()));
+ }
+
+ @Override
+ public boolean performAccessibilityAction(View host, int action, Bundle args) {
+ if (performA11yAction(action)) {
+ return true;
+ }
+ return super.performAccessibilityAction(host, action, args);
+ }
+
+ private boolean performA11yAction(int action) {
+ if (action == R.id.accessibility_action_zoom_in) {
+ final float scale = mScale + A11Y_CHANGE_SCALE_DIFFERENCE;
+ setScale(A11Y_ACTION_SCALE_RANGE.clamp(scale));
+ return true;
+ }
+ if (action == R.id.accessibility_action_zoom_out) {
+ final float scale = mScale - A11Y_CHANGE_SCALE_DIFFERENCE;
+ setScale(A11Y_ACTION_SCALE_RANGE.clamp(scale));
+ return true;
+ }
+ if (action == R.id.accessibility_action_move_up) {
+ move(0, -mSourceBounds.height());
+ return true;
+ }
+ if (action == R.id.accessibility_action_move_down) {
+ move(0, mSourceBounds.height());
+ return true;
+ }
+ if (action == R.id.accessibility_action_move_left) {
+ move(-mSourceBounds.width(), 0);
+ return true;
+ }
+ if (action == R.id.accessibility_action_move_right) {
+ move(mSourceBounds.width(), 0);
+ return true;
+ }
+ return false;
+ }
+ }
}
diff --git a/packages/SystemUI/tests/src/com/android/systemui/accessibility/WindowMagnificationControllerTest.java b/packages/SystemUI/tests/src/com/android/systemui/accessibility/WindowMagnificationControllerTest.java
index 539b321..c6440f4 100644
--- a/packages/SystemUI/tests/src/com/android/systemui/accessibility/WindowMagnificationControllerTest.java
+++ b/packages/SystemUI/tests/src/com/android/systemui/accessibility/WindowMagnificationControllerTest.java
@@ -17,10 +17,17 @@
package com.android.systemui.accessibility;
import static android.view.Choreographer.FrameCallback;
+import static android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasItems;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.verify;
@@ -37,17 +44,20 @@
import android.view.SurfaceControl;
import android.view.View;
import android.view.WindowManager;
+import android.view.accessibility.AccessibilityNodeInfo;
import androidx.test.InstrumentationRegistry;
import androidx.test.filters.SmallTest;
import com.android.internal.graphics.SfVsyncFrameCallbackProvider;
+import com.android.systemui.R;
import com.android.systemui.SysuiTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
@@ -71,6 +81,7 @@
private Resources mResources;
private WindowMagnificationController mWindowMagnificationController;
private Instrumentation mInstrumentation;
+ private View mMirrorView;
@Before
public void setUp() {
@@ -83,12 +94,16 @@
).when(mWindowManager).getMaximumWindowMetrics();
mContext.addMockSystemService(Context.WINDOW_SERVICE, mWindowManager);
doAnswer(invocation -> {
- View view = invocation.getArgument(0);
+ mMirrorView = invocation.getArgument(0);
WindowManager.LayoutParams lp = invocation.getArgument(1);
- view.setLayoutParams(lp);
+ mMirrorView.setLayoutParams(lp);
return null;
}).when(mWindowManager).addView(any(View.class), any(WindowManager.LayoutParams.class));
doAnswer(invocation -> {
+ mMirrorView = null;
+ return null;
+ }).when(mWindowManager).removeView(any(View.class));
+ doAnswer(invocation -> {
FrameCallback callback = invocation.getArgument(0);
callback.doFrame(0);
return null;
@@ -147,14 +162,18 @@
}
@Test
- public void setScale_enabled_expectedValue() {
+ public void setScale_enabled_expectedValueAndUpdateStateDescription() {
mInstrumentation.runOnMainSync(
- () -> mWindowMagnificationController.enableWindowMagnification(Float.NaN, Float.NaN,
+ () -> mWindowMagnificationController.enableWindowMagnification(2.0f, Float.NaN,
Float.NaN));
mInstrumentation.runOnMainSync(() -> mWindowMagnificationController.setScale(3.0f));
assertEquals(3.0f, mWindowMagnificationController.getScale(), 0);
+ ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
+ verify(mHandler).postDelayed(runnableArgumentCaptor.capture(), anyLong());
+ runnableArgumentCaptor.getValue().run();
+ assertThat(mMirrorView.getStateDescription().toString(), containsString("300"));
}
@Test
@@ -227,4 +246,52 @@
verify(mResources, atLeastOnce()).getDimensionPixelSize(anyInt());
}
+
+ @Test
+ public void initializeA11yNode_enabled_expectedValues() {
+ mInstrumentation.runOnMainSync(() -> {
+ mWindowMagnificationController.enableWindowMagnification(2.5f, Float.NaN,
+ Float.NaN);
+ });
+ assertNotNull(mMirrorView);
+ final AccessibilityNodeInfo nodeInfo = new AccessibilityNodeInfo();
+
+ mMirrorView.onInitializeAccessibilityNodeInfo(nodeInfo);
+
+ assertNotNull(nodeInfo.getContentDescription());
+ assertThat(nodeInfo.getStateDescription().toString(), containsString("250"));
+ assertThat(nodeInfo.getActionList(),
+ hasItems(new AccessibilityAction(R.id.accessibility_action_zoom_in, null),
+ new AccessibilityAction(R.id.accessibility_action_zoom_out, null),
+ new AccessibilityAction(R.id.accessibility_action_move_right, null),
+ new AccessibilityAction(R.id.accessibility_action_move_left, null),
+ new AccessibilityAction(R.id.accessibility_action_move_down, null),
+ new AccessibilityAction(R.id.accessibility_action_move_up, null)));
+ }
+
+ @Test
+ public void performA11yActions_visible_expectedResults() {
+ mInstrumentation.runOnMainSync(() -> {
+ mWindowMagnificationController.enableWindowMagnification(2.5f, Float.NaN,
+ Float.NaN);
+ });
+ assertNotNull(mMirrorView);
+
+ assertTrue(
+ mMirrorView.performAccessibilityAction(R.id.accessibility_action_zoom_out, null));
+ // Minimum scale is 2.0.
+ assertEquals(2.0f, mWindowMagnificationController.getScale(), 0f);
+
+ assertTrue(mMirrorView.performAccessibilityAction(R.id.accessibility_action_zoom_in, null));
+ assertEquals(3.0f, mWindowMagnificationController.getScale(), 0f);
+
+ // TODO: Verify the final state when the mirror surface is visible.
+ assertTrue(mMirrorView.performAccessibilityAction(R.id.accessibility_action_move_up, null));
+ assertTrue(
+ mMirrorView.performAccessibilityAction(R.id.accessibility_action_move_down, null));
+ assertTrue(
+ mMirrorView.performAccessibilityAction(R.id.accessibility_action_move_right, null));
+ assertTrue(
+ mMirrorView.performAccessibilityAction(R.id.accessibility_action_move_left, null));
+ }
}