Move a11y window bounds computation into a11y package
This moves accessibility windows' touchable region computation and
exposed window selection from the wm package to the a11y package.
This shouldn't change any behavior. Our final goal is to eliminate the
dependency to WM package, but this just moves the code so that future
work will be smoother.
This change is guarded by a new flag.
Bug: 322444245
Test: CtsAccessibilityServiceTestCases CtsAccessibilityTestCases AccessibilityWindowManagerTest
Change-Id: Ib37cb934686d7677f2da74d6ed9ad6e7fcb5efd1
diff --git a/services/accessibility/accessibility.aconfig b/services/accessibility/accessibility.aconfig
index f902439..48e92b3 100644
--- a/services/accessibility/accessibility.aconfig
+++ b/services/accessibility/accessibility.aconfig
@@ -17,6 +17,13 @@
}
flag {
+ name: "compute_window_changes_on_a11y"
+ namespace: "accessibility"
+ description: "Computes accessibility window changes in accessibility instead of wm package."
+ bug: "322444245"
+}
+
+flag {
name: "deprecate_package_list_observer"
namespace: "accessibility"
description: "Stops using the deprecated PackageListObserver."
diff --git a/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java b/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java
index 26c1bc9..b818150 100644
--- a/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java
+++ b/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java
@@ -19,6 +19,8 @@
import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILITY_INTERACTION_CONNECTION;
import static android.accessibilityservice.AccessibilityTrace.FLAGS_WINDOW_MANAGER_INTERNAL;
import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY;
+import static android.view.WindowManager.LayoutParams.TYPE_DOCK_DIVIDER;
+import static android.view.WindowManager.LayoutParams.TYPE_MAGNIFICATION_OVERLAY;
import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED;
import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
@@ -27,6 +29,7 @@
import android.annotation.NonNull;
import android.annotation.Nullable;
+import android.graphics.Point;
import android.graphics.Region;
import android.os.Binder;
import android.os.Handler;
@@ -37,6 +40,7 @@
import android.os.UserHandle;
import android.text.TextUtils;
import android.util.ArrayMap;
+import android.util.ArraySet;
import android.util.Slog;
import android.util.SparseArray;
import android.view.Display;
@@ -52,6 +56,7 @@
import com.android.internal.annotations.VisibleForTesting;
import com.android.server.accessibility.AccessibilitySecurityPolicy.AccessibilityUserManager;
import com.android.server.utils.Slogf;
+import com.android.server.wm.AccessibilityWindowsPopulator.AccessibilityWindow;
import com.android.server.wm.WindowManagerInternal;
import java.io.FileDescriptor;
@@ -60,6 +65,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.stream.Collectors;
/**
@@ -440,7 +446,7 @@
updateWindowsByWindowAttributesLocked(windows);
if (DEBUG) {
Slogf.i(LOG_TAG, "mDisplayId=%d, topFocusedDisplayId=%d, currentUserId=%d, "
- + "visibleBgUsers=%s", mDisplayId, topFocusedDisplayId,
+ + "visibleBgUsers=%s", mDisplayId, topFocusedDisplayId,
mAccessibilityUserManager.getCurrentUserIdLocked(),
mAccessibilityUserManager.getVisibleUserIdsLocked());
if (VERBOSE) {
@@ -460,7 +466,7 @@
mTopFocusedWindowToken = topFocusedWindowToken;
if (DEBUG) {
Slogf.d(LOG_TAG, "onWindowsForAccessibilityChanged(): updating windows for "
- + "display %d and token %s",
+ + "display %d and token %s",
topFocusedDisplayId, topFocusedWindowToken);
}
cacheWindows(windows);
@@ -472,12 +478,168 @@
}
else if (DEBUG) {
Slogf.d(LOG_TAG, "onWindowsForAccessibilityChanged(): NOT updating windows for "
- + "display %d and token %s",
+ + "display %d and token %s",
topFocusedDisplayId, topFocusedWindowToken);
}
}
}
+ /**
+ * Called when the windows for accessibility changed. This is called if
+ * {@link com.android.server.accessibility.Flags.FLAG_COMPUTE_WINDOW_CHANGES_ON_A11Y} is
+ * true.
+ *
+ * @param forceSend Send the windows for accessibility even if they haven't
+ * changed.
+ * @param topFocusedDisplayId The display Id which has the top focused window.
+ * @param topFocusedWindowToken The window token of top focused window.
+ * @param screenSize The size of the display that the change happened.
+ * @param windows The windows for accessibility.
+ */
+ @Override
+ public void onAccessibilityWindowsChanged(boolean forceSend, int topFocusedDisplayId,
+ @NonNull IBinder topFocusedWindowToken, @NonNull Point screenSize,
+ @NonNull List<AccessibilityWindow> windows) {
+ // TODO(b/322444245): Get a screenSize from DisplayManager#getDisplay(int)
+ // .getRealSize().
+ final List<WindowInfo> windowInfoList = createWindowInfoList(screenSize, windows);
+ onWindowsForAccessibilityChanged(forceSend, topFocusedDisplayId,
+ topFocusedWindowToken, windowInfoList);
+ }
+
+ private static List<WindowInfo> createWindowInfoList(@NonNull Point screenSize,
+ @NonNull List<AccessibilityWindow> visibleWindows) {
+ final Set<IBinder> addedWindows = new ArraySet<>();
+ final List<WindowInfo> windows = new ArrayList<>();
+
+ // Avoid allocating Region for each window.
+ final Region regionInWindow = new Region();
+ final Region touchableRegionInScreen = new Region();
+
+ // Iterate until we figure out what is touchable for the entire screen.
+ boolean focusedWindowAdded = false;
+ final Region unaccountedSpace = new Region(0, 0, screenSize.x, screenSize.y);
+ for (final AccessibilityWindow a11yWindow : visibleWindows) {
+ a11yWindow.getTouchableRegionInWindow(regionInWindow);
+ if (windowMattersToAccessibility(a11yWindow, regionInWindow, unaccountedSpace)) {
+ final WindowInfo window = a11yWindow.getWindowInfo();
+ if (window.token != null) {
+ // Even if token is null, the window will be used in calculating visible
+ // windows, but is excluded from the accessibility window list.
+ // TODO(b/322444245): We can call #updateWindowWithWindowAttributes() here.
+ window.regionInScreen.set(regionInWindow);
+ window.layer = addedWindows.size();
+ windows.add(window);
+ addedWindows.add(window.token);
+ }
+
+ if (windowMattersToUnaccountedSpaceComputation(a11yWindow)) {
+ // Account for the space this window takes.
+ a11yWindow.getTouchableRegionInScreen(touchableRegionInScreen);
+ unaccountedSpace.op(touchableRegionInScreen, unaccountedSpace,
+ Region.Op.REVERSE_DIFFERENCE);
+ }
+
+ focusedWindowAdded |= a11yWindow.isFocused();
+ } else if (a11yWindow.isUntouchableNavigationBar()
+ && a11yWindow.getSystemBarInsetsFrame() != null) {
+ // If this widow is navigation bar without touchable region, accounting the
+ // region of navigation bar inset because all touch events from this region
+ // would be received by launcher, i.e. this region is a un-touchable one
+ // for the application.
+ unaccountedSpace.op(
+ a11yWindow.getSystemBarInsetsFrame(),
+ unaccountedSpace,
+ Region.Op.REVERSE_DIFFERENCE);
+ }
+
+ if (unaccountedSpace.isEmpty() && focusedWindowAdded) {
+ break;
+ }
+ }
+
+ // Remove child/parent references to windows that were not added.
+ for (final WindowInfo window : windows) {
+ if (!addedWindows.contains(window.parentToken)) {
+ window.parentToken = null;
+ }
+ if (window.childTokens != null) {
+ final int childTokenCount = window.childTokens.size();
+ for (int j = childTokenCount - 1; j >= 0; j--) {
+ if (!addedWindows.contains(window.childTokens.get(j))) {
+ window.childTokens.remove(j);
+ }
+ }
+ // Leave the child token list if empty.
+ }
+ }
+
+ return windows;
+ }
+
+ private static boolean windowMattersToAccessibility(AccessibilityWindow a11yWindow,
+ Region regionInScreen, Region unaccountedSpace) {
+ if (a11yWindow.ignoreRecentsAnimationForAccessibility()) {
+ return false;
+ }
+
+ if (a11yWindow.isFocused()) {
+ return true;
+ }
+
+ // Ignore non-touchable windows, except the split-screen divider, which is
+ // occasionally non-touchable but still useful for identifying split-screen
+ // mode and the PIP menu.
+ if (!a11yWindow.isTouchable()
+ && a11yWindow.getType() != TYPE_DOCK_DIVIDER && !a11yWindow.isPIPMenu()) {
+ return false;
+ }
+
+ // If the window is completely covered by other windows - ignore.
+ if (unaccountedSpace.quickReject(regionInScreen)) {
+ return false;
+ }
+
+ // Add windows of certain types not covered by modal windows.
+ if (isReportedWindowType(a11yWindow.getType())) {
+ return true;
+ }
+
+ return false;
+ }
+
+ private static boolean isReportedWindowType(int windowType) {
+ return (windowType != WindowManager.LayoutParams.TYPE_WALLPAPER
+ && windowType != WindowManager.LayoutParams.TYPE_BOOT_PROGRESS
+ && windowType != WindowManager.LayoutParams.TYPE_DISPLAY_OVERLAY
+ && windowType != WindowManager.LayoutParams.TYPE_DRAG
+ && windowType != WindowManager.LayoutParams.TYPE_INPUT_CONSUMER
+ && windowType != WindowManager.LayoutParams.TYPE_POINTER
+ && windowType != TYPE_MAGNIFICATION_OVERLAY
+ && windowType != WindowManager.LayoutParams.TYPE_APPLICATION_MEDIA_OVERLAY
+ && windowType != WindowManager.LayoutParams.TYPE_SECURE_SYSTEM_OVERLAY
+ && windowType != WindowManager.LayoutParams.TYPE_PRIVATE_PRESENTATION);
+ }
+
+ // Some windows should be excluded from unaccounted space computation, though they still
+ // should be reported
+ private static boolean windowMattersToUnaccountedSpaceComputation(
+ AccessibilityWindow a11yWindow) {
+ // Do not account space of trusted non-touchable windows, except the split-screen
+ // divider.
+ // If it's not trusted, touch events are not sent to the windows behind it.
+ if (!a11yWindow.isTouchable()
+ && (a11yWindow.getType() != TYPE_DOCK_DIVIDER)
+ && a11yWindow.isTrustedOverlay()) {
+ return false;
+ }
+
+ if (a11yWindow.getType() == WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) {
+ return false;
+ }
+ return true;
+ }
+
private void updateWindowsByWindowAttributesLocked(List<WindowInfo> windows) {
for (int i = windows.size() - 1; i >= 0; i--) {
final WindowInfo windowInfo = windows.get(i);
diff --git a/services/core/Android.bp b/services/core/Android.bp
index 4692099..7940ca6 100644
--- a/services/core/Android.bp
+++ b/services/core/Android.bp
@@ -207,6 +207,7 @@
"notification_flags_lib",
"biometrics_flags_lib",
"am_flags_lib",
+ "com_android_server_accessibility_flags_lib",
"com_android_systemui_shared_flags_lib",
"com_android_wm_shell_flags_lib",
"com.android.server.utils_aconfig-java",
diff --git a/services/core/java/com/android/server/wm/AccessibilityController.java b/services/core/java/com/android/server/wm/AccessibilityController.java
index ee865d3..19dc45c 100644
--- a/services/core/java/com/android/server/wm/AccessibilityController.java
+++ b/services/core/java/com/android/server/wm/AccessibilityController.java
@@ -1539,8 +1539,6 @@
private final Set<IBinder> mTempBinderSet = new ArraySet<>();
- private final Point mTempPoint = new Point();
-
private final Region mTempRegion = new Region();
private final Region mTempRegion2 = new Region();
@@ -1610,8 +1608,9 @@
Slog.i(LOG_TAG, "computeChangedWindows()");
}
- final List<WindowInfo> windows;
+ List<WindowInfo> windows = null;
final List<AccessibilityWindow> visibleWindows = new ArrayList<>();
+ final Point screenSize = new Point();
final int topFocusedDisplayId;
IBinder topFocusedWindowToken = null;
@@ -1639,19 +1638,27 @@
return;
}
final Display display = dc.getDisplay();
- display.getRealSize(mTempPoint);
+ display.getRealSize(screenSize);
mA11yWindowsPopulator.populateVisibleWindowsOnScreenLocked(
mDisplayId, visibleWindows);
- windows = buildWindowInfoListLocked(visibleWindows, mTempPoint);
+ if (!com.android.server.accessibility.Flags.computeWindowChangesOnA11y()) {
+ windows = buildWindowInfoListLocked(visibleWindows, screenSize);
+ }
// Gets the top focused display Id and window token for supporting multi-display.
topFocusedDisplayId = mService.mRoot.getTopFocusedDisplayContent().getDisplayId();
topFocusedWindowToken = topFocusedWindowState.mClient.asBinder();
}
- mCallback.onWindowsForAccessibilityChanged(forceSend, topFocusedDisplayId,
- topFocusedWindowToken, windows);
+
+ if (com.android.server.accessibility.Flags.computeWindowChangesOnA11y()) {
+ mCallback.onAccessibilityWindowsChanged(forceSend, topFocusedDisplayId,
+ topFocusedWindowToken, screenSize, visibleWindows);
+ } else {
+ mCallback.onWindowsForAccessibilityChanged(forceSend, topFocusedDisplayId,
+ topFocusedWindowToken, windows);
+ }
// Recycle the windows as we do not need them.
for (final AccessibilityWindowsPopulator.AccessibilityWindow window : visibleWindows) {
@@ -1660,6 +1667,9 @@
mInitialized = true;
}
+ // Here are old code paths, called when computeWindowChangesOnA11y flag is disabled.
+ // LINT.IfChange
+
/**
* From a list of windows, decides windows to be exposed to accessibility based on touchable
* region in the screen.
@@ -1819,6 +1829,8 @@
&& windowType != WindowManager.LayoutParams.TYPE_PRIVATE_PRESENTATION);
}
+ // LINT.ThenChange(/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java)
+
private WindowState getTopFocusWindow() {
return mService.mRoot.getTopFocusedDisplayContent().mCurrentFocus;
}
diff --git a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java
index 3cf19dd..ac3251c 100644
--- a/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java
+++ b/services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java
@@ -22,6 +22,7 @@
import static com.android.server.wm.utils.RegionUtils.forEachRect;
import android.annotation.NonNull;
+import android.annotation.Nullable;
import android.graphics.Matrix;
import android.graphics.Rect;
import android.graphics.RectF;
@@ -655,6 +656,7 @@
private final Region mTouchableRegionInScreen = new Region();
private final Region mTouchableRegionInWindow = new Region();
private WindowInfo mWindowInfo;
+ private Rect mSystemBarInsetFrame = null;
/**
@@ -718,6 +720,16 @@
Slog.w(TAG, "can't find spec");
}
}
+
+ // Compute system bar insets frame if needed.
+ if (com.android.server.accessibility.Flags.computeWindowChangesOnA11y()
+ && windowState != null && instance.isUntouchableNavigationBar()) {
+ final InsetsSourceProvider provider =
+ windowState.getControllableInsetProvider();
+ if (provider != null) {
+ instance.mSystemBarInsetFrame = provider.getSource().getFrame();
+ }
+ }
return instance;
}
@@ -812,6 +824,15 @@
return mIsPIPMenu;
}
+ /**
+ * @return the system inset frame size if the window is untouchable navigation bar.
+ * Returns null otherwise.
+ */
+ @Nullable
+ public Rect getSystemBarInsetsFrame() {
+ return mSystemBarInsetFrame;
+ }
+
private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion,
Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix) {
// Some modal windows, like the activity with Theme.dialog, has the full screen
diff --git a/services/core/java/com/android/server/wm/WindowManagerInternal.java b/services/core/java/com/android/server/wm/WindowManagerInternal.java
index ae4c3b9..16084c1 100644
--- a/services/core/java/com/android/server/wm/WindowManagerInternal.java
+++ b/services/core/java/com/android/server/wm/WindowManagerInternal.java
@@ -25,6 +25,7 @@
import android.content.ClipData;
import android.content.Context;
import android.graphics.Matrix;
+import android.graphics.Point;
import android.graphics.Rect;
import android.graphics.Region;
import android.hardware.display.DisplayManagerInternal;
@@ -158,7 +159,9 @@
public interface WindowsForAccessibilityCallback {
/**
- * Called when the windows for accessibility changed.
+ * Called when the windows for accessibility changed. This is called if
+ * {@link com.android.server.accessibility.Flags.FLAG_COMPUTE_WINDOW_CHANGES_ON_A11Y} is
+ * false.
*
* @param forceSend Send the windows for accessibility even if they haven't changed.
* @param topFocusedDisplayId The display Id which has the top focused window.
@@ -167,6 +170,23 @@
*/
void onWindowsForAccessibilityChanged(boolean forceSend, int topFocusedDisplayId,
IBinder topFocusedWindowToken, @NonNull List<WindowInfo> windows);
+
+ /**
+ * Called when the windows for accessibility changed. This is called if
+ * {@link com.android.server.accessibility.Flags.FLAG_COMPUTE_WINDOW_CHANGES_ON_A11Y} is
+ * true.
+ * TODO(b/322444245): Remove screenSize parameter by getting it from
+ * DisplayManager#getDisplay(int).getRealSize() on the a11y side.
+ *
+ * @param forceSend Send the windows for accessibility even if they haven't changed.
+ * @param topFocusedDisplayId The display Id which has the top focused window.
+ * @param topFocusedWindowToken The window token of top focused window.
+ * @param screenSize The size of the display that the change happened.
+ * @param windows The windows for accessibility.
+ */
+ void onAccessibilityWindowsChanged(boolean forceSend, int topFocusedDisplayId,
+ @NonNull IBinder topFocusedWindowToken, @NonNull Point screenSize,
+ @NonNull List<AccessibilityWindowsPopulator.AccessibilityWindow> windows);
}
/**