Add ImeTargetChangeListener for IME targets visibility
In order to stablize the IME surface z-ordering use cases from IMMS
side, as IMMS knows the true IME input target availability but lacks
of knowing visibility once the input target removed by itself turns
out unable to hide IME if the next IME layering target on top. Or,
unable to track non-focusable IME layering target visibility
caused IME surface being jumped up-and-down during the IME transition.
In this CL, we added ImeTargetChangeListener for IMMS side
(specifically is ImeVisibilityStateComputer) to track the IME
input target and non-focusable IME layering overlay window visiblity:
- onImeTargetOverlayVisibilityChanged
- onImeInputTargetVisibilityChanged
As this CL is the first-step of providing visibility tracking infra to
ImeVisibilityStateComputer, We will submit follow-up CL with using this
callback for fixing related IME layering issues.
Bug: 258048231
Test: atest WindowStateTests
Change-Id: I90c59c298e2e4568e308fb2c0dc4ca309d5546a3
diff --git a/services/core/java/com/android/server/inputmethod/ImeVisibilityStateComputer.java b/services/core/java/com/android/server/inputmethod/ImeVisibilityStateComputer.java
index ceb9706..25a8835 100644
--- a/services/core/java/com/android/server/inputmethod/ImeVisibilityStateComputer.java
+++ b/services/core/java/com/android/server/inputmethod/ImeVisibilityStateComputer.java
@@ -49,6 +49,7 @@
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.inputmethod.SoftInputShowHideReason;
import com.android.server.LocalServices;
+import com.android.server.wm.ImeTargetChangeListener;
import com.android.server.wm.WindowManagerInternal;
import java.io.PrintWriter;
@@ -172,6 +173,19 @@
mWindowManagerInternal = wmService;
mImeDisplayValidator = imeDisplayValidator;
mPolicy = imePolicy;
+ mWindowManagerInternal.setInputMethodTargetChangeListener(new ImeTargetChangeListener() {
+ @Override
+ public void onImeTargetOverlayVisibilityChanged(IBinder overlayWindowToken,
+ boolean visible, boolean removed) {
+ // TODO(b/258048231): implement logic to fix IME layering overlay visibility issue.
+ }
+
+ @Override
+ public void onImeInputTargetVisibilityChanged(IBinder imeInputTarget,
+ boolean visibleRequested, boolean removed) {
+ // TODO(b/258048231): implement logic to fix IME input target visibility issue.
+ }
+ });
}
/**
diff --git a/services/core/java/com/android/server/wm/DisplayContent.java b/services/core/java/com/android/server/wm/DisplayContent.java
index ef01cc8..ecce913 100644
--- a/services/core/java/com/android/server/wm/DisplayContent.java
+++ b/services/core/java/com/android/server/wm/DisplayContent.java
@@ -656,6 +656,14 @@
*/
private InputTarget mLastImeInputTarget;
+
+ /**
+ * Tracks the windowToken of the input method input target and the corresponding
+ * {@link WindowContainerListener} for monitoring changes (e.g. the requested visibility
+ * change).
+ */
+ private @Nullable Pair<IBinder, WindowContainerListener> mImeTargetTokenListenerPair;
+
/**
* This controls the visibility and animation of the input method window.
*/
@@ -4248,7 +4256,38 @@
@VisibleForTesting
void setImeInputTarget(InputTarget target) {
+ if (mImeTargetTokenListenerPair != null) {
+ // Unregister the listener before changing to the new IME input target.
+ final WindowToken oldToken = mTokenMap.get(mImeTargetTokenListenerPair.first);
+ if (oldToken != null) {
+ oldToken.unregisterWindowContainerListener(mImeTargetTokenListenerPair.second);
+ }
+ mImeTargetTokenListenerPair = null;
+ }
mImeInputTarget = target;
+ // Notify listeners about IME input target window visibility by the target change.
+ if (target != null) {
+ // TODO(b/276743705): Let InputTarget register the visibility change of the hierarchy.
+ final WindowState targetWin = target.getWindowState();
+ if (targetWin != null) {
+ mImeTargetTokenListenerPair = new Pair<>(targetWin.mToken.token,
+ new WindowContainerListener() {
+ @Override
+ public void onVisibleRequestedChanged(boolean isVisibleRequested) {
+ // Notify listeners for IME input target window visibility change
+ // requested by the parent container.
+ mWmService.dispatchImeInputTargetVisibilityChanged(
+ targetWin.mClient.asBinder(), isVisibleRequested,
+ targetWin.mActivityRecord != null
+ && targetWin.mActivityRecord.finishing);
+ }
+ });
+ targetWin.mToken.registerWindowContainerListener(
+ mImeTargetTokenListenerPair.second);
+ mWmService.dispatchImeInputTargetVisibilityChanged(targetWin.mClient.asBinder(),
+ targetWin.isVisible() /* visible */, false /* removed */);
+ }
+ }
if (refreshImeSecureFlag(getPendingTransaction())) {
mWmService.requestTraversal();
}
diff --git a/services/core/java/com/android/server/wm/ImeTargetChangeListener.java b/services/core/java/com/android/server/wm/ImeTargetChangeListener.java
new file mode 100644
index 0000000..8bc445b
--- /dev/null
+++ b/services/core/java/com/android/server/wm/ImeTargetChangeListener.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.wm;
+
+import android.annotation.NonNull;
+import android.os.IBinder;
+
+/**
+ * Callback the IME targeting window visibility change state for
+ * {@link com.android.server.inputmethod.InputMethodManagerService} to manage the IME surface
+ * visibility and z-ordering.
+ */
+public interface ImeTargetChangeListener {
+ /**
+ * Called when a non-IME-focusable overlay window being the IME layering target (e.g. a
+ * window with {@link android.view.WindowManager.LayoutParams#FLAG_NOT_FOCUSABLE} and
+ * {@link android.view.WindowManager.LayoutParams#FLAG_ALT_FOCUSABLE_IM} flags)
+ * has changed its window visibility.
+ *
+ * @param overlayWindowToken the window token of the overlay window.
+ * @param visible the visibility of the overlay window, {@code true} means visible
+ * and {@code false} otherwise.
+ * @param removed Whether the IME target overlay window has being removed.
+ */
+ default void onImeTargetOverlayVisibilityChanged(@NonNull IBinder overlayWindowToken,
+ boolean visible, boolean removed) {
+ }
+
+ /**
+ * Called when the visibility of IME input target window has changed.
+ *
+ * @param imeInputTarget the window token of the IME input target window.
+ * @param visible the new window visibility made by {@param imeInputTarget}. visible is
+ * {@code true} when switching to the new visible IME input target
+ * window and started input, or the same input target relayout to
+ * visible from invisible. In contrast, visible is {@code false} when
+ * closing the input target, or the same input target relayout to
+ * invisible from visible.
+ * @param removed Whether the IME input target window has being removed.
+ */
+ default void onImeInputTargetVisibilityChanged(@NonNull IBinder imeInputTarget, boolean visible,
+ boolean removed) {
+ }
+}
diff --git a/services/core/java/com/android/server/wm/WindowManagerInternal.java b/services/core/java/com/android/server/wm/WindowManagerInternal.java
index 969afe5..4922523 100644
--- a/services/core/java/com/android/server/wm/WindowManagerInternal.java
+++ b/services/core/java/com/android/server/wm/WindowManagerInternal.java
@@ -848,6 +848,16 @@
}
/**
+ * Sets by the {@link com.android.server.inputmethod.InputMethodManagerService} to monitor
+ * the visibility change of the IME targeted windows.
+ *
+ * @see ImeTargetChangeListener#onImeTargetOverlayVisibilityChanged
+ * @see ImeTargetChangeListener#onImeInputTargetVisibilityChanged
+ */
+ public abstract void setInputMethodTargetChangeListener(
+ @NonNull ImeTargetChangeListener listener);
+
+ /**
* Moves the {@link WindowToken} {@code binder} to the display specified by {@code displayId}.
*/
public abstract void moveWindowTokenToDisplay(IBinder binder, int displayId);
diff --git a/services/core/java/com/android/server/wm/WindowManagerService.java b/services/core/java/com/android/server/wm/WindowManagerService.java
index c599da8..497f81f 100644
--- a/services/core/java/com/android/server/wm/WindowManagerService.java
+++ b/services/core/java/com/android/server/wm/WindowManagerService.java
@@ -724,6 +724,9 @@
boolean mHardKeyboardAvailable;
WindowManagerInternal.OnHardKeyboardStatusChangeListener mHardKeyboardStatusChangeListener;
+
+ @Nullable ImeTargetChangeListener mImeTargetChangeListener;
+
SettingsObserver mSettingsObserver;
final EmbeddedWindowController mEmbeddedWindowController;
final AnrController mAnrController;
@@ -1808,6 +1811,10 @@
if (imMayMove) {
displayContent.computeImeTarget(true /* updateImeTarget */);
+ if (win.isImeOverlayLayeringTarget()) {
+ dispatchImeTargetOverlayVisibilityChanged(client.asBinder(),
+ win.isVisibleRequestedOrAdding(), false /* removed */);
+ }
}
// Don't do layout here, the window must call
@@ -2329,6 +2336,8 @@
winAnimator.mSurfaceController.setSecure(win.isSecureLocked());
}
+ final boolean wasVisible = win.isVisible();
+
win.mRelayoutCalled = true;
win.mInRelayout = true;
@@ -2337,7 +2346,6 @@
"Relayout %s: oldVis=%d newVis=%d. %s", win, oldVisibility,
viewVisibility, new RuntimeException().fillInStackTrace());
-
win.setDisplayLayoutNeeded();
win.mGivenInsetsPending = (flags & WindowManagerGlobal.RELAYOUT_INSETS_PENDING) != 0;
@@ -2502,6 +2510,18 @@
}
win.mInRelayout = false;
+ final boolean winVisibleChanged = win.isVisible() != wasVisible;
+ if (win.isImeOverlayLayeringTarget() && winVisibleChanged) {
+ dispatchImeTargetOverlayVisibilityChanged(client.asBinder(),
+ win.isVisible(), false /* removed */);
+ }
+ // Notify listeners about IME input target window visibility change.
+ final boolean isImeInputTarget = win.getDisplayContent().getImeInputTarget() == win;
+ if (isImeInputTarget && winVisibleChanged) {
+ dispatchImeInputTargetVisibilityChanged(win.mClient.asBinder(),
+ win.isVisible() /* visible */, false /* removed */);
+ }
+
if (outSyncIdBundle != null) {
final int maybeSyncSeqId;
if (USE_BLAST_SYNC && win.useBLASTSync() && viewVisibility == View.VISIBLE
@@ -3326,6 +3346,22 @@
});
}
+ void dispatchImeTargetOverlayVisibilityChanged(@NonNull IBinder token, boolean visible,
+ boolean removed) {
+ if (mImeTargetChangeListener != null) {
+ mH.post(() -> mImeTargetChangeListener.onImeTargetOverlayVisibilityChanged(token,
+ visible, removed));
+ }
+ }
+
+ void dispatchImeInputTargetVisibilityChanged(@NonNull IBinder token, boolean visible,
+ boolean removed) {
+ if (mImeTargetChangeListener != null) {
+ mH.post(() -> mImeTargetChangeListener.onImeInputTargetVisibilityChanged(token,
+ visible, removed));
+ }
+ }
+
@Override
public void setSwitchingUser(boolean switching) {
if (!checkCallingPermission(Manifest.permission.INTERACT_ACROSS_USERS_FULL,
@@ -8238,6 +8274,13 @@
}
return null;
}
+
+ @Override
+ public void setInputMethodTargetChangeListener(@NonNull ImeTargetChangeListener listener) {
+ synchronized (mGlobalLock) {
+ mImeTargetChangeListener = listener;
+ }
+ }
}
private final class ImeTargetVisibilityPolicyImpl extends ImeTargetVisibilityPolicy {
diff --git a/services/core/java/com/android/server/wm/WindowState.java b/services/core/java/com/android/server/wm/WindowState.java
index d1bd06f..a18c443 100644
--- a/services/core/java/com/android/server/wm/WindowState.java
+++ b/services/core/java/com/android/server/wm/WindowState.java
@@ -353,6 +353,7 @@
// overlay window is hidden because the owning app is suspended
private boolean mHiddenWhileSuspended;
private boolean mAppOpVisibility = true;
+
boolean mPermanentlyHidden; // the window should never be shown again
// This is a non-system overlay window that is currently force hidden.
private boolean mForceHideNonSystemOverlayWindow;
@@ -2345,6 +2346,10 @@
mWinAnimator.destroySurfaceLocked(getSyncTransaction());
super.removeImmediately();
+ if (isImeOverlayLayeringTarget()) {
+ mWmService.dispatchImeTargetOverlayVisibilityChanged(mClient.asBinder(),
+ false /* visible */, true /* removed */);
+ }
final DisplayContent dc = getDisplayContent();
if (isImeLayeringTarget()) {
// Remove the attached IME screenshot surface.
@@ -2355,6 +2360,8 @@
dc.computeImeTarget(true /* updateImeTarget */);
}
if (dc.getImeInputTarget() == this && !inRelaunchingActivity()) {
+ mWmService.dispatchImeInputTargetVisibilityChanged(mClient.asBinder(),
+ false /* visible */, true /* removed */);
dc.updateImeInputAndControlTarget(null);
}
@@ -5490,6 +5497,14 @@
return getDisplayContent().getImeTarget(IME_TARGET_LAYERING) == this;
}
+ /**
+ * Whether the window is non-focusable IME overlay layering target.
+ */
+ boolean isImeOverlayLayeringTarget() {
+ return isImeLayeringTarget()
+ && (mAttrs.flags & (FLAG_ALT_FOCUSABLE_IM | FLAG_NOT_FOCUSABLE)) != 0;
+ }
+
WindowState getImeLayeringTarget() {
final InsetsControlTarget target = getDisplayContent().getImeTarget(IME_TARGET_LAYERING);
return target != null ? target.getWindow() : null;
diff --git a/services/tests/wmtests/src/com/android/server/wm/WindowStateTests.java b/services/tests/wmtests/src/com/android/server/wm/WindowStateTests.java
index b48fd7d..457929b 100644
--- a/services/tests/wmtests/src/com/android/server/wm/WindowStateTests.java
+++ b/services/tests/wmtests/src/com/android/server/wm/WindowStateTests.java
@@ -20,6 +20,7 @@
import static android.app.WindowConfiguration.WINDOWING_MODE_FULLSCREEN;
import static android.app.WindowConfiguration.WINDOWING_MODE_MULTI_WINDOW;
import static android.app.WindowConfiguration.WINDOWING_MODE_PINNED;
+import static android.view.Display.DEFAULT_DISPLAY;
import static android.view.InsetsSource.ID_IME;
import static android.view.Surface.ROTATION_0;
import static android.view.Surface.ROTATION_270;
@@ -83,17 +84,25 @@
import android.graphics.Matrix;
import android.graphics.Point;
import android.graphics.Rect;
+import android.os.Bundle;
import android.os.IBinder;
import android.os.InputConfig;
import android.os.RemoteException;
import android.platform.test.annotations.Presubmit;
import android.util.ArraySet;
+import android.util.MergedConfiguration;
import android.view.Gravity;
+import android.view.IWindow;
+import android.view.IWindowSessionCallback;
import android.view.InputWindowHandle;
import android.view.InsetsSource;
+import android.view.InsetsSourceControl;
import android.view.InsetsState;
import android.view.SurfaceControl;
+import android.view.View;
+import android.view.WindowInsets;
import android.view.WindowManager;
+import android.window.ClientWindowFrames;
import android.window.ITaskFragmentOrganizer;
import android.window.TaskFragmentOrganizer;
@@ -1250,4 +1259,118 @@
assertEquals(new ArraySet(Arrays.asList(expectedArea1, expectedArea2)),
new ArraySet(unrestrictedKeepClearAreas));
}
+
+ @Test
+ public void testImeTargetChangeListener_OnImeInputTargetVisibilityChanged() {
+ final TestImeTargetChangeListener listener = new TestImeTargetChangeListener();
+ mWm.mImeTargetChangeListener = listener;
+
+ final WindowState imeTarget = createWindow(null /* parent */, TYPE_BASE_APPLICATION,
+ createActivityRecord(mDisplayContent), "imeTarget");
+
+ imeTarget.mActivityRecord.setVisibleRequested(true);
+ makeWindowVisible(imeTarget);
+ mDisplayContent.setImeInputTarget(imeTarget);
+ waitHandlerIdle(mWm.mH);
+
+ assertThat(listener.mImeTargetToken).isEqualTo(imeTarget.mClient.asBinder());
+ assertThat(listener.mIsRemoved).isFalse();
+ assertThat(listener.mIsVisibleForImeInputTarget).isTrue();
+
+ imeTarget.mActivityRecord.setVisibleRequested(false);
+ waitHandlerIdle(mWm.mH);
+
+ assertThat(listener.mImeTargetToken).isEqualTo(imeTarget.mClient.asBinder());
+ assertThat(listener.mIsRemoved).isFalse();
+ assertThat(listener.mIsVisibleForImeInputTarget).isFalse();
+
+ imeTarget.removeImmediately();
+ assertThat(listener.mImeTargetToken).isEqualTo(imeTarget.mClient.asBinder());
+ assertThat(listener.mIsRemoved).isTrue();
+ assertThat(listener.mIsVisibleForImeInputTarget).isFalse();
+ }
+
+ @SetupWindows(addWindows = {W_INPUT_METHOD})
+ @Test
+ public void testImeTargetChangeListener_OnImeTargetOverlayVisibilityChanged() {
+ final TestImeTargetChangeListener listener = new TestImeTargetChangeListener();
+ mWm.mImeTargetChangeListener = listener;
+
+ // Scenario 1: test addWindow/relayoutWindow to add Ime layering overlay window as visible.
+ final WindowToken windowToken = createTestWindowToken(TYPE_APPLICATION_OVERLAY,
+ mDisplayContent);
+ final IWindow client = new TestIWindow();
+ final Session session = new Session(mWm, new IWindowSessionCallback.Stub() {
+ @Override
+ public void onAnimatorScaleChanged(float v) throws RemoteException {
+
+ }
+ });
+ final ClientWindowFrames outFrames = new ClientWindowFrames();
+ final MergedConfiguration outConfig = new MergedConfiguration();
+ final SurfaceControl outSurfaceControl = new SurfaceControl();
+ final InsetsState outInsetsState = new InsetsState();
+ final InsetsSourceControl.Array outControls = new InsetsSourceControl.Array();
+ final Bundle outBundle = new Bundle();
+ final WindowManager.LayoutParams params = new WindowManager.LayoutParams(
+ TYPE_APPLICATION_OVERLAY);
+ params.setTitle("imeLayeringTargetOverlay");
+ params.token = windowToken.token;
+ params.flags = FLAG_NOT_FOCUSABLE | FLAG_ALT_FOCUSABLE_IM;
+
+ mWm.addWindow(session, client, params, View.VISIBLE, DEFAULT_DISPLAY,
+ 0 /* userUd */, WindowInsets.Type.defaultVisible(), null, new InsetsState(),
+ new InsetsSourceControl.Array(), new Rect(), new float[1]);
+ mWm.relayoutWindow(session, client, params, 100, 200, View.VISIBLE, 0, 0, 0,
+ outFrames, outConfig, outSurfaceControl, outInsetsState, outControls, outBundle);
+ waitHandlerIdle(mWm.mH);
+
+ final WindowState imeLayeringTargetOverlay = mDisplayContent.getWindow(
+ w -> w.mClient.asBinder() == client.asBinder());
+ assertThat(imeLayeringTargetOverlay.isVisible()).isTrue();
+ assertThat(listener.mImeTargetToken).isEqualTo(client.asBinder());
+ assertThat(listener.mIsRemoved).isFalse();
+ assertThat(listener.mIsVisibleForImeTargetOverlay).isTrue();
+
+ // Scenario 2: test relayoutWindow to let the Ime layering target overlay window invisible.
+ mWm.relayoutWindow(session, client, params, 100, 200, View.GONE, 0, 0, 0,
+ outFrames, outConfig, outSurfaceControl, outInsetsState, outControls, outBundle);
+ waitHandlerIdle(mWm.mH);
+
+ assertThat(imeLayeringTargetOverlay.isVisible()).isFalse();
+ assertThat(listener.mImeTargetToken).isEqualTo(client.asBinder());
+ assertThat(listener.mIsRemoved).isFalse();
+ assertThat(listener.mIsVisibleForImeTargetOverlay).isFalse();
+
+ // Scenario 3: test removeWindow to remove the Ime layering target overlay window.
+ mWm.removeWindow(session, client);
+ waitHandlerIdle(mWm.mH);
+
+ assertThat(listener.mImeTargetToken).isEqualTo(client.asBinder());
+ assertThat(listener.mIsRemoved).isTrue();
+ assertThat(listener.mIsVisibleForImeTargetOverlay).isFalse();
+ }
+
+ private static class TestImeTargetChangeListener implements ImeTargetChangeListener {
+ private IBinder mImeTargetToken;
+ private boolean mIsRemoved;
+ private boolean mIsVisibleForImeTargetOverlay;
+ private boolean mIsVisibleForImeInputTarget;
+
+ @Override
+ public void onImeTargetOverlayVisibilityChanged(IBinder overlayWindowToken, boolean visible,
+ boolean removed) {
+ mImeTargetToken = overlayWindowToken;
+ mIsVisibleForImeTargetOverlay = visible;
+ mIsRemoved = removed;
+ }
+
+ @Override
+ public void onImeInputTargetVisibilityChanged(IBinder imeInputTarget,
+ boolean visibleRequested, boolean removed) {
+ mImeTargetToken = imeInputTarget;
+ mIsVisibleForImeInputTarget = visibleRequested;
+ mIsRemoved = removed;
+ }
+ }
}