Implement pinch resizing in PiP2
Use transitions and PipTransitionState to allow
pinch-to-resize gestures in PiP2.
At the moment, the animator for when both pointers
are release is not set up so after pinching is complete
and pointers are release a transition is played w/o
any animation - just a jumpcut instead.
Bug: 340883807
Test: enter PiP2 and pinch-to-resize
Change-Id: Ic2a11444506663c96a2c8ac052092e84ea292889
diff --git a/libs/WindowManager/Shell/src/com/android/wm/shell/dagger/pip/Pip2Module.java b/libs/WindowManager/Shell/src/com/android/wm/shell/dagger/pip/Pip2Module.java
index 414a9d1..01364d1 100644
--- a/libs/WindowManager/Shell/src/com/android/wm/shell/dagger/pip/Pip2Module.java
+++ b/libs/WindowManager/Shell/src/com/android/wm/shell/dagger/pip/Pip2Module.java
@@ -133,6 +133,7 @@
PipBoundsAlgorithm pipBoundsAlgorithm,
@NonNull PipBoundsState pipBoundsState,
@NonNull PipTransitionState pipTransitionState,
+ @NonNull PipScheduler pipScheduler,
@NonNull SizeSpecSource sizeSpecSource,
PipMotionHelper pipMotionHelper,
FloatingContentCoordinator floatingContentCoordinator,
@@ -140,7 +141,7 @@
@ShellMainThread ShellExecutor mainExecutor,
Optional<PipPerfHintController> pipPerfHintControllerOptional) {
return new PipTouchHandler(context, shellInit, menuPhoneController, pipBoundsAlgorithm,
- pipBoundsState, pipTransitionState, sizeSpecSource, pipMotionHelper,
+ pipBoundsState, pipTransitionState, pipScheduler, sizeSpecSource, pipMotionHelper,
floatingContentCoordinator, pipUiEventLogger, mainExecutor,
pipPerfHintControllerOptional);
}
diff --git a/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipMotionHelper.java b/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipMotionHelper.java
index a097a0f..be10151 100644
--- a/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipMotionHelper.java
+++ b/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipMotionHelper.java
@@ -58,7 +58,8 @@
* A helper to animate and manipulate the PiP.
*/
public class PipMotionHelper implements PipAppOpsListener.Callback,
- FloatingContentCoordinator.FloatingContent {
+ FloatingContentCoordinator.FloatingContent,
+ PipTransitionState.PipTransitionStateChangedListener {
private static final String TAG = "PipMotionHelper";
private static final String FLING_BOUNDS_CHANGE = "fling_bounds_change";
private static final boolean DEBUG = false;
@@ -181,7 +182,7 @@
}
};
mPipTransitionState = pipTransitionState;
- mPipTransitionState.addPipTransitionStateChangedListener(this::onPipTransitionStateChanged);
+ mPipTransitionState.addPipTransitionStateChangedListener(this);
}
void init() {
@@ -687,7 +688,8 @@
// setAnimatingToBounds(toBounds);
}
- private void onPipTransitionStateChanged(@PipTransitionState.TransitionState int oldState,
+ @Override
+ public void onPipTransitionStateChanged(@PipTransitionState.TransitionState int oldState,
@PipTransitionState.TransitionState int newState,
@Nullable Bundle extra) {
switch (newState) {
diff --git a/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipResizeGestureHandler.java b/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipResizeGestureHandler.java
index 04cf350..b55a41d 100644
--- a/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipResizeGestureHandler.java
+++ b/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipResizeGestureHandler.java
@@ -24,6 +24,7 @@
import android.graphics.PointF;
import android.graphics.Rect;
import android.hardware.input.InputManager;
+import android.os.Bundle;
import android.os.Looper;
import android.view.BatchedInputEventReceiver;
import android.view.Choreographer;
@@ -32,6 +33,7 @@
import android.view.InputEventReceiver;
import android.view.InputMonitor;
import android.view.MotionEvent;
+import android.view.SurfaceControl;
import android.view.ViewConfiguration;
import androidx.annotation.VisibleForTesting;
@@ -51,16 +53,20 @@
* Helper on top of PipTouchHandler that handles inputs OUTSIDE of the PIP window, which is used to
* trigger dynamic resize.
*/
-public class PipResizeGestureHandler {
+public class PipResizeGestureHandler implements
+ PipTransitionState.PipTransitionStateChangedListener {
private static final String TAG = "PipResizeGestureHandler";
private static final int PINCH_RESIZE_SNAP_DURATION = 250;
private static final float PINCH_RESIZE_AUTO_MAX_RATIO = 0.9f;
+ private static final String RESIZE_BOUNDS_CHANGE = "resize_bounds_change";
private final Context mContext;
private final PipBoundsAlgorithm mPipBoundsAlgorithm;
private final PipBoundsState mPipBoundsState;
private final PipTouchState mPipTouchState;
+ private final PipScheduler mPipScheduler;
+ private final PipTransitionState mPipTransitionState;
private final PhonePipMenuController mPhonePipMenuController;
private final PipUiEventLogger mPipUiEventLogger;
private final PipPinchResizingAlgorithm mPinchResizingAlgorithm;
@@ -88,6 +94,7 @@
private boolean mIsSysUiStateValid;
private boolean mThresholdCrossed;
private boolean mOngoingPinchToResize = false;
+ private boolean mWaitingForBoundsChangeTransition = false;
private float mAngle = 0;
int mFirstIndex = -1;
int mSecondIndex = -1;
@@ -104,11 +111,17 @@
private int mCtrlType;
private int mOhmOffset;
- public PipResizeGestureHandler(Context context, PipBoundsAlgorithm pipBoundsAlgorithm,
- PipBoundsState pipBoundsState, PipTouchState pipTouchState,
+ public PipResizeGestureHandler(Context context,
+ PipBoundsAlgorithm pipBoundsAlgorithm,
+ PipBoundsState pipBoundsState,
+ PipTouchState pipTouchState,
+ PipScheduler pipScheduler,
+ PipTransitionState pipTransitionState,
Runnable updateMovementBoundsRunnable,
- PipUiEventLogger pipUiEventLogger, PhonePipMenuController menuActivityController,
- ShellExecutor mainExecutor, @Nullable PipPerfHintController pipPerfHintController) {
+ PipUiEventLogger pipUiEventLogger,
+ PhonePipMenuController menuActivityController,
+ ShellExecutor mainExecutor,
+ @Nullable PipPerfHintController pipPerfHintController) {
mContext = context;
mDisplayId = context.getDisplayId();
mMainExecutor = mainExecutor;
@@ -116,6 +129,11 @@
mPipBoundsAlgorithm = pipBoundsAlgorithm;
mPipBoundsState = pipBoundsState;
mPipTouchState = pipTouchState;
+ mPipScheduler = pipScheduler;
+
+ mPipTransitionState = pipTransitionState;
+ mPipTransitionState.addPipTransitionStateChangedListener(this);
+
mUpdateMovementBoundsRunnable = updateMovementBoundsRunnable;
mPhonePipMenuController = menuActivityController;
mPipUiEventLogger = pipUiEventLogger;
@@ -125,6 +143,7 @@
mUserResizeBounds.set(rect);
// mMotionHelper.synchronizePinnedStackBounds();
mUpdateMovementBoundsRunnable.run();
+ mPipBoundsState.setBounds(rect);
resetState();
};
}
@@ -202,7 +221,7 @@
@VisibleForTesting
void onInputEvent(InputEvent ev) {
if (!mEnablePinchResize) {
- // No need to handle anything if neither form of resizing is enabled.
+ // No need to handle anything if resizing isn't enabled.
return;
}
@@ -227,7 +246,7 @@
}
}
- if (mEnablePinchResize && mOngoingPinchToResize) {
+ if (mOngoingPinchToResize) {
onPinchResize(mv);
}
}
@@ -249,13 +268,11 @@
}
boolean willStartResizeGesture(MotionEvent ev) {
- if (isInValidSysUiState()) {
- if (ev.getActionMasked() == MotionEvent.ACTION_POINTER_DOWN) {
- if (mEnablePinchResize && ev.getPointerCount() == 2) {
- onPinchResize(ev);
- mOngoingPinchToResize = mAllowGesture;
- return mAllowGesture;
- }
+ if (ev.getActionMasked() == MotionEvent.ACTION_POINTER_DOWN) {
+ if (mEnablePinchResize && ev.getPointerCount() == 2) {
+ onPinchResize(ev);
+ mOngoingPinchToResize = mAllowGesture;
+ return mAllowGesture;
}
}
return false;
@@ -284,7 +301,6 @@
mSecondIndex = -1;
mAllowGesture = false;
finishResize();
- cleanUpHighPerfSessionMaybe();
}
if (ev.getPointerCount() != 2) {
@@ -347,10 +363,7 @@
mDownSecondPoint, mLastPoint, mLastSecondPoint, mMinSize, mMaxSize,
mDownBounds, mLastResizeBounds);
- /*
- mPipTaskOrganizer.scheduleUserResizePip(mDownBounds, mLastResizeBounds,
- mAngle, null);
- */
+ mPipScheduler.scheduleUserResizePip(mLastResizeBounds, mAngle);
mPipBoundsState.setHasUserResizedPip(true);
}
}
@@ -399,57 +412,43 @@
}
private void finishResize() {
- if (!mLastResizeBounds.isEmpty()) {
- // Pinch-to-resize needs to re-calculate snap fraction and animate to the snapped
- // position correctly. Drag-resize does not need to move, so just finalize resize.
- if (mOngoingPinchToResize) {
- final Rect startBounds = new Rect(mLastResizeBounds);
- // If user resize is pretty close to max size, just auto resize to max.
- if (mLastResizeBounds.width() >= PINCH_RESIZE_AUTO_MAX_RATIO * mMaxSize.x
- || mLastResizeBounds.height() >= PINCH_RESIZE_AUTO_MAX_RATIO * mMaxSize.y) {
- resizeRectAboutCenter(mLastResizeBounds, mMaxSize.x, mMaxSize.y);
- }
-
- // If user resize is smaller than min size, auto resize to min
- if (mLastResizeBounds.width() < mMinSize.x
- || mLastResizeBounds.height() < mMinSize.y) {
- resizeRectAboutCenter(mLastResizeBounds, mMinSize.x, mMinSize.y);
- }
-
- // get the current movement bounds
- final Rect movementBounds = mPipBoundsAlgorithm
- .getMovementBounds(mLastResizeBounds);
-
- // snap mLastResizeBounds to the correct edge based on movement bounds
- snapToMovementBoundsEdge(mLastResizeBounds, movementBounds);
-
- final float snapFraction = mPipBoundsAlgorithm.getSnapFraction(
- mLastResizeBounds, movementBounds);
- mPipBoundsAlgorithm.applySnapFraction(mLastResizeBounds, snapFraction);
-
- // disable any touch events beyond resizing too
- mPipTouchState.setAllowInputEvents(false);
-
- /*
- mPipTaskOrganizer.scheduleAnimateResizePip(startBounds, mLastResizeBounds,
- PINCH_RESIZE_SNAP_DURATION, mAngle, mUpdateResizeBoundsCallback, () -> {
- // enable touch events
- mPipTouchState.setAllowInputEvents(true);
- });
- */
- } else {
- /*
- mPipTaskOrganizer.scheduleFinishResizePip(mLastResizeBounds,
- TRANSITION_DIRECTION_USER_RESIZE,
- mUpdateResizeBoundsCallback);
- */
- }
- final float magnetRadiusPercent = (float) mLastResizeBounds.width() / mMinSize.x / 2.f;
- mPipUiEventLogger.log(
- PipUiEventLogger.PipUiEventEnum.PICTURE_IN_PICTURE_RESIZE);
- } else {
+ if (mLastResizeBounds.isEmpty()) {
resetState();
}
+ if (!mOngoingPinchToResize) {
+ return;
+ }
+ final Rect startBounds = new Rect(mLastResizeBounds);
+
+ // If user resize is pretty close to max size, just auto resize to max.
+ if (mLastResizeBounds.width() >= PINCH_RESIZE_AUTO_MAX_RATIO * mMaxSize.x
+ || mLastResizeBounds.height() >= PINCH_RESIZE_AUTO_MAX_RATIO * mMaxSize.y) {
+ resizeRectAboutCenter(mLastResizeBounds, mMaxSize.x, mMaxSize.y);
+ }
+
+ // If user resize is smaller than min size, auto resize to min
+ if (mLastResizeBounds.width() < mMinSize.x
+ || mLastResizeBounds.height() < mMinSize.y) {
+ resizeRectAboutCenter(mLastResizeBounds, mMinSize.x, mMinSize.y);
+ }
+
+ // get the current movement bounds
+ final Rect movementBounds = mPipBoundsAlgorithm
+ .getMovementBounds(mLastResizeBounds);
+
+ // snap mLastResizeBounds to the correct edge based on movement bounds
+ snapToMovementBoundsEdge(mLastResizeBounds, movementBounds);
+
+ final float snapFraction = mPipBoundsAlgorithm.getSnapFraction(
+ mLastResizeBounds, movementBounds);
+ mPipBoundsAlgorithm.applySnapFraction(mLastResizeBounds, snapFraction);
+
+ // Update the transition state to schedule a resize transition.
+ Bundle extra = new Bundle();
+ extra.putBoolean(RESIZE_BOUNDS_CHANGE, true);
+ mPipTransitionState.setState(PipTransitionState.SCHEDULED_BOUNDS_CHANGE, extra);
+
+ mPipUiEventLogger.log(PipUiEventLogger.PipUiEventEnum.PICTURE_IN_PICTURE_RESIZE);
}
private void resetState() {
@@ -509,6 +508,40 @@
rect.set(l, t, r, b);
}
+ @Override
+ public void onPipTransitionStateChanged(@PipTransitionState.TransitionState int oldState,
+ @PipTransitionState.TransitionState int newState, @Nullable Bundle extra) {
+ switch (newState) {
+ case PipTransitionState.SCHEDULED_BOUNDS_CHANGE:
+ if (!extra.getBoolean(RESIZE_BOUNDS_CHANGE)) break;
+ mWaitingForBoundsChangeTransition = true;
+ mPipScheduler.scheduleAnimateResizePip(mLastResizeBounds);
+ break;
+ case PipTransitionState.CHANGING_PIP_BOUNDS:
+ if (!mWaitingForBoundsChangeTransition) break;
+
+ // If bounds change transition was scheduled from this class, handle leash updates.
+ mWaitingForBoundsChangeTransition = false;
+
+ SurfaceControl.Transaction startTx = extra.getParcelable(
+ PipTransition.PIP_START_TX, SurfaceControl.Transaction.class);
+ Rect destinationBounds = extra.getParcelable(
+ PipTransition.PIP_DESTINATION_BOUNDS, Rect.class);
+ startTx.setPosition(mPipTransitionState.mPinnedTaskLeash,
+ destinationBounds.left, destinationBounds.top);
+ startTx.apply();
+
+ // All motion operations have actually finished, so make bounds cache updates.
+ cleanUpHighPerfSessionMaybe();
+
+ // Setting state to CHANGED_PIP_BOUNDS applies finishTx and notifies Core.
+ mPipTransitionState.setState(PipTransitionState.CHANGED_PIP_BOUNDS);
+
+ mUpdateResizeBoundsCallback.accept(destinationBounds);
+ break;
+ }
+ }
+
/**
* Dumps the {@link PipResizeGestureHandler} state.
*/
diff --git a/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipScheduler.java b/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipScheduler.java
index c5b0de3..4947507 100644
--- a/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipScheduler.java
+++ b/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipScheduler.java
@@ -24,6 +24,7 @@
import android.content.Context;
import android.content.Intent;
import android.content.IntentFilter;
+import android.graphics.Matrix;
import android.graphics.Rect;
import android.view.SurfaceControl;
import android.window.WindowContainerTransaction;
@@ -165,6 +166,16 @@
* {@link WindowContainerTransaction}.
*/
public void scheduleUserResizePip(Rect toBounds) {
+ scheduleUserResizePip(toBounds, 0f /* degrees */);
+ }
+
+ /**
+ * Directly perform a scaled matrix transformation on the leash. This will not perform any
+ * {@link WindowContainerTransaction}.
+ *
+ * @param degrees the angle to rotate the bounds to.
+ */
+ public void scheduleUserResizePip(Rect toBounds, float degrees) {
if (toBounds.isEmpty()) {
ProtoLog.w(ShellProtoLogGroup.WM_SHELL_PICTURE_IN_PICTURE,
"%s: Attempted to user resize PIP to empty bounds, aborting.", TAG);
@@ -172,7 +183,16 @@
}
SurfaceControl leash = mPipTransitionState.mPinnedTaskLeash;
final SurfaceControl.Transaction tx = new SurfaceControl.Transaction();
- tx.setPosition(leash, toBounds.left, toBounds.top);
+
+ Matrix transformTensor = new Matrix();
+ final float[] mMatrixTmp = new float[9];
+ final float scale = (float) toBounds.width() / mPipBoundsState.getBounds().width();
+
+ transformTensor.setScale(scale, scale);
+ transformTensor.postTranslate(toBounds.left, toBounds.top);
+ transformTensor.postRotate(degrees, toBounds.centerX(), toBounds.centerY());
+
+ tx.setMatrix(leash, transformTensor, mMatrixTmp);
tx.apply();
}
}
diff --git a/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipTouchHandler.java b/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipTouchHandler.java
index 9c6e3ea..319d199 100644
--- a/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipTouchHandler.java
+++ b/libs/WindowManager/Shell/src/com/android/wm/shell/pip2/phone/PipTouchHandler.java
@@ -73,7 +73,7 @@
* Manages all the touch handling for PIP on the Phone, including moving, dismissing and expanding
* the PIP.
*/
-public class PipTouchHandler {
+public class PipTouchHandler implements PipTransitionState.PipTransitionStateChangedListener {
private static final String TAG = "PipTouchHandler";
private static final float DEFAULT_STASH_VELOCITY_THRESHOLD = 18000.f;
@@ -84,6 +84,7 @@
private final PipBoundsAlgorithm mPipBoundsAlgorithm;
@NonNull private final PipBoundsState mPipBoundsState;
@NonNull private final PipTransitionState mPipTransitionState;
+ @NonNull private final PipScheduler mPipScheduler;
@NonNull private final SizeSpecSource mSizeSpecSource;
private final PipUiEventLogger mPipUiEventLogger;
private final PipDismissTargetHandler mPipDismissTargetHandler;
@@ -173,6 +174,7 @@
PipBoundsAlgorithm pipBoundsAlgorithm,
@NonNull PipBoundsState pipBoundsState,
@NonNull PipTransitionState pipTransitionState,
+ @NonNull PipScheduler pipScheduler,
@NonNull SizeSpecSource sizeSpecSource,
PipMotionHelper pipMotionHelper,
FloatingContentCoordinator floatingContentCoordinator,
@@ -188,6 +190,7 @@
mPipTransitionState = pipTransitionState;
mPipTransitionState.addPipTransitionStateChangedListener(this::onPipTransitionStateChanged);
+ mPipScheduler = pipScheduler;
mSizeSpecSource = sizeSpecSource;
mMenuController = menuController;
mPipUiEventLogger = pipUiEventLogger;
@@ -213,10 +216,10 @@
},
menuController::hideMenu,
mainExecutor);
- mPipResizeGestureHandler =
- new PipResizeGestureHandler(context, pipBoundsAlgorithm, pipBoundsState,
- mTouchState, this::updateMovementBounds, pipUiEventLogger,
- menuController, mainExecutor, mPipPerfHintController);
+ mPipResizeGestureHandler = new PipResizeGestureHandler(context, pipBoundsAlgorithm,
+ pipBoundsState, mTouchState, mPipScheduler, mPipTransitionState,
+ this::updateMovementBounds, pipUiEventLogger, menuController, mainExecutor,
+ mPipPerfHintController);
mPipBoundsState.addOnAspectRatioChangedCallback(this::updateMinMaxSize);
if (PipUtils.isPip2ExperimentEnabled()) {
@@ -1075,7 +1078,8 @@
mPipResizeGestureHandler.setOhmOffset(offset);
}
- private void onPipTransitionStateChanged(@PipTransitionState.TransitionState int oldState,
+ @Override
+ public void onPipTransitionStateChanged(@PipTransitionState.TransitionState int oldState,
@PipTransitionState.TransitionState int newState,
@Nullable Bundle extra) {
switch (newState) {