diff --git a/quickstep/src/com/android/quickstep/views/ShelfScrimView.java b/quickstep/src/com/android/quickstep/views/ShelfScrimView.java
index 63c8023..5a2f5c1 100644
--- a/quickstep/src/com/android/quickstep/views/ShelfScrimView.java
+++ b/quickstep/src/com/android/quickstep/views/ShelfScrimView.java
@@ -74,6 +74,9 @@
     private int mMidAlpha;
     private float mMidProgress;
 
+    // The progress at which the drag handle starts moving up with the shelf.
+    private float mDragHandleProgress;
+
     private Interpolator mBeforeMidProgressColorInterpolator = ACCEL;
     private Interpolator mAfterMidProgressColorInterpolator = ACCEL;
 
@@ -150,15 +153,16 @@
 
             if ((OVERVIEW.getVisibleElements(mLauncher) & ALL_APPS_HEADER_EXTRA) == 0) {
                 mMidProgress = 1;
+                mDragHandleProgress = 1;
                 mMidAlpha = 0;
             } else {
                 mMidAlpha = Themes.getAttrInteger(getContext(), R.attr.allAppsInterimScrimAlpha);
+                mMidProgress =  OVERVIEW.getVerticalProgress(mLauncher);
                 Rect hotseatPadding = dp.getHotseatLayoutPadding();
                 int hotseatSize = dp.hotseatBarSizePx + dp.getInsets().bottom
                         - hotseatPadding.bottom - hotseatPadding.top;
-                float arrowTop = Math.min(hotseatSize, OverviewState.getDefaultSwipeHeight(dp));
-                mMidProgress =  1 - (arrowTop / mShiftRange);
-
+                float dragHandleTop = Math.min(hotseatSize, OverviewState.getDefaultSwipeHeight(dp));
+                mDragHandleProgress =  1 - (dragHandleTop / mShiftRange);
             }
             mTopOffset = dp.getInsets().top - mShelfOffset;
             mShelfTopAtThreshold = mShiftRange * SCRIM_CATCHUP_THRESHOLD + mTopOffset;
@@ -199,8 +203,6 @@
                     mProgress, mMidProgress, 1, mMidAlpha, 0, mBeforeMidProgressColorInterpolator));
             mShelfColor = setColorAlphaBound(mEndScrim, alpha);
         } else {
-            mDragHandleOffset += mShiftRange * (mMidProgress - mProgress);
-
             // Note that these ranges and interpolators are inverted because progress goes 1 to 0.
             int alpha = Math.round(
                     Utilities.mapToRange(mProgress, (float) 0, mMidProgress, (float) mEndAlpha,
@@ -212,6 +214,10 @@
                             (float) 0, LINEAR));
             mRemainingScreenColor = setColorAlphaBound(mScrimColor, remainingScrimAlpha);
         }
+
+        if (mProgress < mDragHandleProgress) {
+            mDragHandleOffset += mShiftRange * (mDragHandleProgress - mProgress);
+        }
     }
 
     @Override