diff --git a/src/com/android/settings/localepicker/LocaleDragAndDropAdapter.java b/src/com/android/settings/localepicker/LocaleDragAndDropAdapter.java
index bc17814..f130150 100644
--- a/src/com/android/settings/localepicker/LocaleDragAndDropAdapter.java
+++ b/src/com/android/settings/localepicker/LocaleDragAndDropAdapter.java
@@ -197,6 +197,8 @@
         notifyItemChanged(fromPosition); // to update the numbers
         notifyItemChanged(toPosition);
         notifyItemMoved(fromPosition, toPosition);
+        // We don't call doTheUpdate() here because this method is called for each item swap.
+        // So if we drag something across several positions it will be called several times.
     }
 
     void setRemoveMode(boolean removeMode) {
@@ -222,7 +224,6 @@
         }
         mFeedItemList.remove(position);
         notifyDataSetChanged();
-        doTheUpdate();
     }
 
     void removeChecked() {
diff --git a/src/com/android/settings/localepicker/LocaleLinearLayoutManager.java b/src/com/android/settings/localepicker/LocaleLinearLayoutManager.java
index 630f382..2917cc7 100644
--- a/src/com/android/settings/localepicker/LocaleLinearLayoutManager.java
+++ b/src/com/android/settings/localepicker/LocaleLinearLayoutManager.java
@@ -114,40 +114,46 @@
 
         final int itemCount = this.getItemCount();
         final int position = this.getPosition(host);
+        boolean result = false;
 
         switch (action) {
             case R.id.action_drag_move_up:
                 if (position > 0) {
                     mAdapter.onItemMove(position, position - 1);
-                    return true;
+                    result = true;
                 }
-                return false;
+                break;
             case R.id.action_drag_move_down:
                 if (position + 1 < itemCount) {
                     mAdapter.onItemMove(position, position + 1);
-                    return true;
+                    result = true;
                 }
-                return false;
+                break;
             case R.id.action_drag_move_top:
                 if (position != 0) {
                     mAdapter.onItemMove(position, 0);
-                    return true;
+                    result = true;
                 }
-                return false;
+                break;
             case R.id.action_drag_move_bottom:
                 if (position != itemCount - 1) {
                     mAdapter.onItemMove(position, itemCount - 1);
-                    return true;
+                    result = true;
                 }
-                return false;
+                break;
             case R.id.action_drag_remove:
                 if (itemCount > 1) {
                     mAdapter.removeItem(position);
-                    return true;
+                    result = true;
                 }
-                return false;
+                break;
             default:
                 return super.performAccessibilityActionForItem(recycler, state, host, action, args);
         }
+
+        if (result) {
+            mAdapter.doTheUpdate();
+        }
+        return result;
     }
 }