diff --git a/src/com/android/launcher3/WidgetPreviewLoader.java b/src/com/android/launcher3/WidgetPreviewLoader.java
index 1fb795a..3d50587 100644
--- a/src/com/android/launcher3/WidgetPreviewLoader.java
+++ b/src/com/android/launcher3/WidgetPreviewLoader.java
@@ -212,7 +212,6 @@
     public void removeObsoletePreviews(ArrayList<Object> list) {
         Utilities.assertWorkerThread();
 
-        LongSparseArray<UserHandleCompat> userIdCache = new LongSparseArray<>();
         LongSparseArray<HashSet<String>> validPackages = new LongSparseArray<>();
 
         for (Object obj : list) {
@@ -227,15 +226,7 @@
                 pkg = info.provider.getPackageName();
             }
 
-            int userIdIndex = userIdCache.indexOfValue(user);
-            final long userId;
-            if (userIdIndex < 0) {
-                userId = mUserManager.getSerialNumberForUser(user);
-                userIdCache.put(userId, user);
-            } else {
-                userId = userIdCache.keyAt(userIdIndex);
-            }
-
+            final long userId = mUserManager.getSerialNumberForUser(user);
             HashSet<String> packages = validPackages.get(userId);
             if (packages == null) {
                 packages = new HashSet<>();
diff --git a/src/com/android/launcher3/compat/UserManagerCompatV17.java b/src/com/android/launcher3/compat/UserManagerCompatV17.java
index 1687569..75203b7 100644
--- a/src/com/android/launcher3/compat/UserManagerCompatV17.java
+++ b/src/com/android/launcher3/compat/UserManagerCompatV17.java
@@ -23,10 +23,16 @@
 
 import com.android.launcher3.util.LongArrayMap;
 
+import java.util.HashMap;
+
 @TargetApi(Build.VERSION_CODES.JELLY_BEAN_MR1)
 public class UserManagerCompatV17 extends UserManagerCompatV16 {
 
     protected LongArrayMap<UserHandleCompat> mUsers;
+    // Create a separate reverse map as LongArrayMap.indexOfValue checks if objects are same
+    // and not {@link Object#equals}
+    protected HashMap<UserHandleCompat, Long> mUserToSerialMap;
+
     protected UserManager mUserManager;
 
     UserManagerCompatV17(Context context) {
@@ -35,9 +41,9 @@
 
     public long getSerialNumberForUser(UserHandleCompat user) {
         synchronized (this) {
-            if (mUsers != null) {
-                int index = mUsers.indexOfValue(user);
-                return (index >= 0) ? mUsers.keyAt(index) : 0;
+            if (mUserToSerialMap != null) {
+                Long serial = mUserToSerialMap.get(user);
+                return serial == null ? 0 : serial;
             }
         }
         return mUserManager.getSerialNumberForUser(user.getUser());
@@ -55,9 +61,12 @@
     @Override
     public void enableAndResetCache() {
         synchronized (this) {
-            mUsers = new LongArrayMap<UserHandleCompat>();
+            mUsers = new LongArrayMap<>();
+            mUserToSerialMap = new HashMap<>();
             UserHandleCompat myUser = UserHandleCompat.myUserHandle();
-            mUsers.put(mUserManager.getSerialNumberForUser(myUser.getUser()), myUser);
+            long serial = mUserManager.getSerialNumberForUser(myUser.getUser());
+            mUsers.put(serial, myUser);
+            mUserToSerialMap.put(myUser, serial);
         }
     }
 }
diff --git a/src/com/android/launcher3/compat/UserManagerCompatVL.java b/src/com/android/launcher3/compat/UserManagerCompatVL.java
index 04cd0fd..4d404db 100644
--- a/src/com/android/launcher3/compat/UserManagerCompatVL.java
+++ b/src/com/android/launcher3/compat/UserManagerCompatVL.java
@@ -30,6 +30,7 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 
 @TargetApi(Build.VERSION_CODES.LOLLIPOP)
@@ -48,12 +49,15 @@
     @Override
     public void enableAndResetCache() {
         synchronized (this) {
-            mUsers = new LongArrayMap<UserHandleCompat>();
+            mUsers = new LongArrayMap<>();
+            mUserToSerialMap = new HashMap<>();
             List<UserHandle> users = mUserManager.getUserProfiles();
             if (users != null) {
                 for (UserHandle user : users) {
-                    mUsers.put(mUserManager.getSerialNumberForUser(user),
-                            UserHandleCompat.fromUser(user));
+                    long serial = mUserManager.getSerialNumberForUser(user);
+                    UserHandleCompat userCompat = UserHandleCompat.fromUser(user);
+                    mUsers.put(serial, userCompat);
+                    mUserToSerialMap.put(userCompat, serial);
                 }
             }
         }
@@ -64,9 +68,7 @@
         synchronized (this) {
             if (mUsers != null) {
                 List<UserHandleCompat> users = new ArrayList<>();
-                for (UserHandleCompat user : mUsers) {
-                    users.add(user);
-                }
+                users.addAll(mUserToSerialMap.keySet());
                 return users;
             }
         }