diff --git a/src/com/android/contacts/ContactPhotoManager.java b/src/com/android/contacts/ContactPhotoManager.java
index 4286293..5b6a1ca 100644
--- a/src/com/android/contacts/ContactPhotoManager.java
+++ b/src/com/android/contacts/ContactPhotoManager.java
@@ -582,8 +582,11 @@
             view.setImageBitmap(cachedBitmap);
         }
 
-        // Put the bitmap in the LRU cache
-        mBitmapCache.put(request.getKey(), cachedBitmap);
+        // Put the bitmap in the LRU cache. But only do this for images that are small enough
+        // (we require that at least six of those can be cached at the same time)
+        if (cachedBitmap.getByteCount() < mBitmapCache.maxSize() / 6) {
+            mBitmapCache.put(request.getKey(), cachedBitmap);
+        }
 
         // Soften the reference
         holder.bitmap = null;