diff --git a/src/com/android/contacts/model/EntityDeltaList.java b/src/com/android/contacts/model/EntityDeltaList.java
index e68b6ef..455cced 100644
--- a/src/com/android/contacts/model/EntityDeltaList.java
+++ b/src/com/android/contacts/model/EntityDeltaList.java
@@ -141,7 +141,9 @@
         // Second pass builds actual operations
         for (EntityDelta delta : this) {
             final int firstBatch = diff.size();
-            backRefs[rawContactIndex++] = firstBatch;
+            final boolean isInsert = delta.isContactInsert();
+            backRefs[rawContactIndex++] = isInsert ? firstBatch : -1;
+
             delta.buildDiff(diff);
 
             // If the user chose to join with some other existing raw contact(s) at save time,
@@ -161,7 +163,7 @@
             }
 
             // Only create rules for inserts
-            if (!delta.isContactInsert()) continue;
+            if (!isInsert) continue;
 
             // If we are going to split all contacts, there is no point in first combining them
             if (mSplitRawContacts) continue;
@@ -236,18 +238,25 @@
         builder.withValue(AggregationExceptions.TYPE, AggregationExceptions.TYPE_KEEP_SEPARATE);
 
         Long rawContactId1 = get(index1).getValues().getAsLong(RawContacts._ID);
+        int backRef1 = backRefs[index1];
         if (rawContactId1 != null && rawContactId1 >= 0) {
             builder.withValue(AggregationExceptions.RAW_CONTACT_ID1, rawContactId1);
+        } else if (backRef1 >= 0) {
+            builder.withValueBackReference(AggregationExceptions.RAW_CONTACT_ID1, backRef1);
         } else {
-            builder.withValueBackReference(AggregationExceptions.RAW_CONTACT_ID1, backRefs[index1]);
+            return;
         }
 
         Long rawContactId2 = get(index2).getValues().getAsLong(RawContacts._ID);
+        int backRef2 = backRefs[index2];
         if (rawContactId2 != null && rawContactId2 >= 0) {
             builder.withValue(AggregationExceptions.RAW_CONTACT_ID2, rawContactId2);
+        } else if (backRef2 >= 0) {
+            builder.withValueBackReference(AggregationExceptions.RAW_CONTACT_ID2, backRef2);
         } else {
-            builder.withValueBackReference(AggregationExceptions.RAW_CONTACT_ID2, backRefs[index2]);
+            return;
         }
+
         diff.add(builder.build());
     }