diff --git a/java/com/android/dialer/speeddial/SpeedDialFragment.java b/java/com/android/dialer/speeddial/SpeedDialFragment.java
index 018f978..54709e4 100644
--- a/java/com/android/dialer/speeddial/SpeedDialFragment.java
+++ b/java/com/android/dialer/speeddial/SpeedDialFragment.java
@@ -105,6 +105,10 @@
     LogUtil.enterBlock("SpeedDialFragment.onCreateView");
     View rootLayout = inflater.inflate(R.layout.fragment_speed_dial, container, false);
 
+    speedDialLoaderListener =
+        DialerExecutorComponent.get(getContext())
+            .createUiListener(getChildFragmentManager(), "speed_dial_loader_listener");
+
     // Setup favorite contact context menu
     contextMenu = rootLayout.findViewById(R.id.favorite_contact_context_menu);
     contextMenuBackground = rootLayout.findViewById(R.id.context_menu_background);
@@ -124,7 +128,11 @@
             rootLayout,
             contextMenu,
             contextMenuBackground,
-            new SpeedDialContextMenuItemListener(getActivity(), getChildFragmentManager()),
+            new SpeedDialContextMenuItemListener(
+                getActivity(),
+                getChildFragmentManager(),
+                new UpdateSpeedDialAdapterListener(),
+                speedDialLoaderListener),
             layoutManager);
     adapter =
         new SpeedDialAdapter(getContext(), favoritesListener, suggestedListener, headerListener);
@@ -138,10 +146,6 @@
     ItemTouchHelper touchHelper = new ItemTouchHelper(callback);
     touchHelper.attachToRecyclerView(recyclerView);
     adapter.setItemTouchHelper(touchHelper);
-
-    speedDialLoaderListener =
-        DialerExecutorComponent.get(getContext())
-            .createUiListener(getChildFragmentManager(), "speed_dial_loader_listener");
     return rootLayout;
   }
 
@@ -437,11 +441,18 @@
 
     private final FragmentActivity activity;
     private final FragmentManager childFragmentManager;
+    private final SupportUiListener<ImmutableList<SpeedDialUiItem>> speedDialLoaderListener;
+    private final UpdateSpeedDialAdapterListener updateAdapterListener;
 
     SpeedDialContextMenuItemListener(
-        FragmentActivity activity, FragmentManager childFragmentManager) {
+        FragmentActivity activity,
+        FragmentManager childFragmentManager,
+        UpdateSpeedDialAdapterListener updateAdapterListener,
+        SupportUiListener<ImmutableList<SpeedDialUiItem>> speedDialLoaderListener) {
       this.activity = activity;
       this.childFragmentManager = childFragmentManager;
+      this.updateAdapterListener = updateAdapterListener;
+      this.speedDialLoaderListener = speedDialLoaderListener;
     }
 
     @Override
@@ -472,7 +483,15 @@
 
     @Override
     public void removeFavoriteContact(SpeedDialUiItem speedDialUiItem) {
-      // TODO(calderwoodra): implement remove
+      speedDialLoaderListener.listen(
+          activity,
+          UiItemLoaderComponent.get(activity)
+              .speedDialUiItemMutator()
+              .removeSpeedDialUiItem(speedDialUiItem),
+          updateAdapterListener::updateAdapter,
+          throwable -> {
+            throw new RuntimeException(throwable);
+          });
     }
 
     @Override
@@ -485,6 +504,14 @@
     }
   }
 
+  /** Listener for when a SpeedDialUiItem is updated. */
+  private class UpdateSpeedDialAdapterListener {
+
+    void updateAdapter(ImmutableList<SpeedDialUiItem> speedDialUiItems) {
+      onSpeedDialUiItemListLoaded(speedDialUiItems);
+    }
+  }
+
   /** Interface for {@link SpeedDialFragment} to communicate with its host/parent. */
   public interface HostInterface {
 
diff --git a/java/com/android/dialer/speeddial/loader/SpeedDialUiItemMutator.java b/java/com/android/dialer/speeddial/loader/SpeedDialUiItemMutator.java
index 5dae2ef..e8892c4 100644
--- a/java/com/android/dialer/speeddial/loader/SpeedDialUiItemMutator.java
+++ b/java/com/android/dialer/speeddial/loader/SpeedDialUiItemMutator.java
@@ -54,6 +54,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import javax.inject.Inject;
 import javax.inject.Singleton;
@@ -109,6 +110,85 @@
   }
 
   /**
+   * Delete the SpeedDialUiItem.
+   *
+   * <p>If the item is starred, it's entry will be removed from the SpeedDialEntry database.
+   * Additionally, if the contact only has one entry in the database, it will be unstarred.
+   *
+   * <p>If the item isn't starred, it's usage data will be deleted but the suggestion can come back
+   * if the user calls that contact again.
+   *
+   * @return the updated list of SpeedDialUiItems.
+   */
+  public ListenableFuture<ImmutableList<SpeedDialUiItem>> removeSpeedDialUiItem(
+      SpeedDialUiItem speedDialUiItem) {
+    return dialerFutureSerializer.submit(
+        () -> removeSpeedDialUiItemInternal(speedDialUiItem), backgroundExecutor);
+  }
+
+  @WorkerThread
+  private ImmutableList<SpeedDialUiItem> removeSpeedDialUiItemInternal(
+      SpeedDialUiItem speedDialUiItem) {
+    Assert.isWorkerThread();
+    if (speedDialUiItem.isStarred()) {
+      removeStarredSpeedDialUiItem(speedDialUiItem);
+    } else {
+      removeSuggestedSpeedDialUiItem(speedDialUiItem);
+    }
+    return loadSpeedDialUiItemsInternal();
+  }
+
+  /**
+   * Delete the SpeedDialEntry associated with the passed in SpeedDialUiItem. Additionally, if the
+   * entry being deleted is the only entry for that contact, unstar it in the cp2.
+   */
+  @WorkerThread
+  private void removeStarredSpeedDialUiItem(SpeedDialUiItem speedDialUiItem) {
+    Assert.isWorkerThread();
+    Assert.checkArgument(speedDialUiItem.isStarred());
+    SpeedDialEntryDao db = getSpeedDialEntryDao();
+    ImmutableList<SpeedDialEntry> entries = db.getAllEntries();
+
+    SpeedDialEntry entryToDelete = null;
+    int entriesForTheSameContact = 0;
+    for (SpeedDialEntry entry : entries) {
+      if (entry.contactId() == speedDialUiItem.contactId()) {
+        entriesForTheSameContact++;
+      }
+
+      if (Objects.equals(entry.id(), speedDialUiItem.speedDialEntryId())) {
+        Assert.checkArgument(entryToDelete == null);
+        entryToDelete = entry;
+      }
+    }
+    db.delete(ImmutableList.of(entryToDelete.id()));
+    if (entriesForTheSameContact == 1) {
+      unstarContact(speedDialUiItem);
+    }
+  }
+
+  @WorkerThread
+  private void unstarContact(SpeedDialUiItem speedDialUiItem) {
+    Assert.isWorkerThread();
+    ContentValues contentValues = new ContentValues();
+    contentValues.put(Phone.STARRED, 0);
+    appContext
+        .getContentResolver()
+        .update(
+            Phone.CONTENT_URI,
+            contentValues,
+            Phone.CONTACT_ID + " = ?",
+            new String[] {Long.toString(speedDialUiItem.contactId())});
+  }
+
+  @WorkerThread
+  @SuppressWarnings("unused")
+  private void removeSuggestedSpeedDialUiItem(SpeedDialUiItem speedDialUiItem) {
+    Assert.isWorkerThread();
+    // TODO(calderwoodra): remove strequent contact
+  }
+
+  /**
    * Takes a contact uri from {@link Phone#CONTENT_URI} and updates {@link Phone#STARRED} to be
    * true, if it isn't already or Inserts the contact into the {@link SpeedDialEntryDatabaseHelper}
    */