diff --git a/src/com/android/settings/bluetooth/BluetoothDetailsProfilesController.java b/src/com/android/settings/bluetooth/BluetoothDetailsProfilesController.java
index b0ed056..ddb6179 100644
--- a/src/com/android/settings/bluetooth/BluetoothDetailsProfilesController.java
+++ b/src/com/android/settings/bluetooth/BluetoothDetailsProfilesController.java
@@ -148,15 +148,12 @@
      */
     private void disableProfile(LocalBluetoothProfile profile, BluetoothDevice device,
             SwitchPreference profilePref) {
-        if (profile instanceof PbapServerProfile) {
-            mCachedDevice.setPhonebookPermissionChoice(CachedBluetoothDevice.ACCESS_REJECTED);
-            // We don't need to do the additional steps below for this profile.
-            return;
-        }
         mCachedDevice.disconnect(profile);
         profile.setPreferred(device, false);
         if (profile instanceof MapProfile) {
             mCachedDevice.setMessagePermissionChoice(BluetoothDevice.ACCESS_REJECTED);
+        } else if (profile instanceof PbapServerProfile) {
+            mCachedDevice.setPhonebookPermissionChoice(CachedBluetoothDevice.ACCESS_REJECTED);
         }
     }