diff --git a/src/com/android/settings/accessibility/AccessibilityActivityPreference.java b/src/com/android/settings/accessibility/AccessibilityActivityPreference.java
index e5568f6..914d9cf 100644
--- a/src/com/android/settings/accessibility/AccessibilityActivityPreference.java
+++ b/src/com/android/settings/accessibility/AccessibilityActivityPreference.java
@@ -50,6 +50,7 @@
     private final AccessibilityShortcutInfo mA11yShortcutInfo;
     private final PackageManager mPm;
     private final ComponentName mComponentName;
+    private final CharSequence mLabel;
     private final ListenableFuture mExtraArgumentsFuture;
 
     public AccessibilityActivityPreference(Context context, String packageName, int uid,
@@ -58,9 +59,10 @@
         mPm = context.getPackageManager();
         mA11yShortcutInfo = a11yShortcutInfo;
         mComponentName = a11yShortcutInfo.getComponentName();
+        mLabel = a11yShortcutInfo.getActivityInfo().loadLabel(mPm);
         // setup basic info for a preference
         setKey(mComponentName.flattenToString());
-        setTitle(a11yShortcutInfo.getActivityInfo().loadLabel(mPm));
+        setTitle(mLabel);
         setSummary(a11yShortcutInfo.loadSummary(mPm));
         setFragment(TARGET_FRAGMENT);
         setIconSize(ICON_SIZE_MEDIUM);
@@ -92,6 +94,13 @@
         super.performClick();
     }
 
+    /**
+     * Returns the label of the Accessibility Activity
+     */
+    public CharSequence getLabel() {
+        return mLabel;
+    }
+
     private Drawable getA11yActivityIcon() {
         ActivityInfo activityInfo = mA11yShortcutInfo.getActivityInfo();
         Drawable serviceIcon;
diff --git a/src/com/android/settings/accessibility/AccessibilitySettings.java b/src/com/android/settings/accessibility/AccessibilitySettings.java
index b5df0e2..8441c2a 100644
--- a/src/com/android/settings/accessibility/AccessibilitySettings.java
+++ b/src/com/android/settings/accessibility/AccessibilitySettings.java
@@ -21,7 +21,6 @@
 import android.app.settings.SettingsEnums;
 import android.content.ComponentName;
 import android.content.Context;
-import android.content.pm.ActivityInfo;
 import android.content.pm.ServiceInfo;
 import android.hardware.input.InputManager;
 import android.os.Bundle;
@@ -30,6 +29,7 @@
 import android.provider.Settings;
 import android.text.TextUtils;
 import android.util.ArrayMap;
+import android.util.Pair;
 import android.view.InputDevice;
 import android.view.accessibility.AccessibilityManager;
 
@@ -57,6 +57,8 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 /** Activity with the accessibility settings. */
 @SearchIndexable(forTarget = SearchIndexable.ALL & ~SearchIndexable.ARC)
@@ -454,17 +456,22 @@
         final List<AccessibilityShortcutInfo> installedShortcutList =
                 a11yManager.getInstalledAccessibilityShortcutListAsUser(context,
                         UserHandle.myUserId());
+        final List<AccessibilityActivityPreference> activityList =
+                preferenceHelper.createAccessibilityActivityPreferenceList(installedShortcutList);
+        final Set<Pair<String, CharSequence>> packageLabelPairs =
+                activityList.stream()
+                        .map(a11yActivityPref -> new Pair<>(
+                                a11yActivityPref.getPackageName(), a11yActivityPref.getLabel())
+                        ).collect(Collectors.toSet());
 
         // Remove duplicate item here, new a ArrayList to copy unmodifiable list result
         // (getInstalledAccessibilityServiceList).
         final List<AccessibilityServiceInfo> installedServiceList = new ArrayList<>(
                 a11yManager.getInstalledAccessibilityServiceList());
-        installedServiceList.removeIf(
-                target -> containsTargetNameInList(installedShortcutList, target));
-
-        final List<RestrictedPreference> activityList =
-                preferenceHelper.createAccessibilityActivityPreferenceList(installedShortcutList);
-
+        if (!packageLabelPairs.isEmpty()) {
+            installedServiceList.removeIf(
+                    target -> containsPackageAndLabelInList(packageLabelPairs, target));
+        }
         final List<RestrictedPreference> serviceList =
                 preferenceHelper.createAccessibilityServicePreferenceList(installedServiceList);
 
@@ -475,22 +482,14 @@
         return preferenceList;
     }
 
-    private boolean containsTargetNameInList(List<AccessibilityShortcutInfo> shortcutInfos,
+    private boolean containsPackageAndLabelInList(
+            Set<Pair<String, CharSequence>> packageLabelPairs,
             AccessibilityServiceInfo targetServiceInfo) {
         final ServiceInfo serviceInfo = targetServiceInfo.getResolveInfo().serviceInfo;
         final String servicePackageName = serviceInfo.packageName;
         final CharSequence serviceLabel = serviceInfo.loadLabel(getPackageManager());
 
-        for (int i = 0, count = shortcutInfos.size(); i < count; ++i) {
-            final ActivityInfo activityInfo = shortcutInfos.get(i).getActivityInfo();
-            final String activityPackageName = activityInfo.packageName;
-            final CharSequence activityLabel = activityInfo.loadLabel(getPackageManager());
-            if (servicePackageName.equals(activityPackageName)
-                    && serviceLabel.equals(activityLabel)) {
-                return true;
-            }
-        }
-        return false;
+        return packageLabelPairs.contains(new Pair<>(servicePackageName, serviceLabel));
     }
 
     private void initializePreBundledServicesMapFromArray(String categoryKey, int key) {
diff --git a/src/com/android/settings/accessibility/RestrictedPreferenceHelper.java b/src/com/android/settings/accessibility/RestrictedPreferenceHelper.java
index 34646ba..2cabc76 100644
--- a/src/com/android/settings/accessibility/RestrictedPreferenceHelper.java
+++ b/src/com/android/settings/accessibility/RestrictedPreferenceHelper.java
@@ -101,13 +101,14 @@
     }
 
     /**
-     * Creates the list of {@link RestrictedPreference} with the installedShortcuts arguments.
+     * Creates the list of {@link AccessibilityActivityPreference} with the installedShortcuts
+     * arguments.
      *
      * @param installedShortcuts The list of {@link AccessibilityShortcutInfo}s of the
      *                           installed accessibility shortcuts
-     * @return The list of {@link RestrictedPreference}
+     * @return The list of {@link AccessibilityActivityPreference}
      */
-    public List<RestrictedPreference> createAccessibilityActivityPreferenceList(
+    public List<AccessibilityActivityPreference> createAccessibilityActivityPreferenceList(
             List<AccessibilityShortcutInfo> installedShortcuts) {
         final Set<ComponentName> enabledServices =
                 AccessibilityUtils.getEnabledServicesFromSettings(mContext);
@@ -115,7 +116,7 @@
                 UserHandle.myUserId());
 
         final int installedShortcutsSize = installedShortcuts.size();
-        final List<RestrictedPreference> preferenceList = new ArrayList<>(
+        final List<AccessibilityActivityPreference> preferenceList = new ArrayList<>(
                 installedShortcutsSize);
 
         for (int i = 0; i < installedShortcutsSize; ++i) {
@@ -124,7 +125,7 @@
             final ComponentName componentName = info.getComponentName();
 
             final boolean serviceEnabled = enabledServices.contains(componentName);
-            RestrictedPreference preference = new AccessibilityActivityPreference(
+            AccessibilityActivityPreference preference = new AccessibilityActivityPreference(
                     mContext, componentName.getPackageName(), activityInfo.applicationInfo.uid,
                     info);
             setRestrictedPreferenceEnabled(preference, permittedServices, serviceEnabled);
diff --git a/tests/robotests/src/com/android/settings/accessibility/RestrictedPreferenceHelperTest.java b/tests/robotests/src/com/android/settings/accessibility/RestrictedPreferenceHelperTest.java
index bc9c1d8..b4f4dc1 100644
--- a/tests/robotests/src/com/android/settings/accessibility/RestrictedPreferenceHelperTest.java
+++ b/tests/robotests/src/com/android/settings/accessibility/RestrictedPreferenceHelperTest.java
@@ -137,7 +137,7 @@
         final List<AccessibilityShortcutInfo> infoList = new ArrayList<>(
                 singletonList(mShortcutInfo));
 
-        final List<RestrictedPreference> preferenceList =
+        final List<AccessibilityActivityPreference> preferenceList =
                 mHelper.createAccessibilityActivityPreferenceList(infoList);
         final RestrictedPreference preference = preferenceList.get(0);