diff --git a/dictionaries/en_GB_wordlist.combined.gz b/dictionaries/en_GB_wordlist.combined.gz
index 839f3ef..4f008ed 100644
--- a/dictionaries/en_GB_wordlist.combined.gz
+++ b/dictionaries/en_GB_wordlist.combined.gz
Binary files differ
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
index 506dfa7..9f9fdaa 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
@@ -178,8 +178,6 @@
             if (!state.mIsAlphabetShiftLocked) {
                 setShifted(state.mShiftMode);
             }
-            // TODO: is this the right place to do this? Should we do this in setShift* instead?
-            mSwitchActions.requestUpdatingShiftState();
         } else {
             mPrevMainKeyboardWasShiftLocked = state.mIsAlphabetShiftLocked;
         }
diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
index 4a0ce37..463d093 100644
--- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
@@ -41,8 +41,17 @@
 
     abstract public void clear();
 
+    /**
+     * Add a unigram with an optional shortcut to the dictionary.
+     * @param word The word to add.
+     * @param shortcutTarget A shortcut target for this word, or null if none.
+     * @param frequency The frequency for this unigram.
+     * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored
+     *   if shortcutTarget is null.
+     * @param isNotAWord true if this is not a word, i.e. shortcut only.
+     */
     abstract public void addUnigramWord(final String word, final String shortcutTarget,
-            final int frequency, final boolean isNotAWord);
+            final int frequency, final int shortcutFreq, final boolean isNotAWord);
 
     // TODO: Remove lastModifiedTime after making binary dictionary support forgetting curve.
     abstract public void addBigramWords(final String word0, final String word1,
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index 541e697..fd29698 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -52,6 +52,10 @@
     public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
     @UsedForTesting
     public static final String BIGRAM_COUNT_QUERY = "BIGRAM_COUNT";
+    @UsedForTesting
+    public static final String MAX_UNIGRAM_COUNT_QUERY = "MAX_UNIGRAM_COUNT";
+    @UsedForTesting
+    public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT";
 
     private long mNativeDict;
     private final Locale mLocale;
diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
index ffeb927..47891c6 100644
--- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
@@ -127,7 +127,7 @@
             if (DEBUG) {
                 Log.d(TAG, "loadAccountVocabulary: " + word);
             }
-            super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS,
+            super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS, 0 /* shortcutFreq */,
                     false /* isNotAWord */);
         }
     }
@@ -213,7 +213,7 @@
                         Log.d(TAG, "addName " + name + ", " + word + ", " + prevWord);
                     }
                     super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS,
-                            false /* isNotAWord */);
+                            0 /* shortcutFreq */, false /* isNotAWord */);
                     if (!TextUtils.isEmpty(prevWord)) {
                         if (mUseFirstLastBigrams) {
                             super.addBigram(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM,
diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
index 84abfa6..3df2a2b 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
@@ -62,13 +62,13 @@
     // considering performance regression.
     @Override
     public void addUnigramWord(final String word, final String shortcutTarget, final int frequency,
-            final boolean isNotAWord) {
+            final int shortcutFreq, final boolean isNotAWord) {
         if (shortcutTarget == null) {
             mFusionDictionary.add(word, frequency, null, isNotAWord);
         } else {
             // TODO: Do this in the subclass, with this class taking an arraylist.
             final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList();
-            shortcutTargets.add(new WeightedString(shortcutTarget, frequency));
+            shortcutTargets.add(new WeightedString(shortcutTarget, shortcutFreq));
             mFusionDictionary.add(word, frequency, shortcutTargets, isNotAWord);
         }
     }
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index c79a4ff..eb8650e 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -261,10 +261,16 @@
 
     /**
      * Adds a word unigram to the dictionary. Used for loading a dictionary.
+     * @param word The word to add.
+     * @param shortcutTarget A shortcut target for this word, or null if none.
+     * @param frequency The frequency for this unigram.
+     * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored
+     *   if shortcutTarget is null.
+     * @param isNotAWord true if this is not a word, i.e. shortcut only.
      */
     protected void addWord(final String word, final String shortcutTarget,
-            final int frequency, final boolean isNotAWord) {
-        mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
+            final int frequency, final int shortcutFreq, final boolean isNotAWord) {
+        mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, shortcutFreq, isNotAWord);
     }
 
     /**
@@ -313,7 +319,7 @@
      * Dynamically adds a word unigram to the dictionary. May overwrite an existing entry.
      */
     protected void addWordDynamically(final String word, final String shortcutTarget,
-            final int frequency, final boolean isNotAWord) {
+            final int frequency, final int shortcutFreq, final boolean isNotAWord) {
         if (!mIsUpdatable) {
             Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mFilename);
             return;
@@ -326,7 +332,8 @@
                     mBinaryDictionary.addUnigramWord(word, frequency);
                 } else {
                     // TODO: Remove.
-                    mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
+                    mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, shortcutFreq,
+                            isNotAWord);
                 }
             }
         });
diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java
index d491f98..95c9bca 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java
@@ -156,15 +156,36 @@
         return Constants.DICTIONARY_MAX_WORD_LENGTH;
     }
 
-    public void addWord(final String word, final String shortcutTarget, final int frequency) {
+    /**
+     * Add a word with an optional shortcut to the dictionary.
+     * @param word The word to add.
+     * @param shortcutTarget A shortcut target for this word, or null if none.
+     * @param frequency The frequency for this unigram.
+     * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored
+     *   if shortcutTarget is null.
+     */
+    public void addWord(final String word, final String shortcutTarget, final int frequency,
+            final int shortcutFreq) {
         if (word.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH) {
             return;
         }
-        addWordRec(mRoots, word, 0, shortcutTarget, frequency, null);
+        addWordRec(mRoots, word, 0, shortcutTarget, frequency, shortcutFreq, null);
     }
 
+    /**
+     * Add a word, recursively searching for its correct place in the trie tree.
+     * @param children The node to recursively search for addition. Initially, the root of the tree.
+     * @param word The word to add.
+     * @param depth The current depth in the tree.
+     * @param shortcutTarget A shortcut target for this word, or null if none.
+     * @param frequency The frequency for this unigram.
+     * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored
+     *   if shortcutTarget is null.
+     * @param parentNode The parent node, for up linking. Initially null, as the root has no parent.
+     */
     private void addWordRec(final NodeArray children, final String word, final int depth,
-            final String shortcutTarget, final int frequency, final Node parentNode) {
+            final String shortcutTarget, final int frequency, final int shortcutFreq,
+            final Node parentNode) {
         final int wordLength = word.length();
         if (wordLength <= depth) return;
         final char c = word.charAt(depth);
@@ -204,7 +225,8 @@
         if (childNode.mChildren == null) {
             childNode.mChildren = new NodeArray();
         }
-        addWordRec(childNode.mChildren, word, depth + 1, shortcutTarget, frequency, childNode);
+        addWordRec(childNode.mChildren, word, depth + 1, shortcutTarget, frequency, shortcutFreq,
+                childNode);
     }
 
     @Override
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index d51c63d..0e93590 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -808,6 +808,7 @@
     @SuppressWarnings("deprecation")
     private void onStartInputViewInternal(final EditorInfo editorInfo, final boolean restarting) {
         super.onStartInputView(editorInfo, restarting);
+        mRichImm.clearSubtypeCaches();
         final KeyboardSwitcher switcher = mKeyboardSwitcher;
         final MainKeyboardView mainKeyboardView = switcher.getMainKeyboardView();
         // If we are starting input in a different text field from before, we'll have to reload
@@ -903,12 +904,17 @@
         // Sometimes, while rotating, for some reason the framework tells the app we are not
         // connected to it and that means we can't refresh the cache. In this case, schedule a
         // refresh later.
+        final boolean canReachInputConnection;
         if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(editorInfo.initialSelStart,
                 false /* shouldFinishComposition */)) {
             // We try resetting the caches up to 5 times before giving up.
             mHandler.postResetCaches(isDifferentTextField, 5 /* remainingTries */);
+            canReachInputConnection = false;
         } else {
-            if (isDifferentTextField) mHandler.postResumeSuggestions();
+            if (isDifferentTextField) {
+                mHandler.postResumeSuggestions();
+            }
+            canReachInputConnection = true;
         }
 
         if (isDifferentTextField) {
@@ -920,7 +926,11 @@
                 suggest.setAutoCorrectionThreshold(currentSettingsValues.mAutoCorrectionThreshold);
             }
 
-            switcher.loadKeyboard(editorInfo, currentSettingsValues);
+            if (canReachInputConnection) {
+                // If we can't reach the input connection, we don't want to call loadKeyboard yet.
+                // It will be done in #retryResetCaches.
+                switcher.loadKeyboard(editorInfo, currentSettingsValues);
+            }
         } else if (restarting) {
             // TODO: Come up with a more comprehensive way to reset the keyboard layout when
             // a keyboard layout set doesn't get reloaded in this method.
@@ -1049,7 +1059,6 @@
         // Should do the following in onFinishInputInternal but until JB MR2 it's not called :(
         if (mWordComposer.isComposingWord()) mConnection.finishComposingText();
         resetComposingState(true /* alsoResetLastComposedWord */);
-        mRichImm.clearSubtypeCaches();
         // Notify ResearchLogger
         if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
             ResearchLogger.latinIME_onFinishInputViewInternal(finishingInput, mLastSelectionStart,
@@ -2955,11 +2964,13 @@
         if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(mLastSelectionStart, false)) {
             if (0 < remainingTries) {
                 mHandler.postResetCaches(tryResumeSuggestions, remainingTries - 1);
+                return;
             }
-            return;
+            // If remainingTries is 0, we should stop waiting for new tries, but it's still
+            // better to load the keyboard (less things will be broken).
         }
         tryFixLyingCursorPosition();
-        mKeyboardSwitcher.updateShiftState();
+        mKeyboardSwitcher.loadKeyboard(getCurrentInputEditorInfo(), mSettings.getCurrent());
         if (tryResumeSuggestions) mHandler.postResumeSuggestions();
     }
 
diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java
index 9fd1f53..c270d47 100644
--- a/java/src/com/android/inputmethod/latin/Suggest.java
+++ b/java/src/com/android/inputmethod/latin/Suggest.java
@@ -286,14 +286,16 @@
         // the word *would* have been auto-corrected.
         if (!isCorrectionEnabled || !allowsToBeAutoCorrected || !wordComposer.isComposingWord()
                 || suggestionsSet.isEmpty() || wordComposer.hasDigits()
-                || wordComposer.isMostlyCaps() || wordComposer.isResumed()
-                || !hasMainDictionary()) {
+                || wordComposer.isMostlyCaps() || wordComposer.isResumed() || !hasMainDictionary()
+                || SuggestedWordInfo.KIND_SHORTCUT == suggestionsSet.first().mKind) {
             // If we don't have a main dictionary, we never want to auto-correct. The reason for
             // this is, the user may have a contact whose name happens to match a valid word in
             // their language, and it will unexpectedly auto-correct. For example, if the user
             // types in English with no dictionary and has a "Will" in their contact list, "will"
             // would always auto-correct to "Will" which is unwanted. Hence, no main dict => no
             // auto-correct.
+            // Also, shortcuts should never auto-correct unless they are whitelist entries.
+            // TODO: we may want to have shortcut-only entries auto-correct in the future.
             hasAutoCorrection = false;
         } else {
             hasAutoCorrection = AutoCorrectionUtils.suggestionExceedsAutoCorrectionThreshold(
diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
index 864a173..15b3d8d 100644
--- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
@@ -47,6 +47,9 @@
     private static final String USER_DICTIONARY_ALL_LANGUAGES = "";
     private static final int HISTORICAL_DEFAULT_USER_DICTIONARY_FREQUENCY = 250;
     private static final int LATINIME_DEFAULT_USER_DICTIONARY_FREQUENCY = 160;
+    // Shortcut frequency is 0~15, with 15 = whitelist. We don't want user dictionary entries
+    // to auto-correct, so we set this to the highest frequency that won't, i.e. 14.
+    private static final int USER_DICT_SHORTCUT_FREQUENCY = 14;
 
     // TODO: use Words.SHORTCUT when we target JellyBean or above
     final static String SHORTCUT = "shortcut";
@@ -243,10 +246,12 @@
                 final int adjustedFrequency = scaleFrequencyFromDefaultToLatinIme(frequency);
                 // Safeguard against adding really long words.
                 if (word.length() < MAX_WORD_LENGTH) {
-                    super.addWord(word, null, adjustedFrequency, false /* isNotAWord */);
+                    super.addWord(word, null, adjustedFrequency, 0 /* shortcutFreq */,
+                            false /* isNotAWord */);
                 }
                 if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) {
-                    super.addWord(shortcut, word, adjustedFrequency, true /* isNotAWord */);
+                    super.addWord(shortcut, word, adjustedFrequency, USER_DICT_SHORTCUT_FREQUENCY,
+                            true /* isNotAWord */);
                 }
                 cursor.moveToNext();
             }
diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
index be653fe..3bb218b 100644
--- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
+++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
@@ -367,10 +367,11 @@
      * Helper method to convert a String to an int array.
      */
     static int[] getCodePoints(final String word) {
-        // TODO: this is a copy-paste of the contents of StringUtils.toCodePointArray,
+        // TODO: this is a copy-paste of the old contents of StringUtils.toCodePointArray,
         // which is not visible from the makedict package. Factor this code.
+        final int length = word.length();
+        if (length <= 0) return new int[] {};
         final char[] characters = word.toCharArray();
-        final int length = characters.length;
         final int[] codePoints = new int[Character.codePointCount(characters, 0, length)];
         int codePoint = Character.codePointAt(characters, 0);
         int dsti = 0;
diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
index c8b62b6..a1e3600 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
@@ -138,7 +138,7 @@
         final int frequency = ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
                 (isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS) :
                         FREQUENCY_FOR_TYPED;
-        addWordDynamically(word1, null /* the "shortcut" parameter is null */, frequency,
+        addWordDynamically(word1, null /* shortcutTarget */, frequency, 0 /* shortcutFreq */,
                 false /* isNotAWord */);
         // Do not insert a word as a bigram of itself
         if (word1.equals(word0)) {
@@ -171,11 +171,11 @@
         final OnAddWordListener listener = new OnAddWordListener() {
             @Override
             public void setUnigram(final String word, final String shortcutTarget,
-                    final int frequency) {
+                    final int frequency, final int shortcutFreq) {
                 if (DBG_SAVE_RESTORE) {
                     Log.d(TAG, "load unigram: " + word + "," + frequency);
                 }
-                addWord(word, shortcutTarget, frequency, false /* isNotAWord */);
+                addWord(word, shortcutTarget, frequency, shortcutFreq, false /* isNotAWord */);
                 ++profTotalCount[0];
             }
 
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
index 039b253..6f152bb 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
@@ -75,15 +75,21 @@
     /**
      * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes
      * are done to update the binary dictionary.
+     * @param word The word to add.
+     * @param shortcutTarget A shortcut target for this word, or null if none.
+     * @param frequency The frequency for this unigram.
+     * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored
+     *   if shortcutTarget is null.
+     * @param isNotAWord true if this is not a word, i.e. shortcut only.
      */
     @Override
     public void addUnigramWord(final String word, final String shortcutTarget, final int frequency,
-            final boolean isNotAWord) {
+            final int shortcutFreq, final boolean isNotAWord) {
         if (mBigramList.size() > mMaxHistoryBigrams * 2) {
             // Too many entries: just stop adding new vocabulary and wait next refresh.
             return;
         }
-        mExpandableDictionary.addWord(word, shortcutTarget, frequency);
+        mExpandableDictionary.addWord(word, shortcutTarget, frequency, shortcutFreq);
         mBigramList.addBigram(null, word, (byte)frequency);
     }
 
diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
index ea32a74..635afe7 100644
--- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
@@ -49,7 +49,16 @@
     private static final String LAST_UPDATED_TIME_KEY = "date";
 
     public interface OnAddWordListener {
-        public void setUnigram(final String word, final String shortcutTarget, final int frequency);
+        /**
+         * Callback to be notified when a word is added to the dictionary.
+         * @param word The added word.
+         * @param shortcutTarget A shortcut target for this word, or null if none.
+         * @param frequency The frequency for this word.
+         * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist).
+         *   Unspecified if shortcutTarget is null - do not rely on its value.
+         */
+        public void setUnigram(final String word, final String shortcutTarget, final int frequency,
+                final int shortcutFreq);
         public void setBigram(final String word1, final String word2, final int frequency);
     }
 
@@ -153,7 +162,7 @@
         for (Entry<Integer, String> entry : unigrams.entrySet()) {
             final String word1 = entry.getValue();
             final int unigramFrequency = frequencies.get(entry.getKey());
-            to.setUnigram(word1, null, unigramFrequency);
+            to.setUnigram(word1, null /* shortcutTarget */, unigramFrequency, 0 /* shortcutFreq */);
             final ArrayList<PendingAttribute> attrList = bigrams.get(entry.getKey());
             if (attrList != null) {
                 for (final PendingAttribute attr : attrList) {
diff --git a/native/jni/src/suggest/core/dictionary/shortcut_utils.h b/native/jni/src/suggest/core/dictionary/shortcut_utils.h
index 461d7b4..9ccef02 100644
--- a/native/jni/src/suggest/core/dictionary/shortcut_utils.h
+++ b/native/jni/src/suggest/core/dictionary/shortcut_utils.h
@@ -44,7 +44,7 @@
                 shortcutScore = finalScore;
                 // Protection against int underflow
                 shortcutScore = max(S_INT_MIN + 1, shortcutScore) - 1;
-                kind = Dictionary::KIND_CORRECTION;
+                kind = Dictionary::KIND_SHORTCUT;
             }
             outputTypes[outputWordIndex] = kind;
             frequencies[outputWordIndex] = shortcutScore;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
index a17a0ac..5724c5d 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
@@ -39,7 +39,7 @@
             return false;
         }
         if (!ForgettingCurveUtils::isValidEncodedProbability(newProbability)) {
-            isUselessPtNode = false;
+            isUselessPtNode = true;
         }
     }
     if (mChildrenValue > 0) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
index 3ca2f2a..9755120 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
@@ -60,6 +60,7 @@
 
         bool onDescend(const int ptNodeArrayPos) {
             mValueStack.push_back(0);
+            mChildrenValue = 0;
             return true;
         }
 
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
index 31e3fb4..3d07c9d 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
@@ -37,6 +37,8 @@
 // BinaryDictionaryDecayingTests.
 const char *const DynamicPatriciaTriePolicy::UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
 const char *const DynamicPatriciaTriePolicy::BIGRAM_COUNT_QUERY = "BIGRAM_COUNT";
+const char *const DynamicPatriciaTriePolicy::MAX_UNIGRAM_COUNT_QUERY = "MAX_UNIGRAM_COUNT";
+const char *const DynamicPatriciaTriePolicy::MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT";
 const char *const DynamicPatriciaTriePolicy::SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY =
         "SET_NEEDS_TO_DECAY_FOR_TESTING";
 const int DynamicPatriciaTriePolicy::MAX_DICT_EXTENDED_REGION_SIZE = 1024 * 1024;
@@ -355,6 +357,14 @@
         snprintf(outResult, maxResultLength, "%d", mUnigramCount);
     } else if (strncmp(query, BIGRAM_COUNT_QUERY, maxResultLength) == 0) {
         snprintf(outResult, maxResultLength, "%d", mBigramCount);
+    } else if (strncmp(query, MAX_UNIGRAM_COUNT_QUERY, maxResultLength) == 0) {
+        snprintf(outResult, maxResultLength, "%d",
+                mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_UNIGRAM_COUNT :
+                        DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE);
+    } else if (strncmp(query, MAX_BIGRAM_COUNT_QUERY, maxResultLength) == 0) {
+        snprintf(outResult, maxResultLength, "%d",
+                mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT :
+                        DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE);
     } else if (strncmp(query, SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY, maxResultLength) == 0) {
         mNeedsToDecayForTesting = true;
     }
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
index 903f65e..be97ee1 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
@@ -102,6 +102,8 @@
 
     static const char *const UNIGRAM_COUNT_QUERY;
     static const char *const BIGRAM_COUNT_QUERY;
+    static const char *const MAX_UNIGRAM_COUNT_QUERY;
+    static const char *const MAX_BIGRAM_COUNT_QUERY;
     static const char *const SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY;
     static const int MAX_DICT_EXTENDED_REGION_SIZE;
     static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp
index 601ee66..f108c21 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp
@@ -93,6 +93,12 @@
     if (!listener->onDescend(getPosOfLastPtNodeArrayHead())) {
         return false;
     }
+    if (isEnd()) {
+        // Empty dictionary. Needs to notify the listener of the tail of empty PtNode array.
+        if (!listener->onReadingPtNodeArrayTail()) {
+            return false;
+        }
+    }
     pushReadingStateToStack();
     while (!isEnd()) {
         if (alreadyVisitedAllPtNodesInArray) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h
index 512a4d8..a71c069 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h
@@ -279,7 +279,9 @@
         } else {
             mReadingState = mReadingStateStack.back();
             mReadingStateStack.pop_back();
-            fetchPtNodeInfo();
+            if (!isEnd()) {
+                fetchPtNodeInfo();
+            }
         }
     }
 };
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
index 19ca354..1632fd0 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
@@ -93,8 +93,7 @@
     for (int i = 0; i < decayIterationCount; ++i) {
         const float currentRate = static_cast<float>(currentEncodedProbability)
                 / static_cast<float>(MAX_ENCODED_PROBABILITY);
-        const float thresholdToDecay = MIN_PROBABILITY_TO_DECAY
-                + (1.0f - MIN_PROBABILITY_TO_DECAY) * currentRate;
+        const float thresholdToDecay = (1.0f - MIN_PROBABILITY_TO_DECAY) * currentRate;
         const float randValue = static_cast<float>(rand()) / static_cast<float>(RAND_MAX);
         if (thresholdToDecay < randValue) {
             currentEncodedProbability = max(currentEncodedProbability - ENCODED_PROBABILITY_STEP,
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
index ded8eaa..cecdd2f 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
@@ -19,13 +19,16 @@
 import android.test.AndroidTestCase;
 import android.test.suitebuilder.annotation.LargeTest;
 
+import com.android.inputmethod.latin.makedict.CodePointUtils;
 import com.android.inputmethod.latin.makedict.FormatSpec;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Random;
 
 @LargeTest
 public class BinaryDictionaryDecayingTests extends AndroidTestCase {
@@ -179,4 +182,55 @@
         binaryDictionary.close();
         dictFile.delete();
     }
+
+    public void testAddManyUnigramsToDecayingDict() {
+        final int unigramCount = 30000;
+        final int unigramTypedCount = 100000;
+        final int codePointSetSize = 50;
+        final long seed = System.currentTimeMillis();
+        final Random random = new Random(seed);
+
+        File dictFile = null;
+        try {
+            dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+        } catch (IOException e) {
+            fail("IOException while writing an initial dictionary : " + e);
+        }
+        BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+                0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+                Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+
+        final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
+        final ArrayList<String> words = new ArrayList<String>();
+
+        for (int i = 0; i < unigramCount; i++) {
+            final String word = CodePointUtils.generateWord(random, codePointSet);
+            words.add(word);
+        }
+
+        final int maxUnigramCount = Integer.parseInt(
+                binaryDictionary.getPropertyForTests(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY));
+        for (int i = 0; i < unigramTypedCount; i++) {
+            final String word = words.get(random.nextInt(words.size()));
+            binaryDictionary.addUnigramWord(word, DUMMY_PROBABILITY);
+
+            if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
+                final int unigramCountBeforeGC =
+                        Integer.parseInt(binaryDictionary.getPropertyForTests(
+                                BinaryDictionary.UNIGRAM_COUNT_QUERY));
+                while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
+                    binaryDictionary.flushWithGC();
+                }
+                final int unigramCountAfterGC =
+                        Integer.parseInt(binaryDictionary.getPropertyForTests(
+                                BinaryDictionary.UNIGRAM_COUNT_QUERY));
+                assertTrue(unigramCountBeforeGC > unigramCountAfterGC);
+            }
+        }
+
+        assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTests(
+                BinaryDictionary.UNIGRAM_COUNT_QUERY)) > 0);
+        assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTests(
+                BinaryDictionary.UNIGRAM_COUNT_QUERY)) <= maxUnigramCount);
+    }
 }
diff --git a/tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java b/tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java
index ecf3af7..6aae104 100644
--- a/tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java
@@ -26,13 +26,16 @@
 public class ExpandableDictionaryTests extends AndroidTestCase {
 
     private final static int UNIGRAM_FREQ = 50;
+    // See UserBinaryDictionary for more information about this variable.
+    // For tests, its actual value does not matter.
+    private final static int SHORTCUT_FREQ = 14;
 
     public void testAddWordAndGetWordFrequency() {
         final ExpandableDictionary dict = new ExpandableDictionary(Dictionary.TYPE_USER);
 
         // Add words
-        dict.addWord("abcde", "abcde", UNIGRAM_FREQ);
-        dict.addWord("abcef", null, UNIGRAM_FREQ + 1);
+        dict.addWord("abcde", "abcde", UNIGRAM_FREQ, SHORTCUT_FREQ);
+        dict.addWord("abcef", null, UNIGRAM_FREQ + 1, 0);
 
         // Check words
         assertFalse(dict.isValidWord("abcde"));
@@ -40,16 +43,16 @@
         assertTrue(dict.isValidWord("abcef"));
         assertEquals(UNIGRAM_FREQ+1, dict.getWordFrequency("abcef"));
 
-        dict.addWord("abc", null, UNIGRAM_FREQ + 2);
+        dict.addWord("abc", null, UNIGRAM_FREQ + 2, 0);
         assertTrue(dict.isValidWord("abc"));
         assertEquals(UNIGRAM_FREQ + 2, dict.getWordFrequency("abc"));
 
         // Add existing word with lower frequency
-        dict.addWord("abc", null, UNIGRAM_FREQ);
+        dict.addWord("abc", null, UNIGRAM_FREQ, 0);
         assertEquals(UNIGRAM_FREQ + 2, dict.getWordFrequency("abc"));
 
         // Add existing word with higher frequency
-        dict.addWord("abc", null, UNIGRAM_FREQ + 3);
+        dict.addWord("abc", null, UNIGRAM_FREQ + 3, 0);
         assertEquals(UNIGRAM_FREQ + 3, dict.getWordFrequency("abc"));
     }
 }
diff --git a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
index 3eabe2b..1944fd3 100644
--- a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
+++ b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
@@ -196,8 +196,8 @@
         final UserHistoryDictionaryBigramList resultList = new UserHistoryDictionaryBigramList();
         final OnAddWordListener listener = new OnAddWordListener() {
             @Override
-            public void setUnigram(final String word,
-                    final String shortcutTarget, final int frequency) {
+            public void setUnigram(final String word, final String shortcutTarget,
+                    final int frequency, final int shortcutFreq) {
                 Log.d(TAG, "in: setUnigram: " + word + "," + frequency);
                 resultList.addBigram(null, word, (byte)frequency);
             }
@@ -220,8 +220,8 @@
         final UserHistoryDictionaryBigramList resultList2 = new UserHistoryDictionaryBigramList();
         final OnAddWordListener listener2 = new OnAddWordListener() {
             @Override
-            public void setUnigram(final String word,
-                    final String shortcutTarget, final int frequency) {
+            public void setUnigram(final String word, final String shortcutTarget,
+                    final int frequency, final int shortcutFreq) {
                 Log.d(TAG, "in: setUnigram: " + word + "," + frequency);
                 resultList2.addBigram(null, word, (byte)frequency);
             }