Merge "Add KeyboardTextsSetTests"
diff --git a/java/res/values/attrs.xml b/java/res/values/attrs.xml
index 0550606..475e92f 100644
--- a/java/res/values/attrs.xml
+++ b/java/res/values/attrs.xml
@@ -175,7 +175,7 @@
</declare-styleable>
<declare-styleable name="SuggestionStripView">
- <attr name="suggestionStripOption" format="integer">
+ <attr name="suggestionStripOptions" format="integer">
<!-- This should be aligned with SuggestionStripLayoutHelper.AUTO_CORRECT_* and etc. -->
<flag name="autoCorrectBold" value="0x01" />
<flag name="autoCorrectUnderline" value="0x02" />
diff --git a/java/res/values/themes-gb.xml b/java/res/values/themes-gb.xml
index a460d4f..c189da3 100644
--- a/java/res/values/themes-gb.xml
+++ b/java/res/values/themes-gb.xml
@@ -133,7 +133,7 @@
parent="SuggestionStripView"
>
<item name="android:background">@drawable/keyboard_suggest_strip_gb</item>
- <item name="suggestionStripOption">autoCorrectBold|validTypedWordBold</item>
+ <item name="suggestionStripOptions">autoCorrectBold|validTypedWordBold</item>
<item name="colorValidTypedWord">@color/highlight_color_gb</item>
<item name="colorTypedWord">@color/typed_word_color_gb</item>
<item name="colorAutoCorrect">@color/highlight_color_gb</item>
diff --git a/java/res/values/themes-ics.xml b/java/res/values/themes-ics.xml
index caea921..720eda9 100644
--- a/java/res/values/themes-ics.xml
+++ b/java/res/values/themes-ics.xml
@@ -112,7 +112,7 @@
parent="SuggestionStripView"
>
<item name="android:background">@drawable/keyboard_suggest_strip_holo</item>
- <item name="suggestionStripOption">autoCorrectBold|validTypedWordBold</item>
+ <item name="suggestionStripOptions">autoCorrectBold|validTypedWordBold</item>
<item name="colorValidTypedWord">@color/typed_word_color_ics</item>
<item name="colorTypedWord">@color/typed_word_color_ics</item>
<item name="colorAutoCorrect">@color/highlight_color_ics</item>
diff --git a/java/res/values/themes-klp.xml b/java/res/values/themes-klp.xml
index 0599fb6..8305271 100644
--- a/java/res/values/themes-klp.xml
+++ b/java/res/values/themes-klp.xml
@@ -112,7 +112,7 @@
parent="SuggestionStripView"
>
<item name="android:background">@drawable/keyboard_suggest_strip_holo</item>
- <item name="suggestionStripOption">autoCorrectBold|validTypedWordBold</item>
+ <item name="suggestionStripOptions">autoCorrectBold|validTypedWordBold</item>
<item name="colorValidTypedWord">@color/typed_word_color_klp</item>
<item name="colorTypedWord">@color/typed_word_color_klp</item>
<item name="colorAutoCorrect">@color/highlight_color_klp</item>
diff --git a/java/res/xml/rowkeys_symbols_shift1.xml b/java/res/xml/rowkeys_symbols_shift1.xml
index 7cb3213..4c2722c 100644
--- a/java/res/xml/rowkeys_symbols_shift1.xml
+++ b/java/res/xml/rowkeys_symbols_shift1.xml
@@ -34,11 +34,11 @@
<!-- U+221A: "√" SQUARE ROOT -->
<Key
latin:keySpec="√" />
- <!-- U+03A0: "Π" GREEK CAPITAL LETTER PI
- U+03C0: "π" GREEK SMALL LETTER PI -->
+ <!-- U+03C0: "π" GREEK SMALL LETTER PI
+ U+03A0: "Π" GREEK CAPITAL LETTER PI -->
<Key
- latin:keySpec="Π"
- latin:moreKeys="π" />
+ latin:keySpec="π"
+ latin:moreKeys="Π" />
<!-- U+00F7: "÷" DIVISION SIGN -->
<Key
latin:keySpec="÷" />
diff --git a/java/src/com/android/inputmethod/keyboard/Key.java b/java/src/com/android/inputmethod/keyboard/Key.java
index afaf2cc..816a943 100644
--- a/java/src/com/android/inputmethod/keyboard/Key.java
+++ b/java/src/com/android/inputmethod/keyboard/Key.java
@@ -716,10 +716,14 @@
return (attrs != null) ? attrs.mAltCode : CODE_UNSPECIFIED;
}
+ public int getIconId() {
+ return mIconId;
+ }
+
public Drawable getIcon(final KeyboardIconsSet iconSet, final int alpha) {
final OptionalAttributes attrs = mOptionalAttributes;
final int disabledIconId = (attrs != null) ? attrs.mDisabledIconId : ICON_UNDEFINED;
- final int iconId = mEnabled ? mIconId : disabledIconId;
+ final int iconId = mEnabled ? getIconId() : disabledIconId;
final Drawable icon = iconSet.getIconDrawable(iconId);
if (icon != null) {
icon.setAlpha(alpha);
@@ -731,7 +735,7 @@
final OptionalAttributes attrs = mOptionalAttributes;
final int previewIconId = (attrs != null) ? attrs.mPreviewIconId : ICON_UNDEFINED;
return previewIconId != ICON_UNDEFINED
- ? iconSet.getIconDrawable(previewIconId) : iconSet.getIconDrawable(mIconId);
+ ? iconSet.getIconDrawable(previewIconId) : iconSet.getIconDrawable(getIconId());
}
public int getWidth() {
diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
index f975863..573c605 100644
--- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
+++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
@@ -340,10 +340,6 @@
mState.onCodeInput(code, mLatinIME.getCurrentAutoCapsState());
}
- private boolean isShowingMainKeyboard() {
- return null != mKeyboardView && mKeyboardView.isShown();
- }
-
public boolean isShowingEmojiPalettes() {
return mEmojiPalettesView != null && mEmojiPalettesView.isShown();
}
@@ -376,10 +372,6 @@
}
}
- public boolean isShowingMainKeyboardOrEmojiPalettes() {
- return isShowingMainKeyboard() || isShowingEmojiPalettes();
- }
-
public View onCreateInputView(final boolean isHardwareAcceleratedDrawingEnabled) {
if (mKeyboardView != null) {
mKeyboardView.closing();
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardIconsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardIconsSet.java
index da8bf7d..79d088f 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardIconsSet.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardIconsSet.java
@@ -102,7 +102,7 @@
return isValidIconId(iconId) ? ICON_NAMES[iconId] : "unknown<" + iconId + ">";
}
- static int getIconId(final String name) {
+ public static int getIconId(final String name) {
Integer iconId = sNameToIdsMap.get(name);
if (iconId != null) {
return iconId;
diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java
index 2dba713..cd18a6b 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java
@@ -36,6 +36,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
@@ -142,7 +143,8 @@
@UsedForTesting
public DictionaryFacilitatorForSuggest(final Context context, final Locale locale,
- final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles) {
+ final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles,
+ final Map<String, Map<String, String>> additionalDictAttributes) {
mContext = context;
mLocale = locale;
mLatchForWaitingLoadingMainDictionary = new CountDownLatch(0);
@@ -159,6 +161,10 @@
userHistoryDictionary.reloadDictionaryIfRequired();
userHistoryDictionary.waitAllTasksForTests();
setUserHistoryDictionary(userHistoryDictionary);
+ if (additionalDictAttributes.containsKey(dictType)) {
+ userHistoryDictionary.clearAndFlushDictionaryWithAdditionalAttributes(
+ additionalDictAttributes.get(dictType));
+ }
} else if (dictType.equals(Dictionary.TYPE_PERSONALIZATION)) {
final PersonalizationDictionary personalizationDictionary =
PersonalizationHelper.getPersonalizationDictionary(context, locale);
@@ -167,6 +173,10 @@
personalizationDictionary.reloadDictionaryIfRequired();
personalizationDictionary.waitAllTasksForTests();
setPersonalizationDictionary(personalizationDictionary);
+ if (additionalDictAttributes.containsKey(dictType)) {
+ personalizationDictionary.clearAndFlushDictionaryWithAdditionalAttributes(
+ additionalDictAttributes.get(dictType));
+ }
} else if (dictType.equals(Dictionary.TYPE_USER)) {
final File file = dictionaryFiles.get(dictType);
final UserBinaryDictionary userDictionary = new UserBinaryDictionary(
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index 44282a4..8ab1bb6 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -853,7 +853,7 @@
}
// This will set the punctuation suggestions if next word suggestion is off;
// otherwise it will clear the suggestion strip.
- setNeutralSuggestionStripInternal(false /* needsInputViewShown */);
+ setNeutralSuggestionStripInternal();
mHandler.cancelUpdateSuggestionStrip();
mHandler.cancelDoubleSpacePeriodTimer();
@@ -1018,23 +1018,18 @@
null /* rawSuggestions */, false /* typedWordValid */, false /* willAutoCorrect */,
false /* isObsoleteSuggestions */, false /* isPrediction */);
// When in fullscreen mode, show completions generated by the application forcibly
- setSuggestedWords(suggestedWords, true /* isSuggestionStripVisible */,
- true /* needsInputViewShown */);
+ setSuggestedWords(suggestedWords, true /* isSuggestionStripVisible */);
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
ResearchLogger.latinIME_onDisplayCompletions(applicationSpecifiedCompletions);
}
}
- private void setSuggestionStripShownInternal(final boolean isSuggestionStripVisible,
- final boolean needsInputViewShown) {
+ private void setSuggestionStripShownInternal(final boolean isSuggestionStripVisible) {
// TODO: Modify this if we support suggestions with hard keyboard
if (!onEvaluateInputViewShown() || !hasSuggestionStripView()) {
return;
}
- final boolean inputViewShown = mKeyboardSwitcher.isShowingMainKeyboardOrEmojiPalettes();
- final boolean shouldShowSuggestions = isSuggestionStripVisible
- && (needsInputViewShown ? inputViewShown : true);
- if (shouldShowSuggestions) {
+ if (isSuggestionStripVisible) {
mSuggestionStripView.setVisibility(View.VISIBLE);
} else {
mSuggestionStripView.setVisibility(isFullscreenMode() ? View.GONE : View.INVISIBLE);
@@ -1321,8 +1316,7 @@
// Nothing to do so far.
}
- // TODO[IL]: Define a clear interface for this
- public boolean isSuggestionStripVisible() {
+ private boolean isSuggestionStripVisible() {
if (!hasSuggestionStripView()) {
return false;
}
@@ -1346,7 +1340,6 @@
return currentSettings.isSuggestionsRequested();
}
- @Override
public boolean hasSuggestionStripView() {
return null != mSuggestionStripView;
}
@@ -1366,7 +1359,7 @@
// TODO[IL]: Define a clear interface for this
public void setSuggestedWords(final SuggestedWords suggestedWords,
- final boolean isSuggestionStripVisible, final boolean needsInputViewShown) {
+ final boolean isSuggestionStripVisible) {
mInputLogic.setSuggestedWords(suggestedWords);
if (!hasSuggestionStripView()) {
return;
@@ -1386,7 +1379,7 @@
SubtypeLocaleUtils.isRtlLanguage(mSubtypeSwitcher.getCurrentSubtype()));
}
mKeyboardSwitcher.onAutoCorrectionStateChanged(suggestedWords.mWillAutoCorrect);
- setSuggestionStripShownInternal(isSuggestionStripVisible, needsInputViewShown);
+ setSuggestionStripShownInternal(isSuggestionStripVisible);
}
// TODO[IL]: Move this out of LatinIME.
@@ -1472,8 +1465,7 @@
setNeutralSuggestionStrip();
} else {
mInputLogic.mWordComposer.setAutoCorrection(autoCorrection);
- setSuggestedWords(
- suggestedWords, isSuggestionStripVisible(), true /* needsInputViewShown */);
+ setSuggestedWords(suggestedWords, isSuggestionStripVisible());
}
// Cache the auto-correction in accessibility code so we can speak it if the user
// touches a key that will insert it.
@@ -1502,14 +1494,14 @@
// punctuation suggestions (if it's disabled).
@Override
public void setNeutralSuggestionStrip() {
- setNeutralSuggestionStripInternal(true /* needsInputViewShown */);
+ setNeutralSuggestionStripInternal();
}
- private void setNeutralSuggestionStripInternal(final boolean needsInputViewShown) {
+ private void setNeutralSuggestionStripInternal() {
final SettingsValues currentSettings = mSettings.getCurrent();
final SuggestedWords neutralSuggestions = currentSettings.mBigramPredictionEnabled
? SuggestedWords.EMPTY : currentSettings.mSpacingAndPunctuations.mSuggestPuncList;
- setSuggestedWords(neutralSuggestions, isSuggestionStripVisible(), needsInputViewShown);
+ setSuggestedWords(neutralSuggestions, isSuggestionStripVisible());
}
// TODO: Make this private
diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java
index abf831a..5e14410 100644
--- a/java/src/com/android/inputmethod/latin/Suggest.java
+++ b/java/src/com/android/inputmethod/latin/Suggest.java
@@ -147,6 +147,8 @@
}
}
+ final boolean isPrediction = !wordComposer.isComposingWord();
+
// We allow auto-correction if we have a whitelisted word, or if the word is not a valid
// word of more than 1 char, except if the first suggestion is the same as the typed string
// because in this case if it's strong enough to auto-correct that will mistakenly designate
@@ -165,7 +167,7 @@
// same time, it feels wrong that the SuggestedWord object includes information about
// the current settings. It may also be useful to know, when the setting is off, whether
// the word *would* have been auto-corrected.
- if (!isCorrectionEnabled || !allowsToBeAutoCorrected || !wordComposer.isComposingWord()
+ if (!isCorrectionEnabled || !allowsToBeAutoCorrected || isPrediction
|| suggestionsSet.isEmpty() || wordComposer.hasDigits()
|| wordComposer.isMostlyCaps() || wordComposer.isResumed()
|| !mDictionaryFacilitator.hasMainDictionary()
@@ -225,10 +227,9 @@
// TODO: this first argument is lying. If this is a whitelisted word which is an
// actual word, it says typedWordValid = false, which looks wrong. We should either
// rename the attribute or change the value.
- !allowsToBeAutoCorrected /* typedWordValid */,
+ !isPrediction && !allowsToBeAutoCorrected /* typedWordValid */,
hasAutoCorrection, /* willAutoCorrect */
- false /* isObsoleteSuggestions */,
- !wordComposer.isComposingWord() /* isPrediction */, sequenceNumber));
+ false /* isObsoleteSuggestions */, isPrediction, sequenceNumber));
}
// Retrieves suggestions for the batch input
diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java
index 46df3e8..06bc90c 100644
--- a/java/src/com/android/inputmethod/latin/SuggestedWords.java
+++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java
@@ -69,7 +69,7 @@
final boolean isPrediction,
final int sequenceNumber) {
this(suggestedWordInfoList, rawSuggestions,
- suggestedWordInfoList.isEmpty() ? null
+ (suggestedWordInfoList.isEmpty() || isPrediction) ? null
: suggestedWordInfoList.get(INDEX_OF_TYPED_WORD).mWord,
typedWordValid, willAutoCorrect, isObsoleteSuggestions, isPrediction,
sequenceNumber);
diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
index 045d06f..f2f9f1e 100644
--- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
+++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
@@ -1360,10 +1360,12 @@
}});
} else {
// We found suggestion spans in the word. We'll create the SuggestedWords out of
- // them, and make willAutoCorrect false.
+ // them, and make willAutoCorrect false. We make typedWordValid false, because the
+ // color of the word in the suggestion strip changes according to this parameter,
+ // and false gives the correct color.
final SuggestedWords suggestedWords = new SuggestedWords(suggestions,
null /* rawSuggestions */, typedWord,
- true /* typedWordValid */, false /* willAutoCorrect */,
+ false /* typedWordValid */, false /* willAutoCorrect */,
false /* isObsoleteSuggestions */, false /* isPrediction */,
SuggestedWords.NOT_A_SEQUENCE_NUMBER);
mIsAutoCorrectionIndicatorOn = false;
diff --git a/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java b/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java
index b99e281..ed9c396 100644
--- a/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java
+++ b/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java
@@ -38,6 +38,10 @@
public static final String DICTIONARY_DATE_KEY = "date";
public static final String HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO";
public static final String USES_FORGETTING_CURVE_KEY = "USES_FORGETTING_CURVE";
+ public static final String FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY =
+ "FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP";
+ public static final String FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY =
+ "FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID";
public static final String ATTRIBUTE_VALUE_TRUE = "1";
public DictionaryHeader(final int headerSize, final DictionaryOptions dictionaryOptions,
diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
index d3734d6..8c358cd 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
@@ -50,6 +50,7 @@
public final Locale mLocale;
private final String mDictName;
+ private Map<String, String> mAdditionalAttributeMap = null;
protected DecayingExpandableBinaryDictionaryBase(final Context context,
final String dictName, final Locale locale, final String dictionaryType,
@@ -78,7 +79,10 @@
@Override
protected Map<String, String> getHeaderAttributeMap() {
- HashMap<String, String> attributeMap = new HashMap<String, String>();
+ final Map<String, String> attributeMap = new HashMap<String, String>();
+ if (mAdditionalAttributeMap != null) {
+ attributeMap.putAll(mAdditionalAttributeMap);
+ }
attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
@@ -152,6 +156,13 @@
asyncFlushBinaryDictionary();
}
+ @UsedForTesting
+ public void clearAndFlushDictionaryWithAdditionalAttributes(
+ final Map<String, String> attributeMap) {
+ mAdditionalAttributeMap = attributeMap;
+ clearAndFlushDictionary();
+ }
+
/* package */ void decayIfNeeded() {
runGCIfRequired(false /* mindsBlockByGC */);
}
diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java
index 8ea7128..afa8fe3 100644
--- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java
+++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java
@@ -95,9 +95,9 @@
private static final CharacterStyle BOLD_SPAN = new StyleSpan(Typeface.BOLD);
private static final CharacterStyle UNDERLINE_SPAN = new UnderlineSpan();
- private final int mSuggestionStripOption;
+ private final int mSuggestionStripOptions;
// These constants are the flag values of
- // {@link R.styleable#SuggestionStripView_suggestionStripOption} attribute.
+ // {@link R.styleable#SuggestionStripView_suggestionStripOptions} attribute.
private static final int AUTO_CORRECT_BOLD = 0x01;
private static final int AUTO_CORRECT_UNDERLINE = 0x02;
private static final int VALID_TYPED_WORD_BOLD = 0x04;
@@ -122,8 +122,8 @@
final TypedArray a = context.obtainStyledAttributes(attrs,
R.styleable.SuggestionStripView, defStyle, R.style.SuggestionStripView);
- mSuggestionStripOption = a.getInt(
- R.styleable.SuggestionStripView_suggestionStripOption, 0);
+ mSuggestionStripOptions = a.getInt(
+ R.styleable.SuggestionStripView_suggestionStripOptions, 0);
mAlphaObsoleted = ResourceUtils.getFraction(a,
R.styleable.SuggestionStripView_alphaObsoleted, 1.0f);
mColorValidTypedWord = a.getColor(R.styleable.SuggestionStripView_colorValidTypedWord, 0);
@@ -200,22 +200,24 @@
return null;
}
final String word = suggestedWords.getLabel(indexInSuggestedWords);
- final boolean isAutoCorrect = indexInSuggestedWords == 1
- && suggestedWords.mWillAutoCorrect;
- final boolean isTypedWordValid = indexInSuggestedWords == 0
- && suggestedWords.mTypedWordValid;
- if (!isAutoCorrect && !isTypedWordValid) {
+ // TODO: don't use the index to decide whether this is the auto-correction/typed word, as
+ // this is brittle
+ final boolean isAutoCorrection = suggestedWords.mWillAutoCorrect
+ && indexInSuggestedWords == SuggestedWords.INDEX_OF_AUTO_CORRECTION;
+ final boolean isTypedWordValid = suggestedWords.mTypedWordValid
+ && indexInSuggestedWords == SuggestedWords.INDEX_OF_TYPED_WORD;
+ if (!isAutoCorrection && !isTypedWordValid) {
return word;
}
final int len = word.length();
final Spannable spannedWord = new SpannableString(word);
- final int option = mSuggestionStripOption;
- if ((isAutoCorrect && (option & AUTO_CORRECT_BOLD) != 0)
- || (isTypedWordValid && (option & VALID_TYPED_WORD_BOLD) != 0)) {
+ final int options = mSuggestionStripOptions;
+ if ((isAutoCorrection && (options & AUTO_CORRECT_BOLD) != 0)
+ || (isTypedWordValid && (options & VALID_TYPED_WORD_BOLD) != 0)) {
spannedWord.setSpan(BOLD_SPAN, 0, len, Spanned.SPAN_INCLUSIVE_EXCLUSIVE);
}
- if (isAutoCorrect && (option & AUTO_CORRECT_UNDERLINE) != 0) {
+ if (isAutoCorrection && (options & AUTO_CORRECT_UNDERLINE) != 0) {
spannedWord.setSpan(UNDERLINE_SPAN, 0, len, Spanned.SPAN_INCLUSIVE_EXCLUSIVE);
}
return spannedWord;
@@ -242,22 +244,23 @@
return indexInSuggestedWords;
}
- private int getSuggestionTextColor(final int indexInSuggestedWords,
- final SuggestedWords suggestedWords) {
+ private int getSuggestionTextColor(final SuggestedWords suggestedWords,
+ final int indexInSuggestedWords) {
final int positionInStrip =
getPositionInSuggestionStrip(indexInSuggestedWords, suggestedWords);
- // TODO: Need to revisit this logic with bigram suggestions
- final boolean isSuggested = (indexInSuggestedWords != SuggestedWords.INDEX_OF_TYPED_WORD);
+ // Use identity for strings, not #equals : it's the typed word if it's the same object
+ final boolean isTypedWord =
+ suggestedWords.getWord(indexInSuggestedWords) == suggestedWords.mTypedWord;
final int color;
if (positionInStrip == mCenterPositionInStrip && suggestedWords.mWillAutoCorrect) {
color = mColorAutoCorrect;
- } else if (positionInStrip == mCenterPositionInStrip && suggestedWords.mTypedWordValid) {
+ } else if (isTypedWord && suggestedWords.mTypedWordValid) {
color = mColorValidTypedWord;
- } else if (isSuggested) {
- color = mColorSuggested;
- } else {
+ } else if (isTypedWord) {
color = mColorTypedWord;
+ } else {
+ color = mColorSuggested;
}
if (LatinImeLogger.sDBG && suggestedWords.size() > 1) {
// If we auto-correct, then the autocorrection is in slot 0 and the typed word
@@ -270,7 +273,7 @@
}
}
- if (suggestedWords.mIsObsoleteSuggestions && isSuggested) {
+ if (suggestedWords.mIsObsoleteSuggestions && !isTypedWord) {
return applyAlpha(color, mAlphaObsoleted);
}
return color;
@@ -438,7 +441,7 @@
// {@link SuggestionStripView#onClick(View)}.
wordView.setTag(indexInSuggestedWords);
wordView.setText(getStyledSuggestedWord(suggestedWords, indexInSuggestedWords));
- wordView.setTextColor(getSuggestionTextColor(positionInStrip, suggestedWords));
+ wordView.setTextColor(getSuggestionTextColor(suggestedWords, indexInSuggestedWords));
if (SuggestionStripView.DBG) {
mDebugInfoViews.get(positionInStrip).setText(
suggestedWords.getDebugString(indexInSuggestedWords));
diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripViewAccessor.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripViewAccessor.java
index 60f1c7a..5270845 100644
--- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripViewAccessor.java
+++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripViewAccessor.java
@@ -22,7 +22,6 @@
* An object that gives basic control of a suggestion strip and some info on it.
*/
public interface SuggestionStripViewAccessor {
- public boolean hasSuggestionStripView();
public void showAddToDictionaryHint(final String word);
public boolean isShowingAddToDictionaryHint();
public void dismissAddToDictionaryHint();
diff --git a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java
index a1d6415..562ff9e 100644
--- a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java
+++ b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java
@@ -37,9 +37,9 @@
// non-0. Thus, it's not meaningful to compare 10, 100, and so on.
// TODO: Revise the logic in ForgettingCurveUtils in native code.
private static final int UNIGRAM_PROBABILITY_FOR_VALID_WORD = 100;
- private static final int UNIGRAM_PROBABILITY_FOR_OOV_WORD = 10;
- private static final int BIGRAM_PROBABILITY_FOR_VALID_WORD = 0;
- private static final int BIGRAM_PROBABILITY_FOR_OOV_WORD = 0;
+ private static final int UNIGRAM_PROBABILITY_FOR_OOV_WORD = Dictionary.NOT_A_PROBABILITY;
+ private static final int BIGRAM_PROBABILITY_FOR_VALID_WORD = 10;
+ private static final int BIGRAM_PROBABILITY_FOR_OOV_WORD = Dictionary.NOT_A_PROBABILITY;
public final String mTargetWord;
public final int[] mWord0;
diff --git a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h
index 59748c8..a8dab9f 100644
--- a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h
+++ b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h
@@ -44,8 +44,6 @@
virtual float getMultiWordCostMultiplier() const = 0;
- virtual int getLastDecayedTime() const = 0;
-
virtual void readHeaderValueOrQuestionMark(const char *const key, int *outValue,
int outValueSize) const = 0;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp
index cd22430..7f91667 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp
@@ -37,7 +37,8 @@
if (outProbability) {
if (bigramEntry.hasHistoricalInfo()) {
*outProbability =
- ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo());
+ ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo(),
+ mHeaderPolicy);
} else {
*outProbability = bigramEntry.getProbability();
}
@@ -160,7 +161,7 @@
}
} else if (bigramEntry.hasHistoricalInfo()) {
const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave(
- bigramEntry.getHistoricalInfo());
+ bigramEntry.getHistoricalInfo(), mHeaderPolicy);
if (ForgettingCurveUtils::needsToKeep(&historicalInfo)) {
const BigramEntry updatedBigramEntry =
bigramEntry.updateHistoricalInfoAndGetEntry(&historicalInfo);
@@ -230,7 +231,8 @@
if (mHeaderPolicy->hasHistoricalInfoOfWords()) {
const HistoricalInfo updatedHistoricalInfo =
ForgettingCurveUtils::createUpdatedHistoricalInfo(
- originalBigramEntry->getHistoricalInfo(), newProbability, timestamp);
+ originalBigramEntry->getHistoricalInfo(), newProbability, timestamp,
+ mHeaderPolicy);
return originalBigramEntry->updateHistoricalInfoAndGetEntry(&updatedHistoricalInfo);
} else {
return originalBigramEntry->updateProbabilityAndGetEntry(newProbability);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
index 3ce57d9..2ac417b 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
@@ -18,7 +18,7 @@
namespace latinime {
-// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
+// Note that these are corresponding definitions in Java side in DictionaryHeader.
const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE";
const char *const HeaderPolicy::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
"REQUIRES_GERMAN_UMLAUT_PROCESSING";
@@ -33,8 +33,15 @@
// count.
const char *const HeaderPolicy::HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO";
const char *const HeaderPolicy::LOCALE_KEY = "locale"; // match Java declaration
+const char *const HeaderPolicy::FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY =
+ "FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP";
+const char *const HeaderPolicy::FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY =
+ "FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID";
+
const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100;
const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f;
+const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP = 4;
+const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID = 0;
// Used for logging. Question mark is used to indicate that the key is not found.
void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *outValue,
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
index fc34761..8fa7e16 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
@@ -52,7 +52,13 @@
mExtendedRegionSize(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
EXTENDED_REGION_SIZE_KEY, 0 /* defaultValue */)),
mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
- &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {}
+ &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)),
+ mForgettingCurveOccurrencesToLevelUp(HeaderReadWriteUtils::readIntAttributeValue(
+ &mAttributeMap, FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY,
+ DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP)),
+ mForgettingCurveProbabilityValuesTableId(HeaderReadWriteUtils::readIntAttributeValue(
+ &mAttributeMap, FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY,
+ DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID)) {}
// Constructs header information using an attribute map.
HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion,
@@ -71,8 +77,13 @@
DATE_KEY, TimeKeeper::peekCurrentTime() /* defaultValue */)),
mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0),
mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
- &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {
- }
+ &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)),
+ mForgettingCurveOccurrencesToLevelUp(HeaderReadWriteUtils::readIntAttributeValue(
+ &mAttributeMap, FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY,
+ DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP)),
+ mForgettingCurveProbabilityValuesTableId(HeaderReadWriteUtils::readIntAttributeValue(
+ &mAttributeMap, FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY,
+ DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID)) {}
// Temporary dummy header.
HeaderPolicy()
@@ -80,7 +91,8 @@
mAttributeMap(), mLocale(CharUtils::EMPTY_STRING), mMultiWordCostMultiplier(0.0f),
mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false),
mDate(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
- mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
+ mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false),
+ mForgettingCurveOccurrencesToLevelUp(0), mForgettingCurveProbabilityValuesTableId(0) {}
~HeaderPolicy() {}
@@ -159,6 +171,14 @@
return &mAttributeMap;
}
+ AK_FORCE_INLINE int getForgettingCurveOccurrencesToLevelUp() const {
+ return mForgettingCurveOccurrencesToLevelUp;
+ }
+
+ AK_FORCE_INLINE int getForgettingCurveProbabilityValuesTableId() const {
+ return mForgettingCurveProbabilityValuesTableId;
+ }
+
void readHeaderValueOrQuestionMark(const char *const key,
int *outValue, int outValueSize) const;
@@ -183,8 +203,12 @@
static const char *const EXTENDED_REGION_SIZE_KEY;
static const char *const HAS_HISTORICAL_INFO_KEY;
static const char *const LOCALE_KEY;
+ static const char *const FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY;
+ static const char *const FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY;
static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE;
static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE;
+ static const int DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP;
+ static const int DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID;
const FormatUtils::FORMAT_VERSION mDictFormatVersion;
const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags;
@@ -200,6 +224,8 @@
const int mBigramCount;
const int mExtendedRegionSize;
const bool mHasHistoricalInfoOfWords;
+ const int mForgettingCurveOccurrencesToLevelUp;
+ const int mForgettingCurveProbabilityValuesTableId;
const std::vector<int> readLocale() const;
float readMultipleWordCostMultiplier() const;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
index cb9d450..279f5b3 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
@@ -23,6 +23,13 @@
const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
int *const bigramEntryPos) const {
const BufferWithExtendableBuffer *const bigramListBuffer = getContentBuffer();
+ if (*bigramEntryPos < 0 || *bigramEntryPos >= bigramListBuffer->getTailPosition()) {
+ AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bufSize: %d",
+ *bigramEntryPos, bigramListBuffer->getTailPosition());
+ ASSERT(false);
+ return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
+ Ver4DictConstants::NOT_A_TERMINAL_ID);
+ }
const int bigramFlags = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, bigramEntryPos);
const bool hasNext = (bigramFlags & Ver4DictConstants::BIGRAM_HAS_NEXT_MASK) != 0;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
index 29972a4..64d7bc0 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
@@ -24,6 +24,19 @@
int *const outCodePoint, int *const outCodePointCount, int *const outProbability,
bool *const outhasNext, int *const shortcutEntryPos) const {
const BufferWithExtendableBuffer *const shortcutListBuffer = getContentBuffer();
+ if (*shortcutEntryPos < 0 || *shortcutEntryPos >= shortcutListBuffer->getTailPosition()) {
+ AKLOGE("Invalid shortcut entry position. shortcutEntryPos: %d, bufSize: %d",
+ *shortcutEntryPos, shortcutListBuffer->getTailPosition());
+ ASSERT(false);
+ if (outhasNext) {
+ *outhasNext = false;
+ }
+ if (outCodePointCount) {
+ *outCodePointCount = 0;
+ }
+ return;
+ }
+
const int shortcutFlags = shortcutListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE, shortcutEntryPos);
if (outProbability) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
index 17fc948..f149781 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
@@ -65,7 +65,7 @@
mProbabilityDictContent->getProbabilityEntry(terminalId);
if (probabilityEntry.hasHistoricalInfo()) {
probability = ForgettingCurveUtils::decodeProbability(
- probabilityEntry.getHistoricalInfo());
+ probabilityEntry.getHistoricalInfo(), mHeaderPolicy);
} else {
probability = probabilityEntry.getProbability();
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h
index 9d93245..1db9ea0 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h
@@ -26,6 +26,7 @@
namespace latinime {
class BufferWithExtendableBuffer;
+class HeaderPolicy;
class ProbabilityDictContent;
/*
@@ -35,8 +36,10 @@
class Ver4PatriciaTrieNodeReader : public PtNodeReader {
public:
Ver4PatriciaTrieNodeReader(const BufferWithExtendableBuffer *const buffer,
- const ProbabilityDictContent *const probabilityDictContent)
- : mBuffer(buffer), mProbabilityDictContent(probabilityDictContent) {}
+ const ProbabilityDictContent *const probabilityDictContent,
+ const HeaderPolicy *const headerPolicy)
+ : mBuffer(buffer), mProbabilityDictContent(probabilityDictContent),
+ mHeaderPolicy(headerPolicy) {}
~Ver4PatriciaTrieNodeReader() {}
@@ -50,6 +53,7 @@
const BufferWithExtendableBuffer *const mBuffer;
const ProbabilityDictContent *const mProbabilityDictContent;
+ const HeaderPolicy *const mHeaderPolicy;
const PtNodeParams fetchPtNodeInfoFromBufferAndProcessMovedPtNode(const int ptNodePos,
const int siblingNodePos) const;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
index 32576cf..13ae9d9 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
@@ -159,7 +159,7 @@
toBeUpdatedPtNodeParams->getTerminalId());
if (originalProbabilityEntry.hasHistoricalInfo()) {
const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave(
- originalProbabilityEntry.getHistoricalInfo());
+ originalProbabilityEntry.getHistoricalInfo(), mHeaderPolicy);
const ProbabilityEntry probabilityEntry =
originalProbabilityEntry.createEntryWithUpdatedHistoricalInfo(&historicalInfo);
if (!mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry(
@@ -382,10 +382,11 @@
const ProbabilityEntry *const originalProbabilityEntry, const int newProbability,
const int timestamp) const {
// TODO: Consolidate historical info and probability.
- if (mBuffers->getHeaderPolicy()->hasHistoricalInfoOfWords()) {
+ if (mHeaderPolicy->hasHistoricalInfoOfWords()) {
const HistoricalInfo updatedHistoricalInfo =
ForgettingCurveUtils::createUpdatedHistoricalInfo(
- originalProbabilityEntry->getHistoricalInfo(), newProbability, timestamp);
+ originalProbabilityEntry->getHistoricalInfo(), newProbability, timestamp,
+ mHeaderPolicy);
return originalProbabilityEntry->createEntryWithUpdatedHistoricalInfo(
&updatedHistoricalInfo);
} else {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
index 66845bb..f01b3af 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
@@ -28,6 +28,7 @@
namespace latinime {
class BufferWithExtendableBuffer;
+class HeaderPolicy;
class Ver4BigramListPolicy;
class Ver4DictBuffers;
class Ver4PatriciaTrieNodeReader;
@@ -40,10 +41,11 @@
class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
public:
Ver4PatriciaTrieNodeWriter(BufferWithExtendableBuffer *const trieBuffer,
- Ver4DictBuffers *const buffers, const PtNodeReader *const ptNodeReader,
+ Ver4DictBuffers *const buffers, const HeaderPolicy *const headerPolicy,
+ const PtNodeReader *const ptNodeReader,
const PtNodeArrayReader *const ptNodeArrayReader,
Ver4BigramListPolicy *const bigramPolicy, Ver4ShortcutListPolicy *const shortcutPolicy)
- : mTrieBuffer(trieBuffer), mBuffers(buffers),
+ : mTrieBuffer(trieBuffer), mBuffers(buffers), mHeaderPolicy(headerPolicy),
mReadingHelper(ptNodeReader, ptNodeArrayReader), mBigramPolicy(bigramPolicy),
mShortcutPolicy(shortcutPolicy) {}
@@ -116,6 +118,7 @@
BufferWithExtendableBuffer *const mTrieBuffer;
Ver4DictBuffers *const mBuffers;
+ const HeaderPolicy *const mHeaderPolicy;
DynamicPtReadingHelper mReadingHelper;
Ver4BigramListPolicy *const mBigramPolicy;
Ver4ShortcutListPolicy *const mShortcutPolicy;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
index b5d80be..197250f 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
@@ -382,7 +382,8 @@
bigramWord1CodePoints + codePointCount);
const HistoricalInfo *const historicalInfo = bigramEntry.getHistoricalInfo();
const int probability = bigramEntry.hasHistoricalInfo() ?
- ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo()) :
+ ForgettingCurveUtils::decodeProbability(
+ bigramEntry.getHistoricalInfo(), mHeaderPolicy) :
bigramEntry.getProbability();
bigrams.push_back(WordProperty::BigramProperty(&word1, probability,
historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
index 7796e2d..639c153 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
@@ -47,10 +47,10 @@
mBuffers.get()->getTerminalPositionLookupTable(), mHeaderPolicy),
mShortcutPolicy(mBuffers.get()->getMutableShortcutDictContent(),
mBuffers.get()->getTerminalPositionLookupTable()),
- mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent()),
+ mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent(), mHeaderPolicy),
mPtNodeArrayReader(mDictBuffer),
- mNodeWriter(mDictBuffer, mBuffers.get(), &mNodeReader, &mPtNodeArrayReader,
- &mBigramPolicy, &mShortcutPolicy),
+ mNodeWriter(mDictBuffer, mBuffers.get(), mHeaderPolicy, &mNodeReader,
+ &mPtNodeArrayReader, &mBigramPolicy, &mShortcutPolicy),
mUpdatingHelper(mDictBuffer, &mNodeReader, &mNodeWriter),
mWritingHelper(mBuffers.get()),
mUnigramCount(mHeaderPolicy->getUnigramCount()),
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
index 93053c3..2da2950 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
@@ -74,14 +74,15 @@
const HeaderPolicy *const headerPolicy, Ver4DictBuffers *const buffersToWrite,
int *const outUnigramCount, int *const outBigramCount) {
Ver4PatriciaTrieNodeReader ptNodeReader(mBuffers->getTrieBuffer(),
- mBuffers->getProbabilityDictContent());
+ mBuffers->getProbabilityDictContent(), headerPolicy);
Ver4PtNodeArrayReader ptNodeArrayReader(mBuffers->getTrieBuffer());
Ver4BigramListPolicy bigramPolicy(mBuffers->getMutableBigramDictContent(),
mBuffers->getTerminalPositionLookupTable(), headerPolicy);
Ver4ShortcutListPolicy shortcutPolicy(mBuffers->getMutableShortcutDictContent(),
mBuffers->getTerminalPositionLookupTable());
Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(),
- mBuffers, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy, &shortcutPolicy);
+ mBuffers, headerPolicy, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy,
+ &shortcutPolicy);
DynamicPtReadingHelper readingHelper(&ptNodeReader, &ptNodeArrayReader);
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
@@ -126,7 +127,8 @@
PtNodeWriter::DictPositionRelocationMap dictPositionRelocationMap;
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
Ver4PatriciaTrieNodeWriter ptNodeWriterForNewBuffers(buffersToWrite->getWritableTrieBuffer(),
- buffersToWrite, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy, &shortcutPolicy);
+ buffersToWrite, headerPolicy, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy,
+ &shortcutPolicy);
DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&ptNodeWriterForNewBuffers,
buffersToWrite->getWritableTrieBuffer(), &dictPositionRelocationMap);
@@ -137,14 +139,14 @@
// Create policy instances for the GCed dictionary.
Ver4PatriciaTrieNodeReader newPtNodeReader(buffersToWrite->getTrieBuffer(),
- buffersToWrite->getProbabilityDictContent());
+ buffersToWrite->getProbabilityDictContent(), headerPolicy);
Ver4PtNodeArrayReader newPtNodeArrayreader(buffersToWrite->getTrieBuffer());
Ver4BigramListPolicy newBigramPolicy(buffersToWrite->getMutableBigramDictContent(),
buffersToWrite->getTerminalPositionLookupTable(), headerPolicy);
Ver4ShortcutListPolicy newShortcutPolicy(buffersToWrite->getMutableShortcutDictContent(),
buffersToWrite->getTerminalPositionLookupTable());
Ver4PatriciaTrieNodeWriter newPtNodeWriter(buffersToWrite->getWritableTrieBuffer(),
- buffersToWrite, &newPtNodeReader, &newPtNodeArrayreader, &newBigramPolicy,
+ buffersToWrite, headerPolicy, &newPtNodeReader, &newPtNodeArrayreader, &newBigramPolicy,
&newShortcutPolicy);
// Re-assign terminal IDs for valid terminal PtNodes.
TerminalPositionLookupTable::TerminalIdMap terminalIdMap;
@@ -202,8 +204,9 @@
const ProbabilityEntry probabilityEntry =
mBuffers->getProbabilityDictContent()->getProbabilityEntry(i);
const int probability = probabilityEntry.hasHistoricalInfo() ?
- ForgettingCurveUtils::decodeProbability(probabilityEntry.getHistoricalInfo()) :
- probabilityEntry.getProbability();
+ ForgettingCurveUtils::decodeProbability(
+ probabilityEntry.getHistoricalInfo(), mBuffers->getHeaderPolicy()) :
+ probabilityEntry.getProbability();
priorityQueue.push(DictProbability(terminalPos, probability,
probabilityEntry.getHistoricalInfo()->getTimeStamp()));
}
@@ -245,8 +248,9 @@
continue;
}
const int probability = bigramEntry.hasHistoricalInfo() ?
- ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo()) :
- bigramEntry.getProbability();
+ ForgettingCurveUtils::decodeProbability(
+ bigramEntry.getHistoricalInfo(), mBuffers->getHeaderPolicy()) :
+ bigramEntry.getProbability();
priorityQueue.push(DictProbability(entryPos, probability,
bigramEntry.getHistoricalInfo()->getTimeStamp()));
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
index d58d259..51b0eb2 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
@@ -19,7 +19,7 @@
#include <cmath>
#include <stdlib.h>
-#include "suggest/core/policy/dictionary_header_structure_policy.h"
+#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/utils/probability_utils.h"
#include "utils/time_keeper.h"
@@ -30,11 +30,10 @@
const int ForgettingCurveUtils::MAX_BIGRAM_COUNT = 12000;
const int ForgettingCurveUtils::MAX_BIGRAM_COUNT_AFTER_GC = 10000;
-const int ForgettingCurveUtils::MAX_COMPUTED_PROBABILITY = 127;
+const int ForgettingCurveUtils::MULTIPLIER_TWO_IN_PROBABILITY_SCALE = 8;
const int ForgettingCurveUtils::DECAY_INTERVAL_SECONDS = 2 * 60 * 60;
const int ForgettingCurveUtils::MAX_LEVEL = 3;
-const int ForgettingCurveUtils::MAX_COUNT = 3;
const int ForgettingCurveUtils::MIN_VALID_LEVEL = 1;
const int ForgettingCurveUtils::TIME_STEP_DURATION_IN_SECONDS = 6 * 60 * 60;
const int ForgettingCurveUtils::MAX_ELAPSED_TIME_STEP_COUNT = 15;
@@ -45,7 +44,7 @@
// TODO: Revise the logic to decide the initial probability depending on the given probability.
/* static */ const HistoricalInfo ForgettingCurveUtils::createUpdatedHistoricalInfo(
const HistoricalInfo *const originalHistoricalInfo,
- const int newProbability, const int timestamp) {
+ const int newProbability, const int timestamp, const HeaderPolicy *const headerPolicy) {
if (newProbability != NOT_A_PROBABILITY && originalHistoricalInfo->getLevel() == 0) {
return HistoricalInfo(timestamp, MIN_VALID_LEVEL /* level */, 0 /* count */);
} else if (!originalHistoricalInfo->isValid()) {
@@ -53,7 +52,7 @@
return HistoricalInfo(timestamp, 0 /* level */, 1 /* count */);
} else {
const int updatedCount = originalHistoricalInfo->getCount() + 1;
- if (updatedCount > MAX_COUNT) {
+ if (updatedCount >= headerPolicy->getForgettingCurveOccurrencesToLevelUp()) {
// The count exceeds the max value the level can be incremented.
if (originalHistoricalInfo->getLevel() >= MAX_LEVEL) {
// The level is already max.
@@ -71,9 +70,10 @@
}
/* static */ int ForgettingCurveUtils::decodeProbability(
- const HistoricalInfo *const historicalInfo) {
+ const HistoricalInfo *const historicalInfo, const HeaderPolicy *const headerPolicy) {
const int elapsedTimeStepCount = getElapsedTimeStepCount(historicalInfo->getTimeStamp());
- return sProbabilityTable.getProbability(historicalInfo->getLevel(),
+ return sProbabilityTable.getProbability(
+ headerPolicy->getForgettingCurveProbabilityValuesTableId(), historicalInfo->getLevel(),
min(max(elapsedTimeStepCount, 0), MAX_ELAPSED_TIME_STEP_COUNT));
}
@@ -82,9 +82,11 @@
if (unigramProbability == NOT_A_PROBABILITY) {
return NOT_A_PROBABILITY;
} else if (bigramProbability == NOT_A_PROBABILITY) {
- return min(backoff(unigramProbability), MAX_COMPUTED_PROBABILITY);
+ return min(backoff(unigramProbability), MAX_PROBABILITY);
} else {
- return min(max(unigramProbability, bigramProbability), MAX_COMPUTED_PROBABILITY);
+ // TODO: Investigate better way to handle bigram probability.
+ return min(max(unigramProbability, bigramProbability + MULTIPLIER_TWO_IN_PROBABILITY_SCALE),
+ MAX_PROBABILITY);
}
}
@@ -95,7 +97,8 @@
}
/* static */ const HistoricalInfo ForgettingCurveUtils::createHistoricalInfoToSave(
- const HistoricalInfo *const originalHistoricalInfo) {
+ const HistoricalInfo *const originalHistoricalInfo,
+ const HeaderPolicy *const headerPolicy) {
if (originalHistoricalInfo->getTimeStamp() == NOT_A_TIMESTAMP) {
return HistoricalInfo();
}
@@ -115,8 +118,7 @@
}
/* static */ bool ForgettingCurveUtils::needsToDecay(const bool mindsBlockByDecay,
- const int unigramCount, const int bigramCount,
- const DictionaryHeaderStructurePolicy *const headerPolicy) {
+ const int unigramCount, const int bigramCount, const HeaderPolicy *const headerPolicy) {
if (unigramCount >= ForgettingCurveUtils::MAX_UNIGRAM_COUNT) {
// Unigram count exceeds the limit.
return true;
@@ -137,37 +139,69 @@
// See comments in ProbabilityUtils::backoff().
/* static */ int ForgettingCurveUtils::backoff(const int unigramProbability) {
- if (unigramProbability == NOT_A_PROBABILITY) {
- return NOT_A_PROBABILITY;
- } else {
- return max(unigramProbability - 8, 0);
- }
+ // See TODO comments in ForgettingCurveUtils::getProbability().
+ return unigramProbability;
}
/* static */ int ForgettingCurveUtils::getElapsedTimeStepCount(const int timestamp) {
return (TimeKeeper::peekCurrentTime() - timestamp) / TIME_STEP_DURATION_IN_SECONDS;
}
-ForgettingCurveUtils::ProbabilityTable::ProbabilityTable() : mTable() {
- mTable.resize(MAX_LEVEL + 1);
- for (int level = 0; level <= MAX_LEVEL; ++level) {
- mTable[level].resize(MAX_ELAPSED_TIME_STEP_COUNT + 1);
- const float initialProbability =
- static_cast<float>(MAX_COMPUTED_PROBABILITY / (1 << (MAX_LEVEL - level)));
- for (int timeStepCount = 0; timeStepCount <= MAX_ELAPSED_TIME_STEP_COUNT; ++timeStepCount) {
- if (level == 0) {
- mTable[level][timeStepCount] = NOT_A_PROBABILITY;
- continue;
+const int ForgettingCurveUtils::ProbabilityTable::PROBABILITY_TABLE_COUNT = 4;
+const int ForgettingCurveUtils::ProbabilityTable::WEAK_PROBABILITY_TABLE_ID = 0;
+const int ForgettingCurveUtils::ProbabilityTable::MODEST_PROBABILITY_TABLE_ID = 1;
+const int ForgettingCurveUtils::ProbabilityTable::STRONG_PROBABILITY_TABLE_ID = 2;
+const int ForgettingCurveUtils::ProbabilityTable::AGGRESSIVE_PROBABILITY_TABLE_ID = 3;
+const int ForgettingCurveUtils::ProbabilityTable::WEAK_MAX_PROBABILITY = 127;
+const int ForgettingCurveUtils::ProbabilityTable::MODEST_BASE_PROBABILITY = 32;
+const int ForgettingCurveUtils::ProbabilityTable::STRONG_BASE_PROBABILITY = 35;
+const int ForgettingCurveUtils::ProbabilityTable::AGGRESSIVE_BASE_PROBABILITY = 40;
+
+
+ForgettingCurveUtils::ProbabilityTable::ProbabilityTable() : mTables() {
+ mTables.resize(PROBABILITY_TABLE_COUNT);
+ for (int tableId = 0; tableId < PROBABILITY_TABLE_COUNT; ++tableId) {
+ mTables[tableId].resize(MAX_LEVEL + 1);
+ for (int level = 0; level <= MAX_LEVEL; ++level) {
+ mTables[tableId][level].resize(MAX_ELAPSED_TIME_STEP_COUNT + 1);
+ const float initialProbability = getBaseProbabilityForLevel(tableId, level);
+ const float endProbability = getBaseProbabilityForLevel(tableId, level - 1);
+ for (int timeStepCount = 0; timeStepCount <= MAX_ELAPSED_TIME_STEP_COUNT;
+ ++timeStepCount) {
+ if (level == 0) {
+ mTables[tableId][level][timeStepCount] = NOT_A_PROBABILITY;
+ continue;
+ }
+ const int elapsedTime = timeStepCount * TIME_STEP_DURATION_IN_SECONDS;
+ const float probability = initialProbability
+ * powf(initialProbability / endProbability,
+ -1.0f * static_cast<float>(elapsedTime)
+ / static_cast<float>(TIME_STEP_DURATION_IN_SECONDS
+ * (MAX_ELAPSED_TIME_STEP_COUNT + 1)));
+ mTables[tableId][level][timeStepCount] =
+ min(max(static_cast<int>(probability), 1), MAX_PROBABILITY);
}
- const int elapsedTime = timeStepCount * TIME_STEP_DURATION_IN_SECONDS;
- const float probability = initialProbability
- * powf(2.0f, -1.0f * static_cast<float>(elapsedTime)
- / static_cast<float>(TIME_STEP_DURATION_IN_SECONDS
- * (MAX_ELAPSED_TIME_STEP_COUNT + 1)));
- mTable[level][timeStepCount] =
- min(max(static_cast<int>(probability), 1), MAX_COMPUTED_PROBABILITY);
}
}
}
+/* static */ int ForgettingCurveUtils::ProbabilityTable::getBaseProbabilityForLevel(
+ const int tableId, const int level) {
+ if (tableId == WEAK_PROBABILITY_TABLE_ID) {
+ // Max probability is 127.
+ return static_cast<float>(WEAK_MAX_PROBABILITY / (1 << (MAX_LEVEL - level)));
+ } else if (tableId == MODEST_PROBABILITY_TABLE_ID) {
+ // Max probability is 128.
+ return static_cast<float>(MODEST_BASE_PROBABILITY * (level + 1));
+ } else if (tableId == STRONG_PROBABILITY_TABLE_ID) {
+ // Max probability is 140.
+ return static_cast<float>(STRONG_BASE_PROBABILITY * (level + 1));
+ } else if (tableId == AGGRESSIVE_PROBABILITY_TABLE_ID) {
+ // Max probability is 160.
+ return static_cast<float>(AGGRESSIVE_BASE_PROBABILITY * (level + 1));
+ } else {
+ return NOT_A_PROBABILITY;
+ }
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h
index b373534..1a285e5 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h
@@ -24,7 +24,7 @@
namespace latinime {
-class DictionaryHeaderStructurePolicy;
+class HeaderPolicy;
class ForgettingCurveUtils {
public:
@@ -35,12 +35,14 @@
static const HistoricalInfo createUpdatedHistoricalInfo(
const HistoricalInfo *const originalHistoricalInfo, const int newProbability,
- const int timestamp);
+ const int timestamp, const HeaderPolicy *const headerPolicy);
static const HistoricalInfo createHistoricalInfoToSave(
- const HistoricalInfo *const originalHistoricalInfo);
+ const HistoricalInfo *const originalHistoricalInfo,
+ const HeaderPolicy *const headerPolicy);
- static int decodeProbability(const HistoricalInfo *const historicalInfo);
+ static int decodeProbability(const HistoricalInfo *const historicalInfo,
+ const HeaderPolicy *const headerPolicy);
static int getProbability(const int encodedUnigramProbability,
const int encodedBigramProbability);
@@ -48,7 +50,7 @@
static bool needsToKeep(const HistoricalInfo *const historicalInfo);
static bool needsToDecay(const bool mindsBlockByDecay, const int unigramCount,
- const int bigramCount, const DictionaryHeaderStructurePolicy *const headerPolicy);
+ const int bigramCount, const HeaderPolicy *const headerPolicy);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(ForgettingCurveUtils);
@@ -57,21 +59,34 @@
public:
ProbabilityTable();
- int getProbability(const int level, const int elapsedTimeStepCount) const {
- return mTable[level][elapsedTimeStepCount];
+ int getProbability(const int tableId, const int level,
+ const int elapsedTimeStepCount) const {
+ return mTables[tableId][level][elapsedTimeStepCount];
}
private:
DISALLOW_COPY_AND_ASSIGN(ProbabilityTable);
- std::vector<std::vector<int> > mTable;
+ static const int PROBABILITY_TABLE_COUNT;
+ static const int WEAK_PROBABILITY_TABLE_ID;
+ static const int MODEST_PROBABILITY_TABLE_ID;
+ static const int STRONG_PROBABILITY_TABLE_ID;
+ static const int AGGRESSIVE_PROBABILITY_TABLE_ID;
+
+ static const int WEAK_MAX_PROBABILITY;
+ static const int MODEST_BASE_PROBABILITY;
+ static const int STRONG_BASE_PROBABILITY;
+ static const int AGGRESSIVE_BASE_PROBABILITY;
+
+ std::vector<std::vector<std::vector<int> > > mTables;
+
+ static int getBaseProbabilityForLevel(const int tableId, const int level);
};
- static const int MAX_COMPUTED_PROBABILITY;
+ static const int MULTIPLIER_TWO_IN_PROBABILITY_SCALE;
static const int DECAY_INTERVAL_SECONDS;
static const int MAX_LEVEL;
- static const int MAX_COUNT;
static const int MIN_VALID_LEVEL;
static const int TIME_STEP_DURATION_IN_SECONDS;
static const int MAX_ELAPSED_TIME_STEP_COUNT;