Merge "Create more aggressive probability tables."
diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java
index 2dba713..cd18a6b 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java
@@ -36,6 +36,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
@@ -142,7 +143,8 @@
@UsedForTesting
public DictionaryFacilitatorForSuggest(final Context context, final Locale locale,
- final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles) {
+ final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles,
+ final Map<String, Map<String, String>> additionalDictAttributes) {
mContext = context;
mLocale = locale;
mLatchForWaitingLoadingMainDictionary = new CountDownLatch(0);
@@ -159,6 +161,10 @@
userHistoryDictionary.reloadDictionaryIfRequired();
userHistoryDictionary.waitAllTasksForTests();
setUserHistoryDictionary(userHistoryDictionary);
+ if (additionalDictAttributes.containsKey(dictType)) {
+ userHistoryDictionary.clearAndFlushDictionaryWithAdditionalAttributes(
+ additionalDictAttributes.get(dictType));
+ }
} else if (dictType.equals(Dictionary.TYPE_PERSONALIZATION)) {
final PersonalizationDictionary personalizationDictionary =
PersonalizationHelper.getPersonalizationDictionary(context, locale);
@@ -167,6 +173,10 @@
personalizationDictionary.reloadDictionaryIfRequired();
personalizationDictionary.waitAllTasksForTests();
setPersonalizationDictionary(personalizationDictionary);
+ if (additionalDictAttributes.containsKey(dictType)) {
+ personalizationDictionary.clearAndFlushDictionaryWithAdditionalAttributes(
+ additionalDictAttributes.get(dictType));
+ }
} else if (dictType.equals(Dictionary.TYPE_USER)) {
final File file = dictionaryFiles.get(dictType);
final UserBinaryDictionary userDictionary = new UserBinaryDictionary(
diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
index d3734d6..8c358cd 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
@@ -50,6 +50,7 @@
public final Locale mLocale;
private final String mDictName;
+ private Map<String, String> mAdditionalAttributeMap = null;
protected DecayingExpandableBinaryDictionaryBase(final Context context,
final String dictName, final Locale locale, final String dictionaryType,
@@ -78,7 +79,10 @@
@Override
protected Map<String, String> getHeaderAttributeMap() {
- HashMap<String, String> attributeMap = new HashMap<String, String>();
+ final Map<String, String> attributeMap = new HashMap<String, String>();
+ if (mAdditionalAttributeMap != null) {
+ attributeMap.putAll(mAdditionalAttributeMap);
+ }
attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
@@ -152,6 +156,13 @@
asyncFlushBinaryDictionary();
}
+ @UsedForTesting
+ public void clearAndFlushDictionaryWithAdditionalAttributes(
+ final Map<String, String> attributeMap) {
+ mAdditionalAttributeMap = attributeMap;
+ clearAndFlushDictionary();
+ }
+
/* package */ void decayIfNeeded() {
runGCIfRequired(false /* mindsBlockByGC */);
}
diff --git a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java
index a1d6415..562ff9e 100644
--- a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java
+++ b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java
@@ -37,9 +37,9 @@
// non-0. Thus, it's not meaningful to compare 10, 100, and so on.
// TODO: Revise the logic in ForgettingCurveUtils in native code.
private static final int UNIGRAM_PROBABILITY_FOR_VALID_WORD = 100;
- private static final int UNIGRAM_PROBABILITY_FOR_OOV_WORD = 10;
- private static final int BIGRAM_PROBABILITY_FOR_VALID_WORD = 0;
- private static final int BIGRAM_PROBABILITY_FOR_OOV_WORD = 0;
+ private static final int UNIGRAM_PROBABILITY_FOR_OOV_WORD = Dictionary.NOT_A_PROBABILITY;
+ private static final int BIGRAM_PROBABILITY_FOR_VALID_WORD = 10;
+ private static final int BIGRAM_PROBABILITY_FOR_OOV_WORD = Dictionary.NOT_A_PROBABILITY;
public final String mTargetWord;
public final int[] mWord0;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
index 0bfebf3..a9baafe 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
@@ -30,6 +30,7 @@
const int ForgettingCurveUtils::MAX_BIGRAM_COUNT = 12000;
const int ForgettingCurveUtils::MAX_BIGRAM_COUNT_AFTER_GC = 10000;
+const int ForgettingCurveUtils::MULTIPLIER_TWO_IN_PROBABILITY_SCALE = 8;
const int ForgettingCurveUtils::MAX_COMPUTED_PROBABILITY = 127;
const int ForgettingCurveUtils::DECAY_INTERVAL_SECONDS = 2 * 60 * 60;
@@ -84,7 +85,9 @@
} else if (bigramProbability == NOT_A_PROBABILITY) {
return min(backoff(unigramProbability), MAX_COMPUTED_PROBABILITY);
} else {
- return min(max(unigramProbability, bigramProbability), MAX_COMPUTED_PROBABILITY);
+ // TODO: Investigate better way to handle bigram probability.
+ return min(max(unigramProbability, bigramProbability + MULTIPLIER_TWO_IN_PROBABILITY_SCALE),
+ MAX_COMPUTED_PROBABILITY);
}
}
@@ -137,11 +140,8 @@
// See comments in ProbabilityUtils::backoff().
/* static */ int ForgettingCurveUtils::backoff(const int unigramProbability) {
- if (unigramProbability == NOT_A_PROBABILITY) {
- return NOT_A_PROBABILITY;
- } else {
- return max(unigramProbability - 8, 0);
- }
+ // See TODO comments in ForgettingCurveUtils::getProbability().
+ return unigramProbability;
}
/* static */ int ForgettingCurveUtils::getElapsedTimeStepCount(const int timestamp) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h
index dae905e..1eb6fb7 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h
@@ -82,6 +82,7 @@
static int getBaseProbabilityForLevel(const int tableId, const int level);
};
+ static const int MULTIPLIER_TWO_IN_PROBABILITY_SCALE;
static const int MAX_COMPUTED_PROBABILITY;
static const int DECAY_INTERVAL_SECONDS;