Merge "Share decoders among latinime components."
diff --git a/java/AndroidManifest.xml b/java/AndroidManifest.xml
index 88e867f..97f2d89 100644
--- a/java/AndroidManifest.xml
+++ b/java/AndroidManifest.xml
@@ -133,13 +133,6 @@
</intent-filter>
</receiver>
- <receiver android:name=".personalization.DictionaryDecayBroadcastReciever"
- android:exported="false">
- <intent-filter>
- <action android:name="com.android.inputmethod.latin.personalization.DICT_DECAY" />
- </intent-filter>
- </receiver>
-
<receiver android:name=".DictionaryPackInstallBroadcastReceiver" android:exported="false">
<intent-filter>
<action android:name="com.android.inputmethod.dictionarypack.aosp.UNKNOWN_CLIENT" />
diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java
index 6b939cd..9ea1950 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java
@@ -20,7 +20,7 @@
import android.util.Pair;
import com.android.inputmethod.annotations.UsedForTesting;
-import com.android.inputmethod.keyboard.KeyboardLayout;
+import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.latin.common.ComposedData;
import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion;
import com.android.inputmethod.latin.utils.SuggestionResults;
@@ -157,9 +157,9 @@
// TODO: Revise the way to fusion suggestion results.
SuggestionResults getSuggestionResults(final ComposedData composedData,
- final NgramContext ngramContext, final long proximityInfoHandle,
+ final NgramContext ngramContext, @Nonnull final Keyboard keyboard,
final SettingsValuesForSuggestion settingsValuesForSuggestion, final int sessionId,
- final int inputStyle, final KeyboardLayout keyboardLayout);
+ final int inputStyle);
boolean isValidSpellingWord(final String word);
diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java
index e963000..763cfa2 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java
@@ -22,7 +22,7 @@
import android.util.Pair;
import com.android.inputmethod.annotations.UsedForTesting;
-import com.android.inputmethod.keyboard.KeyboardLayout;
+import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.latin.NgramContext.WordInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.common.ComposedData;
@@ -668,12 +668,14 @@
// TODO: Revise the way to fusion suggestion results.
@Override
public SuggestionResults getSuggestionResults(ComposedData composedData,
- NgramContext ngramContext, long proximityInfoHandle,
+ NgramContext ngramContext, @Nonnull final Keyboard keyboard,
SettingsValuesForSuggestion settingsValuesForSuggestion, int sessionId,
- int inputStyle, KeyboardLayout keyboardLayout) {
+ int inputStyle) {
+ long proximityInfoHandle = keyboard.getProximityInfo().getNativeProximityInfo();
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
final SuggestionResults suggestionResults = new SuggestionResults(
- SuggestedWords.MAX_SUGGESTIONS, ngramContext.isBeginningOfSentenceContext());
+ SuggestedWords.MAX_SUGGESTIONS, ngramContext.isBeginningOfSentenceContext(),
+ false /* firstSuggestionExceedsConfidenceThreshold */);
final float[] weightOfLangModelVsSpatialModel =
new float[] { Dictionary.NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL };
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
diff --git a/java/src/com/android/inputmethod/latin/DictionaryFactory.java b/java/src/com/android/inputmethod/latin/DictionaryFactory.java
index 781ab06..49608d8 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryFactory.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryFactory.java
@@ -80,7 +80,7 @@
* @param context The context to contact the dictionary provider, if possible.
* @param f A file address to the dictionary to kill.
*/
- private static void killDictionary(final Context context, final AssetFileAddress f) {
+ public static void killDictionary(final Context context, final AssetFileAddress f) {
if (f.pointsToPhysicalFile()) {
f.deleteUnderlyingFile();
// Warn the dictionary provider if the dictionary came from there.
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index 064d79b..b0fb91b 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -259,7 +259,7 @@
/**
* Check whether GC is needed and run GC if required.
*/
- protected void runGCIfRequired(final boolean mindsBlockByGC) {
+ public void runGCIfRequired(final boolean mindsBlockByGC) {
asyncExecuteTaskWithWriteLock(new Runnable() {
@Override
public void run() {
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index 5b62135..c5c7d2d 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -73,7 +73,6 @@
import com.android.inputmethod.latin.define.DebugFlags;
import com.android.inputmethod.latin.define.ProductionFlags;
import com.android.inputmethod.latin.inputlogic.InputLogic;
-import com.android.inputmethod.latin.personalization.DictionaryDecayBroadcastReciever;
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.settings.SettingsActivity;
@@ -585,7 +584,6 @@
dictDumpFilter.addAction(DictionaryDumpBroadcastReceiver.DICTIONARY_DUMP_INTENT_ACTION);
registerReceiver(mDictionaryDumpBroadcastReceiver, dictDumpFilter);
- DictionaryDecayBroadcastReciever.setUpIntervalAlarmForDictionaryDecaying(this);
StatsUtils.onCreate(mSettings.getCurrent(), mRichImm);
}
@@ -705,7 +703,6 @@
unregisterReceiver(mDictionaryPackInstallReceiver);
unregisterReceiver(mDictionaryDumpBroadcastReceiver);
mStatsUtilsManager.onDestroy(this /* context */);
- DictionaryDecayBroadcastReciever.cancelIntervalAlarmForDictionaryDecaying(this);
super.onDestroy();
}
@@ -1515,9 +1512,8 @@
callback.onGetSuggestedWords(SuggestedWords.getEmptyInstance());
return;
}
- mInputLogic.getSuggestedWords(mSettings.getCurrent(), keyboard.getProximityInfo(),
- mKeyboardSwitcher.getKeyboardShiftMode(), inputStyle, sequenceNumber, callback,
- keyboard.getKeyboardLayout());
+ mInputLogic.getSuggestedWords(mSettings.getCurrent(), keyboard,
+ mKeyboardSwitcher.getKeyboardShiftMode(), inputStyle, sequenceNumber, callback);
}
@Override
diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java
index 832fcbc..f4680fc 100644
--- a/java/src/com/android/inputmethod/latin/Suggest.java
+++ b/java/src/com/android/inputmethod/latin/Suggest.java
@@ -20,8 +20,7 @@
import static com.android.inputmethod.latin.define.DecoderSpecificConstants.SHOULD_AUTO_CORRECT_USING_NON_WHITE_LISTED_SUGGESTION;
-import com.android.inputmethod.keyboard.KeyboardLayout;
-import com.android.inputmethod.keyboard.ProximityInfo;
+import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.common.Constants;
import com.android.inputmethod.latin.common.StringUtils;
@@ -97,19 +96,17 @@
}
public void getSuggestedWords(final WordComposer wordComposer,
- final NgramContext ngramContext, final ProximityInfo proximityInfo,
+ final NgramContext ngramContext, final Keyboard keyboard,
final SettingsValuesForSuggestion settingsValuesForSuggestion,
final boolean isCorrectionEnabled, final int inputStyle, final int sequenceNumber,
- final OnGetSuggestedWordsCallback callback,
- final KeyboardLayout keyboardLayout) {
+ final OnGetSuggestedWordsCallback callback) {
if (wordComposer.isBatchMode()) {
- getSuggestedWordsForBatchInput(wordComposer, ngramContext, proximityInfo,
- settingsValuesForSuggestion, inputStyle, sequenceNumber, callback,
- keyboardLayout);
+ getSuggestedWordsForBatchInput(wordComposer, ngramContext, keyboard,
+ settingsValuesForSuggestion, inputStyle, sequenceNumber, callback);
} else {
- getSuggestedWordsForNonBatchInput(wordComposer, ngramContext, proximityInfo,
+ getSuggestedWordsForNonBatchInput(wordComposer, ngramContext, keyboard,
settingsValuesForSuggestion, inputStyle, isCorrectionEnabled,
- sequenceNumber, callback, keyboardLayout);
+ sequenceNumber, callback);
}
}
@@ -165,11 +162,10 @@
// Retrieves suggestions for non-batch input (typing, recorrection, predictions...)
// and calls the callback function with the suggestions.
private void getSuggestedWordsForNonBatchInput(final WordComposer wordComposer,
- final NgramContext ngramContext, final ProximityInfo proximityInfo,
+ final NgramContext ngramContext, final Keyboard keyboard,
final SettingsValuesForSuggestion settingsValuesForSuggestion,
final int inputStyleIfNotPrediction, final boolean isCorrectionEnabled,
- final int sequenceNumber, final OnGetSuggestedWordsCallback callback,
- final KeyboardLayout keyboardLayout) {
+ final int sequenceNumber, final OnGetSuggestedWordsCallback callback) {
final String typedWordString = wordComposer.getTypedWord();
final int trailingSingleQuotesCount =
StringUtils.getTrailingSingleQuotesCount(typedWordString);
@@ -178,9 +174,8 @@
: typedWordString;
final SuggestionResults suggestionResults = mDictionaryFacilitator.getSuggestionResults(
- wordComposer.getComposedDataSnapshot(), ngramContext,
- proximityInfo.getNativeProximityInfo(), settingsValuesForSuggestion,
- SESSION_ID_TYPING, inputStyleIfNotPrediction, keyboardLayout);
+ wordComposer.getComposedDataSnapshot(), ngramContext, keyboard,
+ settingsValuesForSuggestion, SESSION_ID_TYPING, inputStyleIfNotPrediction);
final Locale mostProbableLocale = mDictionaryFacilitator.getMostProbableLocale();
final ArrayList<SuggestedWordInfo> suggestionsContainer =
getTransformedSuggestedWordInfoList(wordComposer, suggestionResults,
@@ -215,7 +210,8 @@
}
}
- SuggestedWordInfo.removeDups(typedWordString, suggestionsContainer);
+ final int firstOcurrenceOfTypedWordInSuggestions =
+ SuggestedWordInfo.removeDups(typedWordString, suggestionsContainer);
final SuggestedWordInfo whitelistedWordInfo =
getWhitelistedWordInfoOrNull(suggestionsContainer);
@@ -278,7 +274,8 @@
hasAutoCorrection = false;
} else {
final SuggestedWordInfo firstSuggestion = suggestionResults.first();
- if (suggestionResults.mAutocorrectRecommendation) {
+ if (suggestionResults.mFirstSuggestionExceedsConfidenceThreshold
+ && firstOcurrenceOfTypedWordInSuggestions != 0) {
hasAutoCorrection = true;
} else if (!AutoCorrectionUtils.suggestionExceedsThreshold(
firstSuggestion, consideredWord, mAutoCorrectionThreshold)) {
@@ -319,12 +316,12 @@
} else {
inputStyle = inputStyleIfNotPrediction;
}
+
+ final boolean isTypedWordValid = firstOcurrenceOfTypedWordInSuggestions > -1
+ || (!resultsArePredictions && !allowsToBeAutoCorrected);
callback.onGetSuggestedWords(new SuggestedWords(suggestionsList,
suggestionResults.mRawSuggestions, typedWordInfo,
- // TODO: this first argument is lying. If this is a whitelisted word which is an
- // actual word, it says typedWordValid = false, which looks wrong. We should either
- // rename the attribute or change the value.
- !resultsArePredictions && !allowsToBeAutoCorrected /* typedWordValid */,
+ isTypedWordValid,
hasAutoCorrection /* willAutoCorrect */,
false /* isObsoleteSuggestions */, inputStyle, sequenceNumber));
}
@@ -346,15 +343,13 @@
// Retrieves suggestions for the batch input
// and calls the callback function with the suggestions.
private void getSuggestedWordsForBatchInput(final WordComposer wordComposer,
- final NgramContext ngramContext, final ProximityInfo proximityInfo,
+ final NgramContext ngramContext, final Keyboard keyboard,
final SettingsValuesForSuggestion settingsValuesForSuggestion,
final int inputStyle, final int sequenceNumber,
- final OnGetSuggestedWordsCallback callback,
- final KeyboardLayout keyboardLayout) {
+ final OnGetSuggestedWordsCallback callback) {
final SuggestionResults suggestionResults = mDictionaryFacilitator.getSuggestionResults(
- wordComposer.getComposedDataSnapshot(), ngramContext,
- proximityInfo.getNativeProximityInfo(), settingsValuesForSuggestion,
- SESSION_ID_GESTURE, inputStyle, keyboardLayout);
+ wordComposer.getComposedDataSnapshot(), ngramContext, keyboard,
+ settingsValuesForSuggestion, SESSION_ID_GESTURE, inputStyle);
// For transforming words that don't come from a dictionary, because it's our best bet
final Locale defaultLocale = mDictionaryFacilitator.getMostProbableLocale();
final ArrayList<SuggestedWordInfo> suggestionsContainer =
diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java
index 913b63a..3816c08 100644
--- a/java/src/com/android/inputmethod/latin/SuggestedWords.java
+++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java
@@ -17,6 +17,7 @@
package com.android.inputmethod.latin;
import android.text.TextUtils;
+import android.util.Log;
import android.view.inputmethod.CompletionInfo;
import com.android.inputmethod.annotations.UsedForTesting;
@@ -375,31 +376,45 @@
return mWord + " (" + mDebugString + ")";
}
- // This will always remove the higher index if a duplicate is found.
- public static void removeDups(@Nullable final String typedWord,
- @Nonnull ArrayList<SuggestedWordInfo> candidates) {
+ /**
+ * This will always remove the higher index if a duplicate is found.
+ *
+ * @return position of typed word in the candidate list
+ */
+ public static int removeDups(
+ @Nullable final String typedWord,
+ @Nonnull final ArrayList<SuggestedWordInfo> candidates) {
if (candidates.isEmpty()) {
- return;
+ return -1;
}
+ int firstOccurrenceOfWord = -1;
if (!TextUtils.isEmpty(typedWord)) {
- removeSuggestedWordInfoFromList(typedWord, candidates, -1 /* startIndexExclusive */);
+ firstOccurrenceOfWord = removeSuggestedWordInfoFromList(
+ typedWord, candidates, -1 /* startIndexExclusive */);
}
for (int i = 0; i < candidates.size(); ++i) {
- removeSuggestedWordInfoFromList(candidates.get(i).mWord, candidates,
- i /* startIndexExclusive */);
+ removeSuggestedWordInfoFromList(
+ candidates.get(i).mWord, candidates, i /* startIndexExclusive */);
}
+ return firstOccurrenceOfWord;
}
- private static void removeSuggestedWordInfoFromList(
- @Nonnull final String word, @Nonnull final ArrayList<SuggestedWordInfo> candidates,
+ private static int removeSuggestedWordInfoFromList(
+ @Nonnull final String word,
+ @Nonnull final ArrayList<SuggestedWordInfo> candidates,
final int startIndexExclusive) {
+ int firstOccurrenceOfWord = -1;
for (int i = startIndexExclusive + 1; i < candidates.size(); ++i) {
final SuggestedWordInfo previous = candidates.get(i);
if (word.equals(previous.mWord)) {
+ if (firstOccurrenceOfWord == -1) {
+ firstOccurrenceOfWord = i;
+ }
candidates.remove(i);
--i;
}
}
+ return firstOccurrenceOfWord;
}
}
diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
index 57aa591..7907aaa 100644
--- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
+++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
@@ -32,9 +32,8 @@
import com.android.inputmethod.compat.SuggestionSpanUtils;
import com.android.inputmethod.event.Event;
import com.android.inputmethod.event.InputTransaction;
-import com.android.inputmethod.keyboard.KeyboardLayout;
+import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.keyboard.KeyboardSwitcher;
-import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.LastComposedWord;
@@ -2165,9 +2164,8 @@
}
public void getSuggestedWords(final SettingsValues settingsValues,
- final ProximityInfo proximityInfo, final int keyboardShiftMode, final int inputStyle,
- final int sequenceNumber, final OnGetSuggestedWordsCallback callback,
- final KeyboardLayout keyboardLayout) {
+ final Keyboard keyboard, final int keyboardShiftMode, final int inputStyle,
+ final int sequenceNumber, final OnGetSuggestedWordsCallback callback) {
mWordComposer.adviseCapitalizedModeBeforeFetchingSuggestions(
getActualCapsMode(settingsValues, keyboardShiftMode));
mSuggest.getSuggestedWords(mWordComposer,
@@ -2177,11 +2175,11 @@
// a word, it's whatever is *before* the half-committed word in the buffer,
// hence 2; if we aren't, we should just skip whitespace if any, so 1.
mWordComposer.isComposingWord() ? 2 : 1),
- proximityInfo,
+ keyboard,
new SettingsValuesForSuggestion(settingsValues.mBlockPotentiallyOffensive,
settingsValues.mPhraseGestureEnabled),
settingsValues.mAutoCorrectionEnabledPerUserSettings,
- inputStyle, sequenceNumber, callback, keyboardLayout);
+ inputStyle, sequenceNumber, callback);
}
/**
diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
deleted file mode 100644
index 78b51d9..0000000
--- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.inputmethod.latin.personalization;
-
-import android.content.Context;
-
-import com.android.inputmethod.latin.Dictionary;
-import com.android.inputmethod.latin.ExpandableBinaryDictionary;
-import com.android.inputmethod.latin.makedict.DictionaryHeader;
-
-import java.io.File;
-import java.util.Locale;
-import java.util.Map;
-
-/**
- * This class is a base class of a dictionary that supports decaying for the personalized language
- * model.
- */
-public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary {
- private static final boolean DBG_DUMP_ON_CLOSE = false;
-
- /** Any pair being typed or picked */
- public static final int FREQUENCY_FOR_TYPED = 2;
-
- public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED;
- public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY;
-
- protected DecayingExpandableBinaryDictionaryBase(final Context context,
- final String dictName, final Locale locale, final String dictionaryType,
- final File dictFile) {
- super(context, dictName, locale, dictionaryType, dictFile);
- if (mLocale != null && mLocale.toString().length() > 1) {
- reloadDictionaryIfRequired();
- }
- }
-
- @Override
- public void close() {
- if (DBG_DUMP_ON_CLOSE) {
- dumpAllWordsForDebug();
- }
- // Flush pending writes.
- asyncFlushBinaryDictionary();
- super.close();
- }
-
- @Override
- protected Map<String, String> getHeaderAttributeMap() {
- final Map<String, String> attributeMap = super.getHeaderAttributeMap();
- attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
- DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
- attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
- DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
- return attributeMap;
- }
-
- @Override
- protected void loadInitialContentsLocked() {
- // No initial contents.
- }
-
- /* package */ void runGCIfRequired() {
- runGCIfRequired(false /* mindsBlockByGC */);
- }
-
- @Override
- public boolean isValidWord(final String word) {
- // Strings out of this dictionary should not be considered existing words.
- return false;
- }
-}
diff --git a/java/src/com/android/inputmethod/latin/personalization/DictionaryDecayBroadcastReciever.java b/java/src/com/android/inputmethod/latin/personalization/DictionaryDecayBroadcastReciever.java
deleted file mode 100644
index 0c5e5d0..0000000
--- a/java/src/com/android/inputmethod/latin/personalization/DictionaryDecayBroadcastReciever.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.inputmethod.latin.personalization;
-
-import android.app.AlarmManager;
-import android.app.PendingIntent;
-import android.content.BroadcastReceiver;
-import android.content.Context;
-import android.content.Intent;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Broadcast receiver for periodically updating decaying dictionaries.
- */
-public class DictionaryDecayBroadcastReciever extends BroadcastReceiver {
- /**
- * The root domain for the personalization.
- */
- private static final String PERSONALIZATION_DOMAIN =
- "com.android.inputmethod.latin.personalization";
-
- /**
- * The action of the intent to tell the time to decay dictionaries.
- */
- private static final String DICTIONARY_DECAY_INTENT_ACTION =
- PERSONALIZATION_DOMAIN + ".DICT_DECAY";
-
- /**
- * Interval to update for decaying dictionaries.
- */
- static final long DICTIONARY_DECAY_INTERVAL_IN_MILLIS = TimeUnit.MINUTES.toMillis(60);
-
- private static PendingIntent getPendingIntentForDictionaryDecay(final Context context) {
- final Intent updateIntent = new Intent(DICTIONARY_DECAY_INTENT_ACTION);
- updateIntent.setClass(context, DictionaryDecayBroadcastReciever.class);
- return PendingIntent.getBroadcast(context, 0 /* requestCode */,
- updateIntent, PendingIntent.FLAG_CANCEL_CURRENT);
- }
-
- /**
- * Set up interval alarm for dynamic dictionaries.
- */
- public static void setUpIntervalAlarmForDictionaryDecaying(final Context context) {
- final AlarmManager alarmManager =
- (AlarmManager)context.getSystemService(Context.ALARM_SERVICE);
- if (null == alarmManager) {
- return;
- }
- final long alarmTriggerTimeInMillis =
- System.currentTimeMillis() + DICTIONARY_DECAY_INTERVAL_IN_MILLIS;
- alarmManager.setInexactRepeating(AlarmManager.RTC, alarmTriggerTimeInMillis,
- DICTIONARY_DECAY_INTERVAL_IN_MILLIS, getPendingIntentForDictionaryDecay(context));
- }
-
- /**
- * Cancel interval alarm that has been set up.
- */
- public static void cancelIntervalAlarmForDictionaryDecaying(final Context context) {
- final AlarmManager alarmManager =
- (AlarmManager)context.getSystemService(Context.ALARM_SERVICE);
- if (null == alarmManager) {
- return;
- }
- alarmManager.cancel(getPendingIntentForDictionaryDecay(context));
- }
-
- @Override
- public void onReceive(final Context context, final Intent intent) {
- final String action = intent.getAction();
- if (action.equals(DICTIONARY_DECAY_INTENT_ACTION)) {
- PersonalizationHelper.runGCOnAllOpenedUserHistoryDictionaries();
- }
- }
-}
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
index 1c1cb4f..298e46c 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
@@ -26,7 +26,6 @@
import java.lang.ref.SoftReference;
import java.util.Locale;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -68,60 +67,28 @@
}
}
- private static int sCurrentTimestampForTesting = 0;
- public static void currentTimeChangedForTesting(final int currentTimestamp) {
- if (TimeUnit.MILLISECONDS.toSeconds(
- DictionaryDecayBroadcastReciever.DICTIONARY_DECAY_INTERVAL_IN_MILLIS)
- < currentTimestamp - sCurrentTimestampForTesting) {
- runGCOnAllOpenedUserHistoryDictionaries();
- }
- }
-
- public static void runGCOnAllOpenedUserHistoryDictionaries() {
- runGCOnAllDictionariesIfRequired(sLangUserHistoryDictCache);
- }
-
- private static <T extends DecayingExpandableBinaryDictionaryBase>
- void runGCOnAllDictionariesIfRequired(
- final ConcurrentHashMap<String, SoftReference<T>> dictionaryMap) {
- for (final ConcurrentHashMap.Entry<String, SoftReference<T>> entry
- : dictionaryMap.entrySet()) {
- final DecayingExpandableBinaryDictionaryBase dict = entry.getValue().get();
- if (dict != null) {
- dict.runGCIfRequired();
- } else {
- dictionaryMap.remove(entry.getKey());
- }
- }
- }
-
public static void removeAllUserHistoryDictionaries(final Context context) {
- removeAllDictionaries(context, sLangUserHistoryDictCache,
- UserHistoryDictionary.NAME);
- }
-
- private static <T extends DecayingExpandableBinaryDictionaryBase> void removeAllDictionaries(
- final Context context, final ConcurrentHashMap<String, SoftReference<T>> dictionaryMap,
- final String dictNamePrefix) {
- synchronized (dictionaryMap) {
- for (final ConcurrentHashMap.Entry<String, SoftReference<T>> entry
- : dictionaryMap.entrySet()) {
+ synchronized (sLangUserHistoryDictCache) {
+ for (final ConcurrentHashMap.Entry<String, SoftReference<UserHistoryDictionary>> entry
+ : sLangUserHistoryDictCache.entrySet()) {
if (entry.getValue() != null) {
- final DecayingExpandableBinaryDictionaryBase dict = entry.getValue().get();
+ final UserHistoryDictionary dict = entry.getValue().get();
if (dict != null) {
dict.clear();
}
}
}
- dictionaryMap.clear();
+ sLangUserHistoryDictCache.clear();
final File filesDir = context.getFilesDir();
if (filesDir == null) {
Log.e(TAG, "context.getFilesDir() returned null.");
return;
}
- if (!FileUtils.deleteFilteredFiles(filesDir, new DictFilter(dictNamePrefix))) {
- Log.e(TAG, "Cannot remove all existing dictionary files. filesDir: "
- + filesDir.getAbsolutePath() + ", dictNamePrefix: " + dictNamePrefix);
+ final boolean filesDeleted = FileUtils.deleteFilteredFiles(
+ filesDir, new DictFilter(UserHistoryDictionary.NAME));
+ if (!filesDeleted) {
+ Log.e(TAG, "Cannot remove dictionary files. filesDir: " + filesDir.getAbsolutePath()
+ + ", dictNamePrefix: " + UserHistoryDictionary.NAME);
}
}
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
index b6286b2..54ee68d 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
@@ -25,9 +25,11 @@
import com.android.inputmethod.latin.NgramContext;
import com.android.inputmethod.latin.define.DecoderSpecificConstants;
import com.android.inputmethod.latin.define.ProductionFlags;
+import com.android.inputmethod.latin.makedict.DictionaryHeader;
import java.io.File;
import java.util.Locale;
+import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -37,17 +39,16 @@
* auto-correction cancellation or manual picks. This allows the keyboard to adapt to the
* typist over time.
*/
-public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase {
+public class UserHistoryDictionary extends ExpandableBinaryDictionary {
static final String NAME = UserHistoryDictionary.class.getSimpleName();
// TODO: Make this constructor private
UserHistoryDictionary(final Context context, final Locale locale,
@Nullable final String account) {
- super(context,
- getUserHistoryDictName(NAME, locale, null /* dictFile */, account),
- locale,
- Dictionary.TYPE_USER_HISTORY,
- null /* dictFile */);
+ super(context, getUserHistoryDictName(NAME, locale, null /* dictFile */, account), locale, Dictionary.TYPE_USER_HISTORY, null);
+ if (mLocale != null && mLocale.toString().length() > 1) {
+ reloadDictionaryIfRequired();
+ }
}
/**
@@ -103,4 +104,32 @@
userHistoryDictionary.updateEntriesForWord(ngramContext, word,
isValid, 1 /* count */, timestamp);
}
+
+ @Override
+ public void close() {
+ // Flush pending writes.
+ asyncFlushBinaryDictionary();
+ super.close();
+ }
+
+ @Override
+ protected Map<String, String> getHeaderAttributeMap() {
+ final Map<String, String> attributeMap = super.getHeaderAttributeMap();
+ attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
+ DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
+ attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
+ DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
+ return attributeMap;
+ }
+
+ @Override
+ protected void loadInitialContentsLocked() {
+ // No initial contents.
+ }
+
+ @Override
+ public boolean isValidWord(final String word) {
+ // Strings out of this dictionary should not be considered existing words.
+ return false;
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
index 9822c57..f04f093 100644
--- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
+++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
@@ -28,9 +28,7 @@
import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.keyboard.KeyboardId;
-import com.android.inputmethod.keyboard.KeyboardLayout;
import com.android.inputmethod.keyboard.KeyboardLayoutSet;
-import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.DictionaryFacilitatorLruCache;
import com.android.inputmethod.latin.NgramContext;
@@ -48,6 +46,8 @@
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Semaphore;
+import javax.annotation.Nonnull;
+
/**
* Service for spell checking, using LatinIME's dictionaries and mechanisms.
*/
@@ -193,7 +193,7 @@
public SuggestionResults getSuggestionResults(final Locale locale,
final ComposedData composedData, final NgramContext ngramContext,
- final ProximityInfo proximityInfo, final KeyboardLayout keyboardLayout) {
+ @Nonnull final Keyboard keyboard) {
Integer sessionId = null;
mSemaphore.acquireUninterruptibly();
try {
@@ -201,8 +201,8 @@
DictionaryFacilitator dictionaryFacilitatorForLocale =
mDictionaryFacilitatorCache.get(locale);
return dictionaryFacilitatorForLocale.getSuggestionResults(composedData, ngramContext,
- proximityInfo.getNativeProximityInfo(), mSettingsValuesForSuggestion,
- sessionId, SuggestedWords.INPUT_STYLE_TYPING, keyboardLayout);
+ keyboard, mSettingsValuesForSuggestion,
+ sessionId, SuggestedWords.INPUT_STYLE_TYPING);
} finally {
if (sessionId != null) {
mSessionIdPool.add(sessionId);
diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java
index 1c43a4b..5c1915c 100644
--- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java
+++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java
@@ -29,18 +29,11 @@
import com.android.inputmethod.compat.SuggestionsInfoCompatUtils;
import com.android.inputmethod.keyboard.Keyboard;
-import com.android.inputmethod.keyboard.KeyboardLayout;
-import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.NgramContext;
-import com.android.inputmethod.latin.SuggestedWords;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.WordComposer;
-import com.android.inputmethod.latin.common.ComposedData;
import com.android.inputmethod.latin.common.Constants;
-import com.android.inputmethod.latin.common.CoordinateUtils;
-import com.android.inputmethod.latin.common.InputPointers;
import com.android.inputmethod.latin.common.LocaleUtils;
-import com.android.inputmethod.latin.common.ResizableIntArray;
import com.android.inputmethod.latin.common.StringUtils;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.ScriptUtils;
@@ -272,26 +265,20 @@
false /* reportAsTypo */);
}
final Keyboard keyboard = mService.getKeyboardForLocale(mLocale);
+ if (null == keyboard) {
+ Log.d(TAG, "No keyboard for locale: " + mLocale);
+ // If there is no keyboard for this locale, don't do any spell-checking.
+ return AndroidSpellCheckerService.getNotInDictEmptySuggestions(
+ false /* reportAsTypo */);
+ }
final WordComposer composer = new WordComposer();
final int[] codePoints = StringUtils.toCodePointArray(text);
final int[] coordinates;
- final ProximityInfo proximityInfo;
- final KeyboardLayout keyboardLayout;
- if (null == keyboard) {
- coordinates = CoordinateUtils.newCoordinateArray(codePoints.length,
- Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE);
- proximityInfo = null;
- keyboardLayout = null;
- } else {
- coordinates = keyboard.getCoordinates(codePoints);
- proximityInfo = keyboard.getProximityInfo();
- keyboardLayout = keyboard.getKeyboardLayout();
- }
+ coordinates = keyboard.getCoordinates(codePoints);
composer.setComposingWord(codePoints, coordinates);
// TODO: Don't gather suggestions if the limit is <= 0 unless necessary
final SuggestionResults suggestionResults = mService.getSuggestionResults(
- mLocale, composer.getComposedDataSnapshot(), ngramContext, proximityInfo,
- keyboardLayout);
+ mLocale, composer.getComposedDataSnapshot(), ngramContext, keyboard);
final Result result = getResult(capitalizeType, mLocale, suggestionsLimit,
mService.getRecommendedThreshold(), text, suggestionResults);
isInDict = isInDictForAnyCapitalization(text, capitalizeType);
diff --git a/java/src/com/android/inputmethod/latin/utils/BinaryDictionaryUtils.java b/java/src/com/android/inputmethod/latin/utils/BinaryDictionaryUtils.java
index 23ffde2..892b967 100644
--- a/java/src/com/android/inputmethod/latin/utils/BinaryDictionaryUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/BinaryDictionaryUtils.java
@@ -21,7 +21,6 @@
import com.android.inputmethod.latin.common.StringUtils;
import com.android.inputmethod.latin.makedict.DictionaryHeader;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.personalization.PersonalizationHelper;
import java.io.File;
import java.io.IOException;
@@ -123,8 +122,6 @@
*/
@UsedForTesting
public static int setCurrentTimeForTest(final int currentTime) {
- final int currentNativeTimestamp = setCurrentTimeForTestNative(currentTime);
- PersonalizationHelper.currentTimeChangedForTesting(currentNativeTimestamp);
- return currentNativeTimestamp;
+ return setCurrentTimeForTestNative(currentTime);
}
}
diff --git a/java/src/com/android/inputmethod/latin/utils/SuggestionResults.java b/java/src/com/android/inputmethod/latin/utils/SuggestionResults.java
index 10e3994..9813551 100644
--- a/java/src/com/android/inputmethod/latin/utils/SuggestionResults.java
+++ b/java/src/com/android/inputmethod/latin/utils/SuggestionResults.java
@@ -33,21 +33,18 @@
// TODO: Instead of a boolean , we may want to include the context of this suggestion results,
// such as {@link NgramContext}.
public final boolean mIsBeginningOfSentence;
- public final boolean mAutocorrectRecommendation;
+ public final boolean mFirstSuggestionExceedsConfidenceThreshold;
private final int mCapacity;
- public SuggestionResults(final int capacity, final boolean isBeginningOfSentence) {
- this(sSuggestedWordInfoComparator, capacity, isBeginningOfSentence, false);
- }
-
public SuggestionResults(final int capacity, final boolean isBeginningOfSentence,
- final boolean autocorrectRecommendation) {
+ final boolean firstSuggestionExceedsConfidenceThreshold) {
this(sSuggestedWordInfoComparator, capacity, isBeginningOfSentence,
- autocorrectRecommendation);
+ firstSuggestionExceedsConfidenceThreshold);
}
private SuggestionResults(final Comparator<SuggestedWordInfo> comparator, final int capacity,
- final boolean isBeginningOfSentence, final boolean autocorrectRecommendation) {
+ final boolean isBeginningOfSentence,
+ final boolean firstSuggestionExceedsConfidenceThreshold) {
super(comparator);
mCapacity = capacity;
if (ProductionFlags.INCLUDE_RAW_SUGGESTIONS) {
@@ -56,7 +53,7 @@
mRawSuggestions = null;
}
mIsBeginningOfSentence = isBeginningOfSentence;
- mAutocorrectRecommendation = autocorrectRecommendation;
+ mFirstSuggestionExceedsConfidenceThreshold = firstSuggestionExceedsConfidenceThreshold;
}
@Override
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
deleted file mode 100644
index a51558e..0000000
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
+++ /dev/null
@@ -1,818 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.inputmethod.latin;
-
-import android.test.AndroidTestCase;
-import android.test.suitebuilder.annotation.LargeTest;
-import android.util.Pair;
-
-import com.android.inputmethod.latin.NgramContext.WordInfo;
-import com.android.inputmethod.latin.common.CodePointUtils;
-import com.android.inputmethod.latin.common.FileUtils;
-import com.android.inputmethod.latin.common.LocaleUtils;
-import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
-import com.android.inputmethod.latin.makedict.DictDecoder;
-import com.android.inputmethod.latin.makedict.DictionaryHeader;
-import com.android.inputmethod.latin.makedict.FormatSpec;
-import com.android.inputmethod.latin.makedict.FusionDictionary;
-import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
-import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
-import com.android.inputmethod.latin.utils.WordInputEventForPersonalization;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Locale;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-@LargeTest
-public class BinaryDictionaryDecayingTests extends AndroidTestCase {
- private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
- private static final String TEST_LOCALE = "test";
- private static final int DUMMY_PROBABILITY = 0;
- private static final int[] DICT_FORMAT_VERSIONS =
- new int[] { FormatSpec.VERSION402, FormatSpec.VERSION403, FormatSpec.VERSION4_DEV };
- private static final String DICTIONARY_ID = "TestDecayingBinaryDictionary";
-
- private int mCurrentTime = 0;
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- mCurrentTime = 0;
- mDictFilesToBeDeleted.clear();
- setCurrentTimeForTestMode(mCurrentTime);
- }
-
- @Override
- protected void tearDown() throws Exception {
- for (final File dictFile : mDictFilesToBeDeleted) {
- dictFile.delete();
- }
- mDictFilesToBeDeleted.clear();
- stopTestModeInNativeCode();
- super.tearDown();
- }
-
- private static boolean supportsCountBasedNgram(final int formatVersion) {
- return formatVersion >= FormatSpec.VERSION403;
- }
-
- private static boolean supportsNgram(final int formatVersion) {
- return formatVersion >= FormatSpec.VERSION403;
- }
-
- private void onInputWord(final BinaryDictionary binaryDictionary, final String word,
- final boolean isValidWord) {
- binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.EMPTY_PREV_WORDS_INFO,
- word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */);
- }
-
- private void onInputWordWithPrevWord(final BinaryDictionary binaryDictionary, final String word,
- final boolean isValidWord, final String prevWord) {
- binaryDictionary.updateEntriesForWordWithNgramContext(
- new NgramContext(new WordInfo(prevWord)), word, isValidWord, 1 /* count */,
- mCurrentTime /* timestamp */);
- }
-
- private void onInputWordWithPrevWords(final BinaryDictionary binaryDictionary,
- final String word, final boolean isValidWord, final String prevWord,
- final String prevPrevWord) {
- binaryDictionary.updateEntriesForWordWithNgramContext(
- new NgramContext(new WordInfo(prevWord), new WordInfo(prevPrevWord)), word,
- isValidWord, 1 /* count */, mCurrentTime /* timestamp */);
- }
-
- private void onInputWordWithBeginningOfSentenceContext(
- final BinaryDictionary binaryDictionary, final String word, final boolean isValidWord) {
- binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.BEGINNING_OF_SENTENCE,
- word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */);
- }
-
- private static boolean isValidBigram(final BinaryDictionary binaryDictionary,
- final String word0, final String word1) {
- return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1);
- }
-
- private static boolean isValidTrigram(final BinaryDictionary binaryDictionary,
- final String word0, final String word1, final String word2) {
- return binaryDictionary.isValidNgram(
- new NgramContext(new WordInfo(word1), new WordInfo(word0)), word2);
- }
-
- private void forcePassingShortTime(final BinaryDictionary binaryDictionary) {
- // 30 days.
- final int timeToElapse = (int)TimeUnit.SECONDS.convert(30, TimeUnit.DAYS);
- mCurrentTime += timeToElapse;
- setCurrentTimeForTestMode(mCurrentTime);
- binaryDictionary.flushWithGC();
- }
-
- private void forcePassingLongTime(final BinaryDictionary binaryDictionary) {
- // 365 days.
- final int timeToElapse = (int)TimeUnit.SECONDS.convert(365, TimeUnit.DAYS);
- mCurrentTime += timeToElapse;
- setCurrentTimeForTestMode(mCurrentTime);
- binaryDictionary.flushWithGC();
- }
-
- private HashSet<File> mDictFilesToBeDeleted = new HashSet<>();
-
- private File createEmptyDictionaryAndGetFile(final int formatVersion) {
- return createEmptyDictionaryWithAttributeMapAndGetFile(formatVersion,
- new HashMap<String, String>());
- }
-
- private File createEmptyDictionaryWithAttributeMapAndGetFile(final int formatVersion,
- final HashMap<String, String> attributeMap) {
- try {
- final File dictFile = createEmptyVer4DictionaryAndGetFile(formatVersion,
- attributeMap);
- mDictFilesToBeDeleted.add(dictFile);
- return dictFile;
- } catch (final IOException e) {
- fail(e.toString());
- }
- return null;
- }
-
- private File createEmptyVer4DictionaryAndGetFile(final int formatVersion,
- final HashMap<String, String> attributeMap)
- throws IOException {
- final File file = File.createTempFile(DICTIONARY_ID, TEST_DICT_FILE_EXTENSION,
- getContext().getCacheDir());
- FileUtils.deleteRecursively(file);
- attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, DICTIONARY_ID);
- attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY,
- String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
- attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
- DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
- attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
- DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
- if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), formatVersion,
- LocaleUtils.constructLocaleFromString(TEST_LOCALE), attributeMap)) {
- return file;
- }
- throw new IOException("Empty dictionary " + file.getAbsolutePath()
- + " cannot be created. Foramt version: " + formatVersion);
- }
-
- private static BinaryDictionary getBinaryDictionary(final File dictFile) {
- return new BinaryDictionary(dictFile.getAbsolutePath(),
- 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- }
-
- private static int setCurrentTimeForTestMode(final int currentTime) {
- return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime);
- }
-
- private static int stopTestModeInNativeCode() {
- return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
- }
-
- public void testReadDictInJavaSide() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testReadDictInJavaSide(formatVersion);
- }
- }
-
- private void testReadDictInJavaSide(final int formatVersion) {
- setCurrentTimeForTestMode(mCurrentTime);
- final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- onInputWord(binaryDictionary, "a", true /* isValidWord */);
- onInputWord(binaryDictionary, "ab", true /* isValidWord */);
- onInputWordWithPrevWord(binaryDictionary, "aaa", true /* isValidWord */, "a");
- binaryDictionary.flushWithGC();
- binaryDictionary.close();
-
- final DictDecoder dictDecoder =
- BinaryDictIOUtils.getDictDecoder(dictFile, 0, dictFile.length());
- try {
- final FusionDictionary dict =
- dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
- PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, "a");
- assertNotNull(ptNode);
- assertTrue(ptNode.isTerminal());
- assertNotNull(ptNode.getBigram("aaa"));
- ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, "ab");
- assertNotNull(ptNode);
- assertTrue(ptNode.isTerminal());
- ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa");
- assertNotNull(ptNode);
- assertTrue(ptNode.isTerminal());
- } catch (IOException e) {
- fail("IOException while reading dictionary: " + e);
- } catch (UnsupportedFormatException e) {
- fail("Unsupported format: " + e);
- }
- }
-
- public void testControlCurrentTime() {
- final int TEST_COUNT = 1000;
- final long seed = System.currentTimeMillis();
- final Random random = new Random(seed);
- final int startTime = stopTestModeInNativeCode();
- for (int i = 0; i < TEST_COUNT; i++) {
- final int currentTime = random.nextInt(Integer.MAX_VALUE);
- final int currentTimeInNativeCode = setCurrentTimeForTestMode(currentTime);
- assertEquals(currentTime, currentTimeInNativeCode);
- }
- final int endTime = stopTestModeInNativeCode();
- final int MAX_ALLOWED_ELAPSED_TIME = 10;
- assertTrue(startTime <= endTime && endTime <= startTime + MAX_ALLOWED_ELAPSED_TIME);
- }
-
- public void testAddValidAndInvalidWords() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testAddValidAndInvalidWords(formatVersion);
- }
- }
-
- private void testAddValidAndInvalidWords(final int formatVersion) {
- final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
-
- onInputWord(binaryDictionary, "a", false /* isValidWord */);
- assertFalse(binaryDictionary.isValidWord("a"));
- onInputWord(binaryDictionary, "a", false /* isValidWord */);
- onInputWord(binaryDictionary, "a", false /* isValidWord */);
- assertTrue(binaryDictionary.isValidWord("a"));
-
- onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a");
- assertFalse(isValidBigram(binaryDictionary, "a", "b"));
- onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a");
- assertTrue(binaryDictionary.isValidWord("b"));
- assertTrue(isValidBigram(binaryDictionary, "a", "b"));
-
- onInputWordWithPrevWord(binaryDictionary, "c", true /* isValidWord */, "a");
- assertTrue(isValidBigram(binaryDictionary, "a", "c"));
-
- // Add bigrams of not valid unigrams.
- onInputWordWithPrevWord(binaryDictionary, "y", false /* isValidWord */, "x");
- assertFalse(isValidBigram(binaryDictionary, "x", "y"));
- onInputWordWithPrevWord(binaryDictionary, "y", true /* isValidWord */, "x");
- assertFalse(isValidBigram(binaryDictionary, "x", "y"));
-
- if (!supportsNgram(formatVersion)) {
- return;
- }
-
- onInputWordWithPrevWords(binaryDictionary, "c", true /* isValidWord */, "b", "a");
- assertTrue(isValidTrigram(binaryDictionary, "a", "b", "c"));
- assertTrue(isValidBigram(binaryDictionary, "b", "c"));
- onInputWordWithPrevWords(binaryDictionary, "d", false /* isValidWord */, "c", "b");
- assertFalse(isValidTrigram(binaryDictionary, "b", "c", "d"));
- assertFalse(isValidBigram(binaryDictionary, "c", "d"));
-
- onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "b", "a");
- assertTrue(isValidTrigram(binaryDictionary, "a", "b", "cd"));
- }
-
- public void testDecayingProbability() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testDecayingProbability(formatVersion);
- }
- }
-
- private void testDecayingProbability(final int formatVersion) {
- final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
-
- onInputWord(binaryDictionary, "a", true /* isValidWord */);
- assertTrue(binaryDictionary.isValidWord("a"));
- forcePassingShortTime(binaryDictionary);
- if (supportsCountBasedNgram(formatVersion)) {
- // Count based ngram language model doesn't support decaying based on the elapsed time.
- assertTrue(binaryDictionary.isValidWord("a"));
- } else {
- assertFalse(binaryDictionary.isValidWord("a"));
- }
- forcePassingLongTime(binaryDictionary);
- assertFalse(binaryDictionary.isValidWord("a"));
-
- onInputWord(binaryDictionary, "a", true /* isValidWord */);
- onInputWord(binaryDictionary, "a", true /* isValidWord */);
- onInputWord(binaryDictionary, "a", true /* isValidWord */);
- assertTrue(binaryDictionary.isValidWord("a"));
- forcePassingShortTime(binaryDictionary);
- assertTrue(binaryDictionary.isValidWord("a"));
- forcePassingLongTime(binaryDictionary);
- assertFalse(binaryDictionary.isValidWord("a"));
-
- onInputWord(binaryDictionary, "a", true /* isValidWord */);
- onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
- assertTrue(isValidBigram(binaryDictionary, "a", "b"));
- forcePassingShortTime(binaryDictionary);
- if (supportsCountBasedNgram(formatVersion)) {
- assertTrue(isValidBigram(binaryDictionary, "a", "b"));
- } else {
- assertFalse(isValidBigram(binaryDictionary, "a", "b"));
- }
- forcePassingLongTime(binaryDictionary);
- assertFalse(isValidBigram(binaryDictionary, "a", "b"));
-
- onInputWord(binaryDictionary, "a", true /* isValidWord */);
- onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
- onInputWord(binaryDictionary, "a", true /* isValidWord */);
- onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
- onInputWord(binaryDictionary, "a", true /* isValidWord */);
- onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
- assertTrue(isValidBigram(binaryDictionary, "a", "b"));
- forcePassingShortTime(binaryDictionary);
- assertTrue(isValidBigram(binaryDictionary, "a", "b"));
- forcePassingLongTime(binaryDictionary);
- assertFalse(isValidBigram(binaryDictionary, "a", "b"));
-
- if (!supportsNgram(formatVersion)) {
- return;
- }
-
- onInputWord(binaryDictionary, "ab", true /* isValidWord */);
- onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab");
- onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab");
- assertTrue(isValidTrigram(binaryDictionary, "ab", "bc", "cd"));
- forcePassingLongTime(binaryDictionary);
- assertFalse(isValidTrigram(binaryDictionary, "ab", "bc", "cd"));
-
- onInputWord(binaryDictionary, "ab", true /* isValidWord */);
- onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab");
- onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab");
- onInputWord(binaryDictionary, "ab", true /* isValidWord */);
- onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab");
- onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab");
- onInputWord(binaryDictionary, "ab", true /* isValidWord */);
- onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab");
- onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab");
- forcePassingShortTime(binaryDictionary);
- assertTrue(isValidTrigram(binaryDictionary, "ab", "bc", "cd"));
- forcePassingLongTime(binaryDictionary);
- assertFalse(isValidTrigram(binaryDictionary, "ab", "bc", "cd"));
-
- binaryDictionary.close();
- }
-
- public void testAddManyUnigramsToDecayingDict() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testAddManyUnigramsToDecayingDict(formatVersion);
- }
- }
-
- private void testAddManyUnigramsToDecayingDict(final int formatVersion) {
- final int unigramCount = 30000;
- final int unigramTypedCount = 100000;
- final int codePointSetSize = 50;
- final long seed = System.currentTimeMillis();
- final Random random = new Random(seed);
- final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- setCurrentTimeForTestMode(mCurrentTime);
-
- final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
- final ArrayList<String> words = new ArrayList<>();
-
- for (int i = 0; i < unigramCount; i++) {
- final String word = CodePointUtils.generateWord(random, codePointSet);
- words.add(word);
- }
-
- final int maxUnigramCount = Integer.parseInt(
- binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY));
- for (int i = 0; i < unigramTypedCount; i++) {
- final String word = words.get(random.nextInt(words.size()));
- onInputWord(binaryDictionary, word, true /* isValidWord */);
-
- if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
- final int unigramCountBeforeGC =
- Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.UNIGRAM_COUNT_QUERY));
- while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
- forcePassingShortTime(binaryDictionary);
- }
- final int unigramCountAfterGC =
- Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.UNIGRAM_COUNT_QUERY));
- assertTrue(unigramCountBeforeGC > unigramCountAfterGC);
- }
- }
-
- assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.UNIGRAM_COUNT_QUERY)) > 0);
- assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.UNIGRAM_COUNT_QUERY)) <= maxUnigramCount);
- forcePassingLongTime(binaryDictionary);
- assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.UNIGRAM_COUNT_QUERY)));
- }
-
- public void testOverflowUnigrams() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testOverflowUnigrams(formatVersion);
- }
- }
-
- private void testOverflowUnigrams(final int formatVersion) {
- final int unigramCount = 20000;
- final int eachUnigramTypedCount = 2;
- final int strongUnigramTypedCount = 20;
- final int weakUnigramTypedCount = 1;
- final int codePointSetSize = 50;
- final long seed = System.currentTimeMillis();
- final Random random = new Random(seed);
- final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- setCurrentTimeForTestMode(mCurrentTime);
- final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
-
- final String strong = "strong";
- final String weak = "weak";
- for (int j = 0; j < strongUnigramTypedCount; j++) {
- onInputWord(binaryDictionary, strong, true /* isValidWord */);
- }
- for (int j = 0; j < weakUnigramTypedCount; j++) {
- onInputWord(binaryDictionary, weak, true /* isValidWord */);
- }
- assertTrue(binaryDictionary.isValidWord(strong));
- assertTrue(binaryDictionary.isValidWord(weak));
-
- for (int i = 0; i < unigramCount; i++) {
- final String word = CodePointUtils.generateWord(random, codePointSet);
- for (int j = 0; j < eachUnigramTypedCount; j++) {
- onInputWord(binaryDictionary, word, true /* isValidWord */);
- }
- if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
- final int unigramCountBeforeGC =
- Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.UNIGRAM_COUNT_QUERY));
- assertTrue(binaryDictionary.isValidWord(strong));
- assertTrue(binaryDictionary.isValidWord(weak));
- binaryDictionary.flushWithGC();
- final int unigramCountAfterGC =
- Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.UNIGRAM_COUNT_QUERY));
- assertTrue(unigramCountBeforeGC > unigramCountAfterGC);
- assertFalse(binaryDictionary.isValidWord(weak));
- assertTrue(binaryDictionary.isValidWord(strong));
- break;
- }
- }
- }
-
- public void testAddManyBigramsToDecayingDict() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testAddManyBigramsToDecayingDict(formatVersion);
- }
- }
-
- private void testAddManyBigramsToDecayingDict(final int formatVersion) {
- final int maxUnigramCount = 5000;
- final int maxBigramCount = 10000;
- final HashMap<String, String> attributeMap = new HashMap<>();
- attributeMap.put(DictionaryHeader.MAX_UNIGRAM_COUNT_KEY, String.valueOf(maxUnigramCount));
- attributeMap.put(DictionaryHeader.MAX_BIGRAM_COUNT_KEY, String.valueOf(maxBigramCount));
-
- final int unigramCount = 5000;
- final int bigramCount = 30000;
- final int bigramTypedCount = 100000;
- final int codePointSetSize = 50;
- final long seed = System.currentTimeMillis();
- final Random random = new Random(seed);
-
- setCurrentTimeForTestMode(mCurrentTime);
- final File dictFile = createEmptyDictionaryWithAttributeMapAndGetFile(formatVersion,
- attributeMap);
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
-
- final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
- final ArrayList<String> words = new ArrayList<>();
- final ArrayList<Pair<String, String>> bigrams = new ArrayList<>();
-
- for (int i = 0; i < unigramCount; ++i) {
- final String word = CodePointUtils.generateWord(random, codePointSet);
- words.add(word);
- }
- for (int i = 0; i < bigramCount; ++i) {
- final int word0Index = random.nextInt(words.size());
- int word1Index = random.nextInt(words.size() - 1);
- if (word1Index >= word0Index) {
- word1Index += 1;
- }
- final String word0 = words.get(word0Index);
- final String word1 = words.get(word1Index);
- final Pair<String, String> bigram = new Pair<>(word0, word1);
- bigrams.add(bigram);
- }
-
- for (int i = 0; i < bigramTypedCount; ++i) {
- final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size()));
- onInputWord(binaryDictionary, bigram.first, true /* isValidWord */);
- onInputWordWithPrevWord(binaryDictionary, bigram.second, true /* isValidWord */,
- bigram.first);
-
- if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
- final int bigramCountBeforeGC =
- Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.BIGRAM_COUNT_QUERY));
- while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
- forcePassingShortTime(binaryDictionary);
- }
- final int bigramCountAfterGC =
- Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.BIGRAM_COUNT_QUERY));
- assertTrue(bigramCountBeforeGC > bigramCountAfterGC);
- }
- }
- forcePassingShortTime(binaryDictionary);
- assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.BIGRAM_COUNT_QUERY)) > 0);
- assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.BIGRAM_COUNT_QUERY)) <= maxBigramCount);
- forcePassingLongTime(binaryDictionary);
- assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.BIGRAM_COUNT_QUERY)));
- }
-
- public void testOverflowBigrams() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testOverflowBigrams(formatVersion);
- }
- }
-
- private void testOverflowBigrams(final int formatVersion) {
- final int maxUnigramCount = 5000;
- final int maxBigramCount = 10000;
- final HashMap<String, String> attributeMap = new HashMap<>();
- attributeMap.put(DictionaryHeader.MAX_UNIGRAM_COUNT_KEY, String.valueOf(maxUnigramCount));
- attributeMap.put(DictionaryHeader.MAX_BIGRAM_COUNT_KEY, String.valueOf(maxBigramCount));
-
- final int bigramCount = 20000;
- final int unigramCount = 1000;
- final int unigramTypedCount = 20;
- final int eachBigramTypedCount = 2;
- final int strongBigramTypedCount = 20;
- final int weakBigramTypedCount = 1;
- final int codePointSetSize = 50;
- final long seed = System.currentTimeMillis();
- final Random random = new Random(seed);
- setCurrentTimeForTestMode(mCurrentTime);
- final File dictFile = createEmptyDictionaryWithAttributeMapAndGetFile(formatVersion,
- attributeMap);
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
-
- final ArrayList<String> words = new ArrayList<>();
- for (int i = 0; i < unigramCount; i++) {
- final String word = CodePointUtils.generateWord(random, codePointSet);
- words.add(word);
- for (int j = 0; j < unigramTypedCount; j++) {
- onInputWord(binaryDictionary, word, true /* isValidWord */);
- }
- }
- final String strong = "strong";
- final String weak = "weak";
- final String target = "target";
- for (int j = 0; j < unigramTypedCount; j++) {
- onInputWord(binaryDictionary, weak, true /* isValidWord */);
- onInputWord(binaryDictionary, strong, true /* isValidWord */);
- }
- binaryDictionary.flushWithGC();
- for (int j = 0; j < strongBigramTypedCount; j++) {
- onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, strong);
- }
- for (int j = 0; j < weakBigramTypedCount; j++) {
- onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, weak);
- }
- assertTrue(isValidBigram(binaryDictionary, strong, target));
- assertTrue(isValidBigram(binaryDictionary, weak, target));
-
- for (int i = 0; i < bigramCount; i++) {
- final int word0Index = random.nextInt(words.size());
- final String word0 = words.get(word0Index);
- final int index = random.nextInt(words.size() - 1);
- final int word1Index = (index >= word0Index) ? index + 1 : index;
- final String word1 = words.get(word1Index);
-
- for (int j = 0; j < eachBigramTypedCount; j++) {
- onInputWordWithPrevWord(binaryDictionary, word1, true /* isValidWord */, word0);
- }
- if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
- final int bigramCountBeforeGC =
- Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.BIGRAM_COUNT_QUERY));
- binaryDictionary.flushWithGC();
- final int bigramCountAfterGC =
- Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
- BinaryDictionary.BIGRAM_COUNT_QUERY));
- assertTrue(bigramCountBeforeGC > bigramCountAfterGC);
- assertTrue(isValidBigram(binaryDictionary, strong, target));
- assertFalse(isValidBigram(binaryDictionary, weak, target));
- break;
- }
- }
- }
-
- public void testDictMigration() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testDictMigration(FormatSpec.VERSION4_ONLY_FOR_TESTING, formatVersion);
- }
- }
-
- private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) {
- setCurrentTimeForTestMode(mCurrentTime);
- final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion);
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- onInputWord(binaryDictionary, "aaa", true /* isValidWord */);
- assertTrue(binaryDictionary.isValidWord("aaa"));
- onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
- onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
- onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
- onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
- onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
-
- onInputWordWithPrevWord(binaryDictionary, "abc", true /* isValidWord */, "aaa");
- assertTrue(isValidBigram(binaryDictionary, "aaa", "abc"));
- onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa");
- assertFalse(binaryDictionary.isValidWord("bbb"));
- assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb"));
-
- if (supportsNgram(toFormatVersion)) {
- onInputWordWithPrevWords(binaryDictionary, "xyz", true, "abc", "aaa");
- assertTrue(isValidTrigram(binaryDictionary, "aaa", "abc", "xyz"));
- onInputWordWithPrevWords(binaryDictionary, "def", false, "abc", "aaa");
- assertFalse(isValidTrigram(binaryDictionary, "aaa", "abc", "def"));
- }
-
- assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion());
- assertTrue(binaryDictionary.migrateTo(toFormatVersion));
- assertTrue(binaryDictionary.isValidDictionary());
- assertEquals(toFormatVersion, binaryDictionary.getFormatVersion());
- assertTrue(binaryDictionary.isValidWord("aaa"));
- assertFalse(binaryDictionary.isValidWord("bbb"));
- if (supportsCountBasedNgram(toFormatVersion)) {
- assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc"));
- onInputWord(binaryDictionary, "bbb", false /* isValidWord */);
- assertTrue(binaryDictionary.isValidWord("bbb"));
- }
- assertTrue(isValidBigram(binaryDictionary, "aaa", "abc"));
- assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb"));
- if (supportsCountBasedNgram(toFormatVersion)) {
- onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa");
- assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
- }
- if (supportsNgram(toFormatVersion)) {
- assertTrue(isValidTrigram(binaryDictionary, "aaa", "abc", "xyz"));
- assertFalse(isValidTrigram(binaryDictionary, "aaa", "abc", "def"));
- onInputWordWithPrevWords(binaryDictionary, "def", false, "abc", "aaa");
- assertTrue(isValidTrigram(binaryDictionary, "aaa", "abc", "def"));
- }
-
- binaryDictionary.close();
- }
-
- public void testBeginningOfSentence() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testBeginningOfSentence(formatVersion);
- }
- }
-
- private void testBeginningOfSentence(final int formatVersion) {
- setCurrentTimeForTestMode(mCurrentTime);
- final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
-
- binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY,
- true /* isBeginningOfSentence */, true /* isNotAWord */,
- false /* isPossiblyOffensive */, mCurrentTime);
- final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE;
- onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
- assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
- onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
- assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
- onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
- onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
- assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
- onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
- assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
- assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
- forcePassingLongTime(binaryDictionary);
- assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
- assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
- onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
- onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
- onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
- onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
- assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
- assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
- binaryDictionary.close();
- }
-
- public void testRemoveUnigrams() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testRemoveUnigrams(formatVersion);
- }
- }
-
- private void testRemoveUnigrams(final int formatVersion) {
- final int unigramInputCount = 20;
- setCurrentTimeForTestMode(mCurrentTime);
- final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
-
- onInputWord(binaryDictionary, "aaa", false /* isValidWord */);
- assertFalse(binaryDictionary.isValidWord("aaa"));
- for (int i = 0; i < unigramInputCount; i++) {
- onInputWord(binaryDictionary, "aaa", false /* isValidWord */);
- }
- assertTrue(binaryDictionary.isValidWord("aaa"));
- assertTrue(binaryDictionary.removeUnigramEntry("aaa"));
- assertFalse(binaryDictionary.isValidWord("aaa"));
- onInputWord(binaryDictionary, "aaa", false /* isValidWord */);
- assertFalse(binaryDictionary.isValidWord("aaa"));
- onInputWord(binaryDictionary, "aaa", false /* isValidWord */);
- assertTrue(binaryDictionary.isValidWord("aaa"));
- assertTrue(binaryDictionary.removeUnigramEntry("aaa"));
- assertFalse(binaryDictionary.isValidWord("aaa"));
- binaryDictionary.close();
- }
-
- public void testUpdateEntriesForInputEvents() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testUpdateEntriesForInputEvents(formatVersion);
- }
- }
-
- private void testUpdateEntriesForInputEvents(final int formatVersion) {
- setCurrentTimeForTestMode(mCurrentTime);
- final int codePointSetSize = 20;
- final int EVENT_COUNT = 1000;
- final double CONTINUE_RATE = 0.9;
- final long seed = System.currentTimeMillis();
- final Random random = new Random(seed);
- final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
-
- final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
- final ArrayList<String> unigrams = new ArrayList<>();
- final ArrayList<Pair<String, String>> bigrams = new ArrayList<>();
- final ArrayList<Pair<Pair<String, String>, String>> trigrams = new ArrayList<>();
-
- final WordInputEventForPersonalization[] inputEvents =
- new WordInputEventForPersonalization[EVENT_COUNT];
- NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
- int prevWordCount = 0;
- for (int i = 0; i < inputEvents.length; i++) {
- final String word = CodePointUtils.generateWord(random, codePointSet);
- inputEvents[i] = new WordInputEventForPersonalization(word, ngramContext, mCurrentTime);
- unigrams.add(word);
- if (prevWordCount >= 2) {
- final Pair<String, String> prevWordsPair = bigrams.get(bigrams.size() - 1);
- trigrams.add(new Pair<>(prevWordsPair, word));
- }
- if (prevWordCount >= 1) {
- bigrams.add(new Pair<>(ngramContext.getNthPrevWord(1 /* n */).toString(), word));
- }
- if (random.nextDouble() > CONTINUE_RATE) {
- ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
- prevWordCount = 0;
- } else {
- ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
- prevWordCount++;
- }
- }
- final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- binaryDictionary.updateEntriesForInputEvents(inputEvents);
-
- for (final String word : unigrams) {
- assertTrue(binaryDictionary.isInDictionary(word));
- }
- for (final Pair<String, String> bigram : bigrams) {
- assertTrue(isValidBigram(binaryDictionary, bigram.first, bigram.second));
- }
- if (!supportsNgram(formatVersion)) {
- return;
- }
- for (final Pair<Pair<String, String>, String> trigram : trigrams) {
- assertTrue(isValidTrigram(binaryDictionary, trigram.first.first, trigram.first.second,
- trigram.second));
- }
- }
-}
diff --git a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java
index f658d72..657cec8 100644
--- a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java
+++ b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java
@@ -59,6 +59,14 @@
SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */);
}
+ private static ArrayList<SuggestedWordInfo> createCorrectionWordInfos(final String... words) {
+ final ArrayList<SuggestedWordInfo> infos = new ArrayList<>();
+ for (final String word : words) {
+ infos.add(createCorrectionWordInfo(word));
+ }
+ return infos;
+ }
+
// Helper for testGetTransformedWordInfo
private static SuggestedWordInfo transformWordInfo(final String info,
final int trailingSingleQuotesCount) {
@@ -72,6 +80,30 @@
return returnedWordInfo;
}
+ public void testRemoveDupesNoDupes() {
+ final ArrayList<SuggestedWordInfo> infos = createCorrectionWordInfos("a", "c");
+ assertEquals(-1, SuggestedWordInfo.removeDups("b", infos));
+ assertEquals(2, infos.size());
+ }
+
+ public void testRemoveDupesTypedWordNotDupe() {
+ final ArrayList<SuggestedWordInfo> infos = createCorrectionWordInfos("a", "a", "c");
+ assertEquals(-1, SuggestedWordInfo.removeDups("b", infos));
+ assertEquals(2, infos.size());
+ }
+
+ public void testRemoveDupesTypedWordOnlyDupe() {
+ final ArrayList<SuggestedWordInfo> infos = createCorrectionWordInfos("a", "b", "c");
+ assertEquals(1, SuggestedWordInfo.removeDups("b", infos));
+ assertEquals(2, infos.size());
+ }
+
+ public void testRemoveDupesTypedWordNotOnlyDupe() {
+ final ArrayList<SuggestedWordInfo> infos = createCorrectionWordInfos("a", "b", "b", "c");
+ assertEquals(1, SuggestedWordInfo.removeDups("b", infos));
+ assertEquals(2, infos.size());
+ }
+
public void testGetTransformedSuggestedWordInfo() {
SuggestedWordInfo result = transformWordInfo("word", 0);
assertEquals(result.mWord, "word");
diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
index e1260f6..559f286 100644
--- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
@@ -21,15 +21,11 @@
import android.util.Log;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
-import com.android.inputmethod.latin.NgramContext;
-import com.android.inputmethod.latin.NgramContext.WordInfo;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import java.io.File;
-import java.util.List;
import java.util.Locale;
import java.util.Random;
-import java.util.concurrent.TimeUnit;
/**
* Unit tests for UserHistoryDictionary
@@ -87,20 +83,6 @@
setCurrentTimeForTestMode(mCurrentTime);
}
- private void forcePassingShortTime() {
- // 3 days.
- final int timeToElapse = (int)TimeUnit.DAYS.toSeconds(3);
- mCurrentTime += timeToElapse;
- setCurrentTimeForTestMode(mCurrentTime);
- }
-
- private void forcePassingLongTime() {
- // 365 days.
- final int timeToElapse = (int)TimeUnit.DAYS.toSeconds(365);
- mCurrentTime += timeToElapse;
- setCurrentTimeForTestMode(mCurrentTime);
- }
-
private static int setCurrentTimeForTestMode(final int currentTime) {
return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime);
}
@@ -228,46 +210,4 @@
numberOfWords, random, true /* checksContents */, mCurrentTime));
assertDictionaryExists(dict, dictFile);
}
-
- public void testDecaying() {
- doTestDecaying(TEST_ACCOUNT);
- }
-
- public void testDecaying_NullAccount() {
- doTestDecaying(null /* testAccount */);
- }
-
- private void doTestDecaying(final String testAccount) {
- final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("decaying");
- final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
- getContext(), dummyLocale, testAccount);
- resetCurrentTimeForTestMode();
- clearHistory(dict);
- dict.waitAllTasksForTests();
-
- final int numberOfWords = 5000;
- final Random random = new Random(123456);
- final List<String> words = UserHistoryDictionaryTestsHelper.generateWords(numberOfWords,
- random);
- NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
- for (final String word : words) {
- UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime);
- ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
- dict.waitAllTasksForTests();
- assertTrue(dict.isInDictionary(word));
- }
- forcePassingShortTime();
- dict.runGCIfRequired();
- dict.waitAllTasksForTests();
- for (final String word : words) {
- assertTrue(dict.isInDictionary(word));
- }
- // Long term decay results in words removed from the dictionary.
- forcePassingLongTime();
- dict.runGCIfRequired();
- dict.waitAllTasksForTests();
- for (final String word : words) {
- assertFalse(dict.isInDictionary(word));
- }
- }
}
\ No newline at end of file