Merge "Add unittest for SuggestionSpanUtils"
diff --git a/dictionaries/de_wordlist.combined.gz b/dictionaries/de_wordlist.combined.gz
index 400718d..803211c 100644
--- a/dictionaries/de_wordlist.combined.gz
+++ b/dictionaries/de_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/en_GB_wordlist.combined.gz b/dictionaries/en_GB_wordlist.combined.gz
index bf637e9..1fa9b85 100644
--- a/dictionaries/en_GB_wordlist.combined.gz
+++ b/dictionaries/en_GB_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/en_US_wordlist.combined.gz b/dictionaries/en_US_wordlist.combined.gz
index 9ea04b1..2e039ff 100644
--- a/dictionaries/en_US_wordlist.combined.gz
+++ b/dictionaries/en_US_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/en_wordlist.combined.gz b/dictionaries/en_wordlist.combined.gz
index 87a8633..e845346 100644
--- a/dictionaries/en_wordlist.combined.gz
+++ b/dictionaries/en_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/es_wordlist.combined.gz b/dictionaries/es_wordlist.combined.gz
index 8bb2102..3391e64 100644
--- a/dictionaries/es_wordlist.combined.gz
+++ b/dictionaries/es_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/ro_wordlist.combined.gz b/dictionaries/ro_wordlist.combined.gz
index 829abd9..5307855 100644
--- a/dictionaries/ro_wordlist.combined.gz
+++ b/dictionaries/ro_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/ru_wordlist.combined.gz b/dictionaries/ru_wordlist.combined.gz
index 7794f7b..401ad08 100644
--- a/dictionaries/ru_wordlist.combined.gz
+++ b/dictionaries/ru_wordlist.combined.gz
Binary files differ
diff --git a/java/res/raw/main_de.dict b/java/res/raw/main_de.dict
index 3cbf710..45b2883 100644
--- a/java/res/raw/main_de.dict
+++ b/java/res/raw/main_de.dict
Binary files differ
diff --git a/java/res/raw/main_en.dict b/java/res/raw/main_en.dict
index d631d6f..5bbb857 100644
--- a/java/res/raw/main_en.dict
+++ b/java/res/raw/main_en.dict
Binary files differ
diff --git a/java/res/raw/main_es.dict b/java/res/raw/main_es.dict
index 8c42fda..fae1318 100644
--- a/java/res/raw/main_es.dict
+++ b/java/res/raw/main_es.dict
Binary files differ
diff --git a/java/res/raw/main_ru.dict b/java/res/raw/main_ru.dict
index 9fdaf31..8d15769 100644
--- a/java/res/raw/main_ru.dict
+++ b/java/res/raw/main_ru.dict
Binary files differ
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index 6f3c48c..10dea74 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -42,6 +42,8 @@
import java.util.Locale;
import java.util.Map;
+import javax.annotation.Nonnull;
+
/**
* Implements a static, compacted, binary dictionary of standard words.
*/
@@ -199,6 +201,7 @@
int[] word, int probability, int timestamp);
private static native boolean removeNgramEntryNative(long dict,
int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word);
+ // TODO: Rename to updateEntriesForWordWithNgramContextNative.
private static native boolean updateCounterNative(long dict,
int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray,
int[] word, boolean isValidWord, int count, int timestamp);
@@ -493,6 +496,24 @@
return true;
}
+ // Update entries for the word occurrence with the ngramContext.
+ public boolean updateEntriesForWordWithNgramContext(@Nonnull final NgramContext ngramContext,
+ final String word, final boolean isValidWord, final int count, final int timestamp) {
+ if (TextUtils.isEmpty(word)) {
+ return false;
+ }
+ final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][];
+ final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()];
+ ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray);
+ final int[] wordCodePoints = StringUtils.toCodePointArray(word);
+ if (!updateCounterNative(mNativeDict, prevWordCodePointArrays,
+ isBeginningOfSentenceArray, wordCodePoints, isValidWord, count, timestamp)) {
+ return false;
+ }
+ mHasUpdated = true;
+ return true;
+ }
+
@UsedForTesting
public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) {
if (!isValidDictionary()) return;
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index 1bdadc3..d24f80a 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -46,6 +46,8 @@
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.annotation.Nonnull;
+
/**
* Abstract base class for an expandable dictionary that can be created and updated dynamically
* during runtime. When updated it automatically generates a new binary dictionary to handle future
@@ -292,13 +294,9 @@
}
}
- /**
- * Adds unigram information of a word to the dictionary. May overwrite an existing entry.
- */
- public void addUnigramEntryWithCheckingDistracter(final String word, final int frequency,
- final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
- final boolean isBlacklisted, final int timestamp,
- final DistracterFilter distracterFilter) {
+ private void updateDictionaryWithWriteLockIfWordIsNotADistracter(
+ @Nonnull final Runnable updateTask,
+ @Nonnull final String word, @Nonnull final DistracterFilter distracterFilter) {
reloadDictionaryIfRequired();
asyncPreCheckAndExecuteTaskWithWriteLock(
new Callable<Boolean>() {
@@ -315,12 +313,27 @@
return;
}
runGCIfRequiredLocked(true /* mindsBlockByGC */);
- addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq,
- isNotAWord, isBlacklisted, timestamp);
+ updateTask.run();
}
});
}
+ /**
+ * Adds unigram information of a word to the dictionary. May overwrite an existing entry.
+ */
+ public void addUnigramEntryWithCheckingDistracter(final String word, final int frequency,
+ final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
+ final boolean isBlacklisted, final int timestamp,
+ @Nonnull final DistracterFilter distracterFilter) {
+ updateDictionaryWithWriteLockIfWordIsNotADistracter(new Runnable() {
+ @Override
+ public void run() {
+ addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq,
+ isNotAWord, isBlacklisted, timestamp);
+ }
+ }, word, distracterFilter);
+ }
+
protected void addUnigramLocked(final String word, final int frequency,
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) {
@@ -354,7 +367,7 @@
/**
* Adds n-gram information of a word to the dictionary. May overwrite an existing entry.
*/
- public void addNgramEntry(final NgramContext ngramContext, final String word,
+ public void addNgramEntry(@Nonnull final NgramContext ngramContext, final String word,
final int frequency, final int timestamp) {
reloadDictionaryIfRequired();
asyncExecuteTaskWithWriteLock(new Runnable() {
@@ -369,7 +382,7 @@
});
}
- protected void addNgramEntryLocked(final NgramContext ngramContext, final String word,
+ protected void addNgramEntryLocked(@Nonnull final NgramContext ngramContext, final String word,
final int frequency, final int timestamp) {
if (!mBinaryDictionary.addNgramEntry(ngramContext, word, frequency, timestamp)) {
if (DEBUG) {
@@ -383,7 +396,8 @@
* Dynamically remove the n-gram entry in the dictionary.
*/
@UsedForTesting
- public void removeNgramDynamically(final NgramContext ngramContext, final String word) {
+ public void removeNgramDynamically(@Nonnull final NgramContext ngramContext,
+ final String word) {
reloadDictionaryIfRequired();
asyncExecuteTaskWithWriteLock(new Runnable() {
@Override
@@ -402,6 +416,26 @@
});
}
+ /**
+ * Update dictionary for the word with the ngramContext if the word is not a distracter.
+ */
+ public void updateEntriesForWordWithCheckingDistracter(@Nonnull final NgramContext ngramContext,
+ final String word, final boolean isValidWord, final int count, final int timestamp,
+ @Nonnull final DistracterFilter distracterFilter) {
+ updateDictionaryWithWriteLockIfWordIsNotADistracter(new Runnable() {
+ @Override
+ public void run() {
+ if (!mBinaryDictionary.updateEntriesForWordWithNgramContext(ngramContext, word,
+ isValidWord, count, timestamp)) {
+ if (DEBUG) {
+ Log.e(TAG, "Cannot update counter. word: " + word
+ + " context: "+ ngramContext.toString());
+ }
+ }
+ }
+ }, word, distracterFilter);
+ }
+
public interface AddMultipleDictionaryEntriesCallback {
public void onFinished();
}
@@ -410,7 +444,7 @@
* Dynamically add multiple entries to the dictionary.
*/
public void addMultipleDictionaryEntriesDynamically(
- final ArrayList<LanguageModelParam> languageModelParams,
+ @Nonnull final ArrayList<LanguageModelParam> languageModelParams,
final AddMultipleDictionaryEntriesCallback callback) {
reloadDictionaryIfRequired();
asyncExecuteTaskWithWriteLock(new Runnable() {
diff --git a/java/src/com/android/inputmethod/latin/NgramContext.java b/java/src/com/android/inputmethod/latin/NgramContext.java
index 6d43858..a02531c 100644
--- a/java/src/com/android/inputmethod/latin/NgramContext.java
+++ b/java/src/com/android/inputmethod/latin/NgramContext.java
@@ -158,11 +158,6 @@
}
}
- public NgramContext getTrimmedNgramContext(final int maxPrevWordCount) {
- final int newSize = Math.min(maxPrevWordCount, mPrevWordsCount);
- return new NgramContext(this /* prevWordsInfo */, newSize);
- }
-
public int getPrevWordCount() {
return mPrevWordsCount;
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
index d616846..5976154 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
@@ -65,34 +65,7 @@
if (word.length() > Constants.DICTIONARY_MAX_WORD_LENGTH) {
return;
}
- final int frequency = isValid ?
- FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS;
- userHistoryDictionary.addUnigramEntryWithCheckingDistracter(word, frequency,
- null /* shortcutTarget */, 0 /* shortcutFreq */, false /* isNotAWord */,
- false /* isBlacklisted */, timestamp, distracterFilter);
-
- final boolean isBeginningOfSentenceContext = ngramContext.isBeginningOfSentenceContext();
- final NgramContext ngramContextToBeSaved =
- ngramContext.getTrimmedNgramContext(SUPPORTED_NGRAM - 1);
- for (int i = 0; i < ngramContextToBeSaved.getPrevWordCount(); i++) {
- final CharSequence prevWord = ngramContextToBeSaved.getNthPrevWord(1 /* n */);
- if (prevWord == null || (prevWord.length() > Constants.DICTIONARY_MAX_WORD_LENGTH)) {
- return;
- }
- // Do not insert a word as a bigram of itself
- if (i == 0 && TextUtils.equals(word, prevWord)) {
- return;
- }
- if (isBeginningOfSentenceContext) {
- // Beginning-of-Sentence n-gram entry is added as an n-gram entry of an OOV word.
- userHistoryDictionary.addNgramEntry(
- ngramContextToBeSaved.getTrimmedNgramContext(i + 1), word,
- FREQUENCY_FOR_WORDS_NOT_IN_DICTS, timestamp);
- } else {
- userHistoryDictionary.addNgramEntry(
- ngramContextToBeSaved.getTrimmedNgramContext(i + 1), word, frequency,
- timestamp);
- }
- }
+ userHistoryDictionary.updateEntriesForWordWithCheckingDistracter(ngramContext, word,
+ isValid, 1 /* count */, timestamp, distracterFilter);
}
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp
index 4a740d4..ef6166f 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp
@@ -74,8 +74,8 @@
return false;
}
writingPos += getEntrySize();
- mSize++;
}
+ mSize = terminalId + 1;
}
return writeEntry(probabilityEntry, entryPos);
}
@@ -100,7 +100,6 @@
bool ProbabilityDictContent::runGC(
const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
const ProbabilityDictContent *const originalProbabilityDictContent) {
- mSize = 0;
for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
it != terminalIdMap->end(); ++it) {
const ProbabilityEntry probabilityEntry =
@@ -109,7 +108,6 @@
AKLOGE("Cannot set probability entry in runGC. terminalId: %d", it->second);
return false;
}
- mSize++;
}
return true;
}
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
index 6b6ad21..c3c2bb2 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
@@ -75,20 +75,25 @@
return formatVersion > FormatSpec.VERSION401;
}
- private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
- final int probability) {
- binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */,
- BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
- false /* isBeginningOfSentence */, false /* isNotAWord */,
- false /* isBlacklisted */, mCurrentTime /* timestamp */);
+ private void onInputWord(final BinaryDictionary binaryDictionary, final String word,
+ final boolean isValidWord) {
+ binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.EMPTY_PREV_WORDS_INFO,
+ word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */);
}
- private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0,
- final String word1, final int probability) {
- binaryDictionary.addNgramEntry(new NgramContext(new WordInfo(word0)), word1, probability,
+ private void onInputWordWithPrevWord(final BinaryDictionary binaryDictionary, final String word,
+ final boolean isValidWord, final String prevWord) {
+ binaryDictionary.updateEntriesForWordWithNgramContext(
+ new NgramContext(new WordInfo(prevWord)), word, isValidWord, 1 /* count */,
mCurrentTime /* timestamp */);
}
+ private void onInputWordWithBeginningOfSentenceContext(
+ final BinaryDictionary binaryDictionary, final String word, final boolean isValidWord) {
+ binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.BEGINNING_OF_SENTENCE,
+ word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */);
+ }
+
private static boolean isValidBigram(final BinaryDictionary binaryDictionary,
final String word0, final String word1) {
return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1);
@@ -175,10 +180,9 @@
setCurrentTimeForTestMode(mCurrentTime);
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "ab", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "aaa", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWord(binaryDictionary, "ab", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "aaa", true /* isValidWord */, "a");
binaryDictionary.flushWithGC();
binaryDictionary.close();
@@ -229,28 +233,27 @@
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
+ onInputWord(binaryDictionary, "a", false /* isValidWord */);
assertFalse(binaryDictionary.isValidWord("a"));
- addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
- addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
+ onInputWord(binaryDictionary, "a", false /* isValidWord */);
+ onInputWord(binaryDictionary, "a", false /* isValidWord */);
assertTrue(binaryDictionary.isValidWord("a"));
- addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "b", true /* isValidWord */);
assertTrue(binaryDictionary.isValidWord("b"));
- addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a");
assertFalse(isValidBigram(binaryDictionary, "a", "b"));
- addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a");
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
- addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "c", true /* isValidWord */, "a");
assertTrue(isValidBigram(binaryDictionary, "a", "c"));
// Add bigrams of not valid unigrams.
- addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "y", false /* isValidWord */, "x");
assertFalse(isValidBigram(binaryDictionary, "x", "y"));
- addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "y", true /* isValidWord */, "x");
assertFalse(isValidBigram(binaryDictionary, "x", "y"));
binaryDictionary.close();
@@ -266,36 +269,32 @@
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
assertTrue(binaryDictionary.isValidWord("a"));
forcePassingShortTime(binaryDictionary);
assertFalse(binaryDictionary.isValidWord("a"));
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
assertTrue(binaryDictionary.isValidWord("a"));
forcePassingShortTime(binaryDictionary);
assertTrue(binaryDictionary.isValidWord("a"));
forcePassingLongTime(binaryDictionary);
assertFalse(binaryDictionary.isValidWord("a"));
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
forcePassingShortTime(binaryDictionary);
assertFalse(isValidBigram(binaryDictionary, "a", "b"));
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
forcePassingShortTime(binaryDictionary);
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
@@ -333,7 +332,7 @@
binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY));
for (int i = 0; i < unigramTypedCount; i++) {
final String word = words.get(random.nextInt(words.size()));
- addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, word, true /* isValidWord */);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int unigramCountBeforeGC =
@@ -380,10 +379,10 @@
final String strong = "strong";
final String weak = "weak";
for (int j = 0; j < strongUnigramTypedCount; j++) {
- addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, strong, true /* isValidWord */);
}
for (int j = 0; j < weakUnigramTypedCount; j++) {
- addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, weak, true /* isValidWord */);
}
assertTrue(binaryDictionary.isValidWord(strong));
assertTrue(binaryDictionary.isValidWord(weak));
@@ -391,7 +390,7 @@
for (int i = 0; i < unigramCount; i++) {
final String word = CodePointUtils.generateWord(random, codePointSet);
for (int j = 0; j < eachUnigramTypedCount; j++) {
- addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, word, true /* isValidWord */);
}
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int unigramCountBeforeGC =
@@ -450,12 +449,13 @@
}
final int maxBigramCount = Integer.parseInt(
- binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY));
+ binaryDictionary.getPropertyForGettingStats(
+ BinaryDictionary.MAX_BIGRAM_COUNT_QUERY));
for (int i = 0; i < bigramTypedCount; ++i) {
final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size()));
- addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, bigram.second, DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, bigram.first, bigram.second, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, bigram.first, true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, bigram.second, true /* isValidWord */,
+ bigram.first);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int bigramCountBeforeGC =
@@ -506,23 +506,22 @@
final String word = CodePointUtils.generateWord(random, codePointSet);
words.add(word);
for (int j = 0; j < unigramTypedCount; j++) {
- addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, word, true /* isValidWord */);
}
}
final String strong = "strong";
final String weak = "weak";
final String target = "target";
for (int j = 0; j < unigramTypedCount; j++) {
- addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, target, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, weak, true /* isValidWord */);
+ onInputWord(binaryDictionary, strong, true /* isValidWord */);
}
binaryDictionary.flushWithGC();
for (int j = 0; j < strongBigramTypedCount; j++) {
- addBigramWords(binaryDictionary, strong, target, DUMMY_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, strong);
}
for (int j = 0; j < weakBigramTypedCount; j++) {
- addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, weak);
}
assertTrue(isValidBigram(binaryDictionary, strong, target));
assertTrue(isValidBigram(binaryDictionary, weak, target));
@@ -535,7 +534,7 @@
final String word1 = words.get(word1Index);
for (int j = 0; j < eachBigramTypedCount; j++) {
- addBigramWords(binaryDictionary, word0, word1, DUMMY_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, word1, true /* isValidWord */, word0);
}
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int bigramCountBeforeGC =
@@ -563,19 +562,18 @@
setCurrentTimeForTestMode(mCurrentTime);
final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion);
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "aaa", true /* isValidWord */);
assertTrue(binaryDictionary.isValidWord("aaa"));
- addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY);
- assertFalse(binaryDictionary.isValidWord("bbb"));
- addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
+ onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
+ onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
+ onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
+ onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
+
+ onInputWordWithPrevWord(binaryDictionary, "abc", true /* isValidWord */, "aaa");
assertTrue(isValidBigram(binaryDictionary, "aaa", "abc"));
- addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa");
+ assertFalse(binaryDictionary.isValidWord("bbb"));
assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb"));
assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion());
@@ -585,11 +583,11 @@
assertTrue(binaryDictionary.isValidWord("aaa"));
assertFalse(binaryDictionary.isValidWord("bbb"));
assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc"));
- addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY);
+ onInputWord(binaryDictionary, "bbb", false /* isValidWord */);
assertTrue(binaryDictionary.isValidWord("bbb"));
assertTrue(isValidBigram(binaryDictionary, "aaa", "abc"));
assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb"));
- addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa");
assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
binaryDictionary.close();
}
@@ -612,28 +610,19 @@
true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */,
mCurrentTime);
final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE;
- addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
- binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY,
- mCurrentTime);
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
- binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY,
- mCurrentTime);
- addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY);
- binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", DUMMY_PROBABILITY,
- mCurrentTime);
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
-
forcePassingLongTime(binaryDictionary);
assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
- addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
- binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY,
- mCurrentTime);
- addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY);
- binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", DUMMY_PROBABILITY,
- mCurrentTime);
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
binaryDictionary.close();
@@ -651,10 +640,10 @@
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY);
+ onInputWord(binaryDictionary, "aaa", false /* isValidWord */);
assertFalse(binaryDictionary.isValidWord("aaa"));
for (int i = 0; i < unigramInputCount; i++) {
- addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY);
+ onInputWord(binaryDictionary, "aaa", false /* isValidWord */);
}
assertTrue(binaryDictionary.isValidWord("aaa"));
assertTrue(binaryDictionary.removeUnigramEntry("aaa"));