Merge "Re-call getBuffer after writing in bigram/shortcut policy."
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index b92283c..740b86d 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -24,6 +24,7 @@
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
+import com.android.inputmethod.latin.personalization.DynamicPredictionDictionaryBase;
import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
@@ -72,7 +73,7 @@
private BinaryDictionary mBinaryDictionary;
/** The in-memory dictionary used to generate the binary dictionary. */
- private AbstractDictionaryWriter mDictionaryWriter;
+ protected AbstractDictionaryWriter mDictionaryWriter;
/**
* The name of this dictionary, used as the filename for storing the binary dictionary. Multiple
@@ -84,9 +85,11 @@
/** Whether to support dynamically updating the dictionary */
private final boolean mIsUpdatable;
+ // TODO: remove, once dynamic operations is serialized
/** Controls access to the shared binary dictionary file across multiple instances. */
private final DictionaryController mSharedDictionaryController;
+ // TODO: remove, once dynamic operations is serialized
/** Controls access to the local binary dictionary for this instance. */
private final DictionaryController mLocalDictionaryController = new DictionaryController();
@@ -446,9 +449,9 @@
private final void syncReloadDictionaryInternal() {
// Ensure that only one thread attempts to read or write to the shared binary dictionary
// file at the same time.
- mLocalDictionaryController.writeLock().lock();
+ mSharedDictionaryController.writeLock().lock();
try {
- mSharedDictionaryController.writeLock().lock();
+ mLocalDictionaryController.writeLock().lock();
try {
final long time = SystemClock.uptimeMillis();
final boolean dictionaryFileExists = dictionaryFileExists();
@@ -483,10 +486,10 @@
}
mLocalDictionaryController.mLastUpdateTime = time;
} finally {
- mSharedDictionaryController.writeLock().unlock();
+ mLocalDictionaryController.writeLock().unlock();
}
} finally {
- mLocalDictionaryController.writeLock().unlock();
+ mSharedDictionaryController.writeLock().unlock();
}
}
@@ -519,16 +522,16 @@
private class AsyncLoadDictionaryToMemoryTask extends Thread {
@Override
public void run() {
- mLocalDictionaryController.writeLock().lock();
+ mSharedDictionaryController.readLock().lock();
try {
- mSharedDictionaryController.readLock().lock();
+ mLocalDictionaryController.writeLock().lock();
try {
loadDictionaryAsync();
} finally {
- mSharedDictionaryController.readLock().unlock();
+ mLocalDictionaryController.writeLock().unlock();
}
} finally {
- mLocalDictionaryController.writeLock().unlock();
+ mSharedDictionaryController.readLock().unlock();
}
}
}
@@ -622,4 +625,35 @@
mLocalDictionaryController.writeLock().unlock();
}
}
+
+ // TODO: Implement native binary methods once the dynamic dictionary implementation is done.
+ @UsedForTesting
+ public boolean isInDictionaryForTests(final String word) {
+ mLocalDictionaryController.writeLock().lock();
+ try {
+ if (mDictType == Dictionary.TYPE_USER_HISTORY) {
+ return ((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
+ .isInDictionaryForTests(word);
+ }
+ } finally {
+ mLocalDictionaryController.writeLock().unlock();
+ }
+ return false;
+ }
+
+ // TODO: Remove and use addToPersonalizationPredictionDictionary instead!!!!!!!!!!!!!!!!
+ @UsedForTesting
+ public void forceAddWordForTest(
+ final String word0, final String word1, final boolean isValid) {
+ mLocalDictionaryController.writeLock().lock();
+ try {
+ mDictionaryWriter.addUnigramWord(word1, null /* the "shortcut" parameter is null */,
+ DynamicPredictionDictionaryBase.FREQUENCY_FOR_TYPED, false /* isNotAWord */);
+ mDictionaryWriter.addBigramWords(word0, word1,
+ DynamicPredictionDictionaryBase.FREQUENCY_FOR_TYPED, isValid,
+ 0 /* lastTouchedTime */);
+ } finally {
+ mLocalDictionaryController.writeLock().unlock();
+ }
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java
index 342dcfc..ba7d1a2 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java
@@ -265,10 +265,10 @@
return (node == null) ? false : !node.mShortcutOnly;
}
- public boolean removeBigram(final String word1, final String word2) {
+ public boolean removeBigram(final String word0, final String word1) {
// Refer to addOrSetBigram() about word1.toLowerCase()
- final Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null);
- final Node secondWord = searchWord(mRoots, word2, 0, null);
+ final Node firstWord = searchWord(mRoots, word0.toLowerCase(), 0, null);
+ final Node secondWord = searchWord(mRoots, word1, 0, null);
LinkedList<NextWord> bigrams = firstWord.mNGrams;
NextWord bigramNode = null;
if (bigrams == null || bigrams.size() == 0) {
@@ -297,10 +297,10 @@
return (node == null) ? -1 : node.mFrequency;
}
- public NextWord getBigramWord(final String word1, final String word2) {
- // Refer to addOrSetBigram() about word1.toLowerCase()
- final Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null);
- final Node secondWord = searchWord(mRoots, word2, 0, null);
+ public NextWord getBigramWord(final String word0, final String word1) {
+ // Refer to addOrSetBigram() about word0.toLowerCase()
+ final Node firstWord = searchWord(mRoots, word0.toLowerCase(), 0, null);
+ final Node secondWord = searchWord(mRoots, word1, 0, null);
LinkedList<NextWord> bigrams = firstWord.mNGrams;
if (bigrams == null || bigrams.size() == 0) {
return null;
@@ -473,37 +473,41 @@
}
}
- public int setBigramAndGetFrequency(final String word1, final String word2,
+ public int setBigramAndGetFrequency(final String word0, final String word1,
final int frequency) {
- return setBigramAndGetFrequency(word1, word2, frequency, null /* unused */);
+ return setBigramAndGetFrequency(word0, word1, frequency, null /* unused */);
}
- public int setBigramAndGetFrequency(final String word1, final String word2,
+ public int setBigramAndGetFrequency(final String word0, final String word1,
final ForgettingCurveParams fcp) {
- return setBigramAndGetFrequency(word1, word2, 0 /* unused */, fcp);
+ return setBigramAndGetFrequency(word0, word1, 0 /* unused */, fcp);
}
/**
* Adds bigrams to the in-memory trie structure that is being used to retrieve any word
- * @param word1 the first word of this bigram
- * @param word2 the second word of this bigram
+ * @param word0 the first word of this bigram
+ * @param word1 the second word of this bigram
* @param frequency frequency for this bigram
* @param fcp an instance of ForgettingCurveParams to use for decay policy
* @return returns the final bigram frequency
*/
- private int setBigramAndGetFrequency(final String word1, final String word2,
+ private int setBigramAndGetFrequency(final String word0, final String word1,
final int frequency, final ForgettingCurveParams fcp) {
+ if (TextUtils.isEmpty(word0)) {
+ Log.e(TAG, "Invalid bigram previous word: " + word0);
+ return frequency;
+ }
// We don't want results to be different according to case of the looked up left hand side
// word. We do want however to return the correct case for the right hand side.
// So we want to squash the case of the left hand side, and preserve that of the right
// hand side word.
- final String word1Lower = word1.toLowerCase();
- if (TextUtils.isEmpty(word1Lower) || TextUtils.isEmpty(word2)) {
- Log.e(TAG, "Invalid bigram pair: " + word1 + ", " + word1Lower + ", " + word2);
+ final String word0Lower = word0.toLowerCase();
+ if (TextUtils.isEmpty(word0Lower) || TextUtils.isEmpty(word1)) {
+ Log.e(TAG, "Invalid bigram pair: " + word0 + ", " + word0Lower + ", " + word1);
return frequency;
}
- final Node firstWord = searchWord(mRoots, word1Lower, 0, null);
- final Node secondWord = searchWord(mRoots, word2, 0, null);
+ final Node firstWord = searchWord(mRoots, word0Lower, 0, null);
+ final Node secondWord = searchWord(mRoots, word1, 0, null);
LinkedList<NextWord> bigrams = firstWord.mNGrams;
if (bigrams == null || bigrams.size() == 0) {
firstWord.mNGrams = CollectionUtils.newLinkedList();
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index a8a29a1..3d29c5a 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -2532,11 +2532,8 @@
false /* isPrediction */);
}
- private void showSuggestionStrip(final SuggestedWords suggestedWords) {
- if (suggestedWords.isEmpty()) {
- clearSuggestionStrip();
- return;
- }
+ private void setAutoCorrection(final SuggestedWords suggestedWords) {
+ if (suggestedWords.isEmpty()) return;
final String autoCorrection;
if (suggestedWords.mWillAutoCorrect) {
autoCorrection = suggestedWords.getWord(SuggestedWords.INDEX_OF_AUTO_CORRECTION);
@@ -2544,17 +2541,21 @@
autoCorrection = suggestedWords.getWord(SuggestedWords.INDEX_OF_TYPED_WORD);
}
mWordComposer.setAutoCorrection(autoCorrection);
+ }
+
+ private void showSuggestionStrip(final SuggestedWords suggestedWords) {
+ if (suggestedWords.isEmpty()) {
+ clearSuggestionStrip();
+ return;
+ }
+ setAutoCorrection(suggestedWords);
final boolean isAutoCorrection = suggestedWords.willAutoCorrect();
setSuggestedWords(suggestedWords, isAutoCorrection);
setAutoCorrectionIndicator(isAutoCorrection);
setSuggestionStripShown(isSuggestionsStripVisible());
}
- private void commitCurrentAutoCorrection(final String separator, final Runnable callback) {
- // Complete any pending suggestions query first
- if (mHandler.hasPendingUpdateSuggestions()) {
- updateSuggestionStrip();
- }
+ private void completeCommitCurrentAutoCorrection(final String separator) {
final String typedAutoCorrection = mWordComposer.getAutoCorrectionOrNull();
final String typedWord = mWordComposer.getTypedWord();
final String autoCorrection = (typedAutoCorrection != null)
@@ -2588,9 +2589,22 @@
typedWord, autoCorrection));
}
}
- if (callback != null) {
- callback.run();
- }
+ }
+
+ private void commitCurrentAutoCorrection(final String separator, final Runnable callback) {
+ getSuggestedWordsOrOlderSuggestionsAsync(Suggest.SESSION_TYPING,
+ new OnGetSuggestedWordsCallback() {
+ @Override
+ public void onGetSuggestedWords(final SuggestedWords suggestedWords) {
+ if (suggestedWords != null) {
+ setAutoCorrection(suggestedWords);
+ }
+ completeCommitCurrentAutoCorrection(separator);
+ if (callback != null) {
+ callback.run();
+ }
+ }
+ });
}
// Called from {@link SuggestionStripView} through the {@link SuggestionStripView#Listener}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
index 1727382..5a21341 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
@@ -27,7 +27,6 @@
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
-import java.util.Iterator;
/**
* Encodes binary files for a FusionDictionary.
@@ -432,7 +431,7 @@
* @param formatOptions file format options.
* @return the same array it was passed. The nodes have been updated for address and size.
*/
- private static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict,
+ /* package */ static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict,
final ArrayList<PtNodeArray> flatNodes, final FormatOptions formatOptions) {
// First get the worst possible sizes and offsets
for (final PtNodeArray n : flatNodes) calculatePtNodeArrayMaximumSize(n, formatOptions);
@@ -484,7 +483,7 @@
*
* @param arrays the list of node arrays to check
*/
- private static void checkFlatPtNodeArrayList(final ArrayList<PtNodeArray> arrays) {
+ /* package */ static void checkFlatPtNodeArrayList(final ArrayList<PtNodeArray> arrays) {
int offset = 0;
int index = 0;
for (final PtNodeArray ptNodeArray : arrays) {
@@ -508,7 +507,8 @@
* @param position the position to write.
* @return the size in bytes the address actually took.
*/
- private static int writeChildrenPosition(final byte[] buffer, int index, final int position) {
+ /* package */ static int writeChildrenPosition(final byte[] buffer, int index,
+ final int position) {
switch (getByteSize(position)) {
case 1:
buffer[index++] = (byte)position;
@@ -537,7 +537,7 @@
* @param position the position to write.
* @return the size in bytes the address actually took.
*/
- private static int writeSignedChildrenPosition(final byte[] buffer, int index,
+ /* package */ static int writeSignedChildrenPosition(final byte[] buffer, int index,
final int position) {
if (!BinaryDictIOUtils.hasChildrenAddress(position)) {
buffer[index] = buffer[index + 1] = buffer[index + 2] = 0;
@@ -598,7 +598,7 @@
return flags;
}
- private static byte makePtNodeFlags(final PtNode node, final int ptNodeAddress,
+ /* package */ static byte makePtNodeFlags(final PtNode node, final int ptNodeAddress,
final int childrenOffset, final FormatOptions formatOptions) {
return (byte) makePtNodeFlags(node.mChars.length > 1, node.mFrequency >= 0,
getByteSize(childrenOffset),
@@ -616,7 +616,7 @@
* @param word the second bigram, for debugging purposes
* @return the flags
*/
- private static final int makeBigramFlags(final boolean more, final int offset,
+ /* package */ static final int makeBigramFlags(final boolean more, final int offset,
int bigramFrequency, final int unigramFrequency, final String word) {
int bigramFlags = (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0)
+ (offset < 0 ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0);
@@ -702,7 +702,7 @@
+ (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY);
}
- private static final int writeParentAddress(final byte[] buffer, final int index,
+ /* package */ static final int writeParentAddress(final byte[] buffer, final int index,
final int address, final FormatOptions formatOptions) {
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
if (address == FormatSpec.NO_PARENT_ADDRESS) {
@@ -721,7 +721,7 @@
}
}
- private static final int getChildrenPosition(final PtNode ptNode,
+ /* package */ static final int getChildrenPosition(final PtNode ptNode,
final FormatOptions formatOptions) {
int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate
+ getNodeHeaderSize(ptNode, formatOptions);
@@ -736,35 +736,29 @@
* Write a PtNodeArray to memory. The PtNodeArray is expected to have its final position cached.
*
* @param dict the dictionary the node array is a part of (for relative offsets).
- * @param buffer the memory buffer to write to.
+ * @param dictEncoder the dictionary encoder.
* @param ptNodeArray the node array to write.
* @param formatOptions file format options.
- * @return the address of the END of the node.
*/
@SuppressWarnings("unused")
- private static int writePlacedNode(final FusionDictionary dict, byte[] buffer,
- final PtNodeArray ptNodeArray, final FormatOptions formatOptions) {
+ /* package */ static void writePlacedNode(final FusionDictionary dict,
+ final DictEncoder dictEncoder, final PtNodeArray ptNodeArray,
+ final FormatOptions formatOptions) {
// TODO: Make the code in common with BinaryDictIOUtils#writePtNode
- int index = ptNodeArray.mCachedAddressAfterUpdate;
+ dictEncoder.setPosition(ptNodeArray.mCachedAddressAfterUpdate);
final int ptNodeCount = ptNodeArray.mData.size();
- final int countSize = getPtNodeCountSize(ptNodeArray);
- final int parentAddress = ptNodeArray.mCachedParentAddress;
- if (1 == countSize) {
- buffer[index++] = (byte)ptNodeCount;
- } else if (2 == countSize) {
- // We need to signal 2-byte size by setting the top bit of the MSB to 1, so
- // we | 0x80 to do this.
- buffer[index++] = (byte)((ptNodeCount >> 8) | 0x80);
- buffer[index++] = (byte)(ptNodeCount & 0xFF);
- } else {
- throw new RuntimeException("Strange size from getGroupCountSize : " + countSize);
- }
+ dictEncoder.writePtNodeCount(ptNodeCount);
+ final int parentPosition =
+ (ptNodeArray.mCachedParentAddress == FormatSpec.NO_PARENT_ADDRESS)
+ ? FormatSpec.NO_PARENT_ADDRESS
+ : ptNodeArray.mCachedParentAddress + ptNodeArray.mCachedAddressAfterUpdate;
for (int i = 0; i < ptNodeCount; ++i) {
final PtNode ptNode = ptNodeArray.mData.get(i);
- if (index != ptNode.mCachedAddressAfterUpdate) {
+ if (dictEncoder.getPosition() != ptNode.mCachedAddressAfterUpdate) {
throw new RuntimeException("Bug: write index is not the same as the cached address "
- + "of the node : " + index + " <> " + ptNode.mCachedAddressAfterUpdate);
+ + "of the node : " + dictEncoder.getPosition() + " <> "
+ + ptNode.mCachedAddressAfterUpdate);
}
// Sanity checks.
if (DBG && ptNode.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) {
@@ -773,85 +767,23 @@
+ " : " + ptNode.mFrequency);
}
- final int childrenPosition = getChildrenPosition(ptNode, formatOptions);
- buffer[index++] = makePtNodeFlags(ptNode, index, childrenPosition,
- formatOptions);
-
- if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) {
- index = writeParentAddress(buffer, index, parentAddress, formatOptions);
- } else {
- index = writeParentAddress(buffer, index, parentAddress
- + (ptNodeArray.mCachedAddressAfterUpdate
- - ptNode.mCachedAddressAfterUpdate),
- formatOptions);
- }
-
- index = CharEncoding.writeCharArray(ptNode.mChars, buffer, index);
- if (ptNode.hasSeveralChars()) {
- buffer[index++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR;
- }
- if (ptNode.mFrequency >= 0) {
- buffer[index++] = (byte) ptNode.mFrequency;
- }
-
- if (formatOptions.mSupportsDynamicUpdate) {
- index += writeSignedChildrenPosition(buffer, index, childrenPosition);
- } else {
- index += writeChildrenPosition(buffer, index, childrenPosition);
- }
-
- // Write shortcuts
- if (null != ptNode.mShortcutTargets && !ptNode.mShortcutTargets.isEmpty()) {
- final int indexOfShortcutByteSize = index;
- index += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE;
- final Iterator<WeightedString> shortcutIterator =
- ptNode.mShortcutTargets.iterator();
- while (shortcutIterator.hasNext()) {
- final WeightedString target = shortcutIterator.next();
- int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(),
- target.mFrequency);
- buffer[index++] = (byte)shortcutFlags;
- final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord);
- index += shortcutShift;
- }
- final int shortcutByteSize = index - indexOfShortcutByteSize;
- if (shortcutByteSize > 0xFFFF) {
- throw new RuntimeException("Shortcut list too large");
- }
- buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8);
- buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF);
- }
- // Write bigrams
- if (null != ptNode.mBigrams) {
- final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator();
- while (bigramIterator.hasNext()) {
- final WeightedString bigram = bigramIterator.next();
- final PtNode target =
- FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord);
- final int addressOfBigram = target.mCachedAddressAfterUpdate;
- final int unigramFrequencyForThisWord = target.mFrequency;
- final int offset = addressOfBigram
- - (index + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
- int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset,
- bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
- buffer[index++] = (byte)bigramFlags;
- final int bigramShift = writeChildrenPosition(buffer, index, Math.abs(offset));
- index += bigramShift;
- }
- }
-
+ dictEncoder.writePtNodeFlags(ptNode, parentPosition, formatOptions);
+ dictEncoder.writeParentPosition(parentPosition, ptNode, formatOptions);
+ dictEncoder.writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
+ dictEncoder.writeFrequency(ptNode.mFrequency);
+ dictEncoder.writeChildrenPosition(ptNode, formatOptions);
+ dictEncoder.writeShortcuts(ptNode.mShortcutTargets);
+ dictEncoder.writeBigrams(ptNode.mBigrams, dict);
}
if (formatOptions.mSupportsDynamicUpdate) {
- buffer[index] = buffer[index + 1] = buffer[index + 2]
- = FormatSpec.NO_FORWARD_LINK_ADDRESS;
- index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
+ dictEncoder.writeForwardLinkAddress(FormatSpec.NO_FORWARD_LINK_ADDRESS);
}
- if (index != ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize) {
- throw new RuntimeException(
- "Not the same size : written " + (index - ptNodeArray.mCachedAddressAfterUpdate)
+ if (dictEncoder.getPosition() != ptNodeArray.mCachedAddressAfterUpdate
+ + ptNodeArray.mCachedSize) {
+ throw new RuntimeException("Not the same size : written "
+ + (dictEncoder.getPosition() - ptNodeArray.mCachedAddressAfterUpdate)
+ " bytes from a node that should have " + ptNodeArray.mCachedSize + " bytes");
}
- return index;
}
/**
@@ -862,7 +794,7 @@
*
* @param ptNodeArrays the list of PtNode arrays.
*/
- private static void showStatistics(ArrayList<PtNodeArray> ptNodeArrays) {
+ /* package */ static void showStatistics(ArrayList<PtNodeArray> ptNodeArrays) {
int firstTerminalAddress = Integer.MAX_VALUE;
int lastTerminalAddress = Integer.MIN_VALUE;
int size = 0;
@@ -968,49 +900,4 @@
headerBuffer.close();
}
-
- /**
- * Dumps a FusionDictionary to a file.
- *
- * @param destination the stream to write the dictionary body to.
- * @param dict the dictionary to write.
- * @param formatOptions file format options.
- */
- /* package */ static void writeDictionaryBody(final OutputStream destination,
- final FusionDictionary dict, final FormatOptions formatOptions) throws IOException {
-
- // Addresses are limited to 3 bytes, but since addresses can be relative to each node
- // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding
- // the order of the PtNode arrays becomes a quite complicated problem, because though the
- // dictionary itself does not have a size limit, each node array must still be within 16MB
- // of all its children and parents. As long as this is ensured, the dictionary file may
- // grow to any size.
-
- // Leave the choice of the optimal node order to the flattenTree function.
- MakedictLog.i("Flattening the tree...");
- ArrayList<PtNodeArray> flatNodes = flattenTree(dict.mRootNodeArray);
-
- MakedictLog.i("Computing addresses...");
- computeAddresses(dict, flatNodes, formatOptions);
- MakedictLog.i("Checking PtNode array...");
- if (DBG) checkFlatPtNodeArrayList(flatNodes);
-
- // Create a buffer that matches the final dictionary size.
- final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
- final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize;
- final byte[] buffer = new byte[bufferSize];
-
- MakedictLog.i("Writing file...");
- int dataEndOffset = 0;
- for (PtNodeArray nodeArray : flatNodes) {
- dataEndOffset = writePlacedNode(dict, buffer, nodeArray, formatOptions);
- }
-
- if (DBG) showStatistics(flatNodes);
-
- destination.write(buffer, 0, dataEndOffset);
-
- destination.close();
- MakedictLog.i("Done");
- }
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java
index 89c982e..d1589a3 100644
--- a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java
@@ -17,8 +17,11 @@
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
+import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.io.IOException;
+import java.util.ArrayList;
/**
* An interface of binary dictionary encoder.
@@ -26,4 +29,32 @@
public interface DictEncoder {
public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions)
throws IOException, UnsupportedFormatException;
+
+ public void setPosition(final int position);
+ public int getPosition();
+ public void writePtNodeCount(final int ptNodeCount);
+ public void writePtNodeFlags(final PtNode ptNode, final int parentAddress,
+ final FormatOptions formatOptions);
+ public void writeParentPosition(final int parentPosition, final PtNode ptNode,
+ final FormatOptions formatOptions);
+ public void writeCharacters(final int[] characters, final boolean hasSeveralChars);
+ public void writeFrequency(final int frequency);
+ public void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions);
+
+ /**
+ * Write a shortcut attributes list to memory.
+ *
+ * @param shortcuts the shortcut attributes list.
+ */
+ public void writeShortcuts(final ArrayList<WeightedString> shortcuts);
+
+ /**
+ * Write a bigram attributes list to memory.
+ *
+ * @param bigrams the bigram attributes list.
+ * @param dict the dictionary the node array is a part of (for relative offsets).
+ */
+ public void writeBigrams(final ArrayList<WeightedString> bigrams, final FusionDictionary dict);
+
+ public void writeForwardLinkAddress(final int forwardLinkAddress);
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java
index 4e28fea..9385ba3 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java
@@ -16,13 +16,19 @@
package com.android.inputmethod.latin.makedict;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
+import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
/**
* An implementation of DictEncoder for version 3 binary dictionary.
@@ -31,10 +37,13 @@
private final File mDictFile;
private OutputStream mOutStream;
+ private byte[] mBuffer;
+ private int mPosition;
public Ver3DictEncoder(final File dictFile) {
mDictFile = dictFile;
mOutStream = null;
+ mBuffer = null;
}
// This constructor is used only by BinaryDictOffdeviceUtilsTests.
@@ -57,13 +66,166 @@
}
@Override
- public void writeDictionary(FusionDictionary dict, FormatOptions formatOptions)
+ public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions)
throws IOException, UnsupportedFormatException {
if (mOutStream == null) {
openStream();
}
BinaryDictEncoderUtils.writeDictionaryHeader(mOutStream, dict, formatOptions);
- BinaryDictEncoderUtils.writeDictionaryBody(mOutStream, dict, formatOptions);
+
+ // Addresses are limited to 3 bytes, but since addresses can be relative to each node
+ // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding
+ // the order of the PtNode arrays becomes a quite complicated problem, because though the
+ // dictionary itself does not have a size limit, each node array must still be within 16MB
+ // of all its children and parents. As long as this is ensured, the dictionary file may
+ // grow to any size.
+
+ // Leave the choice of the optimal node order to the flattenTree function.
+ MakedictLog.i("Flattening the tree...");
+ ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray);
+
+ MakedictLog.i("Computing addresses...");
+ BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions);
+ MakedictLog.i("Checking PtNode array...");
+ if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes);
+
+ // Create a buffer that matches the final dictionary size.
+ final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
+ final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize;
+ mBuffer = new byte[bufferSize];
+
+ MakedictLog.i("Writing file...");
+
+ for (PtNodeArray nodeArray : flatNodes) {
+ BinaryDictEncoderUtils.writePlacedNode(dict, this, nodeArray, formatOptions);
+ }
+ if (MakedictLog.DBG) BinaryDictEncoderUtils.showStatistics(flatNodes);
+ mOutStream.write(mBuffer, 0, mPosition);
+
+ MakedictLog.i("Done");
close();
}
+
+ @Override
+ public void setPosition(final int position) {
+ if (mBuffer == null || position < 0 || position >= mBuffer.length) return;
+ mPosition = position;
+ }
+
+ @Override
+ public int getPosition() {
+ return mPosition;
+ }
+
+ @Override
+ public void writePtNodeCount(final int ptNodeCount) {
+ final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount);
+ if (1 == countSize) {
+ mBuffer[mPosition++] = (byte) ptNodeCount;
+ } else if (2 == countSize) {
+ mBuffer[mPosition++] = (byte) ((ptNodeCount >> 8) & 0xFF);
+ mBuffer[mPosition++] = (byte) (ptNodeCount & 0xFF);
+ } else {
+ throw new RuntimeException("Strange size from getGroupCountSize : " + countSize);
+ }
+ }
+
+ @Override
+ public void writePtNodeFlags(final PtNode ptNode, final int parentAddress,
+ final FormatOptions formatOptions) {
+ final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
+ mBuffer[mPosition++] = BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mPosition,
+ childrenPos, formatOptions);
+ }
+
+ @Override
+ public void writeParentPosition(final int parentPosition, final PtNode ptNode,
+ final FormatOptions formatOptions) {
+ if (parentPosition == FormatSpec.NO_PARENT_ADDRESS) {
+ mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition,
+ parentPosition, formatOptions);
+ } else {
+ mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition,
+ parentPosition - ptNode.mCachedAddressAfterUpdate, formatOptions);
+ }
+ }
+
+ @Override
+ public void writeCharacters(final int[] codePoints, final boolean hasSeveralChars) {
+ mPosition = CharEncoding.writeCharArray(codePoints, mBuffer, mPosition);
+ if (hasSeveralChars) {
+ mBuffer[mPosition++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR;
+ }
+ }
+
+ @Override
+ public void writeFrequency(final int frequency) {
+ if (frequency >= 0) {
+ mBuffer[mPosition++] = (byte) frequency;
+ }
+ }
+
+ @Override
+ public void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions) {
+ final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
+ if (formatOptions.mSupportsDynamicUpdate) {
+ mPosition += BinaryDictEncoderUtils.writeSignedChildrenPosition(mBuffer, mPosition,
+ childrenPos);
+ } else {
+ mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition,
+ childrenPos);
+ }
+ }
+
+ @Override
+ public void writeShortcuts(final ArrayList<WeightedString> shortcuts) {
+ if (null == shortcuts || shortcuts.isEmpty()) return;
+
+ final int indexOfShortcutByteSize = mPosition;
+ mPosition += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE;
+ final Iterator<WeightedString> shortcutIterator = shortcuts.iterator();
+ while (shortcutIterator.hasNext()) {
+ final WeightedString target = shortcutIterator.next();
+ final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
+ shortcutIterator.hasNext(),
+ target.mFrequency);
+ mBuffer[mPosition++] = (byte)shortcutFlags;
+ final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord);
+ mPosition += shortcutShift;
+ }
+ final int shortcutByteSize = mPosition - indexOfShortcutByteSize;
+ if (shortcutByteSize > 0xFFFF) {
+ throw new RuntimeException("Shortcut list too large");
+ }
+ mBuffer[indexOfShortcutByteSize] = (byte)((shortcutByteSize >> 8) & 0xFF);
+ mBuffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF);
+ }
+
+ @Override
+ public void writeBigrams(final ArrayList<WeightedString> bigrams, final FusionDictionary dict) {
+ if (bigrams == null) return;
+
+ final Iterator<WeightedString> bigramIterator = bigrams.iterator();
+ while (bigramIterator.hasNext()) {
+ final WeightedString bigram = bigramIterator.next();
+ final PtNode target =
+ FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord);
+ final int addressOfBigram = target.mCachedAddressAfterUpdate;
+ final int unigramFrequencyForThisWord = target.mFrequency;
+ final int offset = addressOfBigram
+ - (mPosition + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
+ int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(),
+ offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
+ mBuffer[mPosition++] = (byte) bigramFlags;
+ mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition,
+ Math.abs(offset));
+ }
+ }
+
+ @Override
+ public void writeForwardLinkAddress(final int forwardLinkAddress) {
+ mBuffer[mPosition++] = (byte) ((forwardLinkAddress >> 16) & 0xFF);
+ mBuffer[mPosition++] = (byte) ((forwardLinkAddress >> 8) & 0xFF);
+ mBuffer[mPosition++] = (byte) (forwardLinkAddress & 0xFF);
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
index d446606..e43e74d 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
@@ -18,6 +18,7 @@
import android.content.Context;
+import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.AbstractDictionaryWriter;
import com.android.inputmethod.latin.ExpandableDictionary;
@@ -156,4 +157,10 @@
public boolean isValidWord(final String word) {
return mExpandableDictionary.isValidWord(word);
}
+
+ @UsedForTesting
+ public boolean isInDictionaryForTests(final String word) {
+ // TODO: Use native method to determine whether the word is in dictionary or not
+ return mBigramList.containsKey(word);
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
index a08145b..be4c7c4 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
@@ -196,12 +196,6 @@
return mLocale;
}
- @UsedForTesting
- /* package for test */ void forceAddWordForTest(
- final String word0, final String word1, final boolean isValid) {
- addToPersonalizationPredictionDictionary(word0, word1, isValid);
- }
-
public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {
session.setPredictionDictionary(this);
mSessions.add(session);
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
index 6c2c9e2..4c1803b 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
@@ -97,6 +97,10 @@
return mBigramMap.isEmpty();
}
+ public boolean containsKey(String word) {
+ return mBigramMap.containsKey(word);
+ }
+
public Set<String> keySet() {
return mBigramMap.keySet();
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
index 6bb90fc..405628b 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
@@ -43,10 +43,19 @@
dictBuf, mFlags, MAX_WORD_LENGTH, &pos);
}
if (isTerminal()) {
+ mProbabilityFieldPos = pos;
+ if (usesAdditionalBuffer) {
+ mProbabilityFieldPos += mBuffer->getOriginalBufferSize();
+ }
mProbability = PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(dictBuf, &pos);
} else {
+ mProbabilityFieldPos = NOT_A_DICT_POS;
mProbability = NOT_A_PROBABILITY;
}
+ mChildrenPosFieldPos = pos;
+ if (usesAdditionalBuffer) {
+ mChildrenPosFieldPos += mBuffer->getOriginalBufferSize();
+ }
mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
dictBuf, mFlags, &pos);
if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h
index acc68b3..2ee7c24 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h
@@ -40,9 +40,11 @@
const DictionaryShortcutsStructurePolicy *const shortcutsPolicy)
: mBuffer(buffer), mBigramsPolicy(bigramsPolicy),
mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0),
- mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbability(NOT_A_PROBABILITY),
- mChildrenPos(NOT_A_DICT_POS), mShortcutPos(NOT_A_DICT_POS),
- mBigramPos(NOT_A_DICT_POS), mSiblingPos(NOT_A_VALID_WORD_POS) {}
+ mParentPos(NOT_A_DICT_POS), mCodePointCount(0),
+ mProbabilityFieldPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY),
+ mChildrenPosFieldPos(NOT_A_DICT_POS), mChildrenPos(NOT_A_DICT_POS),
+ mShortcutPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS),
+ mSiblingPos(NOT_A_VALID_WORD_POS) {}
~DynamicPatriciaTrieNodeReader() {}
@@ -95,11 +97,19 @@
}
// Probability
+ AK_FORCE_INLINE int getProbabilityFieldPos() const {
+ return mProbabilityFieldPos;
+ }
+
AK_FORCE_INLINE int getProbability() const {
return mProbability;
}
- // Children node group position
+ // Children PtNode array position
+ AK_FORCE_INLINE int getChildrenPosFieldPos() const {
+ return mChildrenPosFieldPos;
+ }
+
AK_FORCE_INLINE int getChildrenPos() const {
return mChildrenPos;
}
@@ -129,7 +139,9 @@
DynamicPatriciaTrieReadingUtils::NodeFlags mFlags;
int mParentPos;
uint8_t mCodePointCount;
+ int mProbabilityFieldPos;
int mProbability;
+ int mChildrenPosFieldPos;
int mChildrenPos;
int mShortcutPos;
int mBigramPos;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
index cf70f92..6d1bd02 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
@@ -26,6 +26,9 @@
namespace latinime {
+// TODO: Enable dynamic update and remove this flag.
+const bool DynamicPatriciaTrieWritingHelper::ENABLE_DYNAMIC_UPDATE = false;
+
bool DynamicPatriciaTrieWritingHelper::addUnigramWord(
DynamicPatriciaTrieReadingHelper *const readingHelper,
const int *const wordCodePoints, const int codePointCount, const int probability) {
@@ -56,16 +59,21 @@
}
// All characters are matched.
if (codePointCount == readingHelper->getTotalCodePointCount()) {
- if (nodeReader->isTerminal()) {
- // TODO: Update probability.
+ if (ENABLE_DYNAMIC_UPDATE) {
+ return setPtNodeProbability(nodeReader, probability,
+ readingHelper->getMergedNodeCodePoints());
} else {
- // TODO: Make it terminal and update probability.
+ return false;
}
- return false;
}
if (!nodeReader->hasChildren()) {
- // TODO: Create children node array and add new node as a child.
- return false;
+ if (ENABLE_DYNAMIC_UPDATE) {
+ return createChildrenPtNodeArrayAndAChildPtNode(nodeReader, probability,
+ wordCodePoints + readingHelper->getTotalCodePointCount(),
+ codePointCount - readingHelper->getTotalCodePointCount());
+ } else {
+ return false;
+ }
}
// Advance to the children nodes.
parentPos = nodeReader->getNodePos();
@@ -76,12 +84,14 @@
return false;
}
int pos = readingHelper->getPosOfLastForwardLinkField();
- // TODO: Remove.
- return false;
- return createAndInsertNodeIntoPtNodeArray(parentPos,
- wordCodePoints + readingHelper->getPrevTotalCodePointCount(),
- codePointCount - readingHelper->getPrevTotalCodePointCount(),
- probability, &pos);
+ if (ENABLE_DYNAMIC_UPDATE) {
+ return createAndInsertNodeIntoPtNodeArray(parentPos,
+ wordCodePoints + readingHelper->getPrevTotalCodePointCount(),
+ codePointCount - readingHelper->getPrevTotalCodePointCount(),
+ probability, &pos);
+ } else {
+ return false;
+ }
}
bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const int word1Pos,
@@ -198,12 +208,58 @@
newPtNodeArrayPos, forwardLinkFieldPos)) {
return false;
}
- int writingPos = newPtNodeArrayPos;
+ return createNewPtNodeArrayWithAChildPtNode(parentPos, nodeCodePoints, nodeCodePointCount,
+ probability);
+}
+
+bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability(
+ const DynamicPatriciaTrieNodeReader *const originalPtNode, const int probability,
+ const int *const codePoints) {
+ if (originalPtNode->isTerminal()) {
+ // Overwrites the probability.
+ int probabilityFieldPos = originalPtNode->getProbabilityFieldPos();
+ if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(mBuffer,
+ probability, &probabilityFieldPos)) {
+ return false;
+ }
+ } else {
+ // Make the node terminal and write the probability.
+ int movedPos = mBuffer->getTailPosition();
+ if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos)) {
+ return false;
+ }
+ if (!writeNodeToBuffer(originalPtNode->isBlacklisted(), originalPtNode->isNotAWord(),
+ originalPtNode->getParentPos(), codePoints, originalPtNode->getCodePointCount(),
+ probability, originalPtNode->getChildrenPos(), originalPtNode->getBigramsPos(),
+ originalPtNode->getShortcutPos(), &movedPos)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool DynamicPatriciaTrieWritingHelper::createChildrenPtNodeArrayAndAChildPtNode(
+ const DynamicPatriciaTrieNodeReader *const parentNode, const int probability,
+ const int *const codePoints, const int codePointCount) {
+ const int newPtNodeArrayPos = mBuffer->getTailPosition();
+ int childrenPosFieldPos = parentNode->getChildrenPosFieldPos();
+ if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer,
+ newPtNodeArrayPos, &childrenPosFieldPos)) {
+ return false;
+ }
+ return createNewPtNodeArrayWithAChildPtNode(parentNode->getNodePos(), codePoints,
+ codePointCount, probability);
+}
+
+bool DynamicPatriciaTrieWritingHelper::createNewPtNodeArrayWithAChildPtNode(
+ const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount,
+ const int probability) {
+ int writingPos = mBuffer->getTailPosition();
if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer,
1 /* arraySize */, &writingPos)) {
return false;
}
- if (!writeNodeToBuffer(false /* isBlacklisted */, false /* isNotAWord */, parentPos,
+ if (!writeNodeToBuffer(false /* isBlacklisted */, false /* isNotAWord */, parentPtNodePos,
nodeCodePoints, nodeCodePointCount, probability, NOT_A_DICT_POS /* childrenPos */,
NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* originalShortcutPos */,
&writingPos)) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
index 524f361..16b84ba 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
@@ -49,6 +49,7 @@
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper);
+ static const bool ENABLE_DYNAMIC_UPDATE;
BufferWithExtendableBuffer *const mBuffer;
DynamicBigramListPolicy *const mBigramPolicy;
DynamicShortcutListPolicy *const mShortcutPolicy;
@@ -63,6 +64,16 @@
bool createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints,
const int nodeCodePointCount, const int probability, int *const forwardLinkFieldPos);
+
+ bool setPtNodeProbability(const DynamicPatriciaTrieNodeReader *const originalNode,
+ const int probability, const int *const codePoints);
+
+ bool createChildrenPtNodeArrayAndAChildPtNode(
+ const DynamicPatriciaTrieNodeReader *const parentNode, const int probability,
+ const int *const codePoints, const int codePointCount);
+
+ bool createNewPtNodeArrayWithAChildPtNode(const int parentPos, const int *const nodeCodePoints,
+ const int nodeCodePointCount, const int probability);
};
} // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */
diff --git a/tests/src/com/android/inputmethod/latin/InputTestsBase.java b/tests/src/com/android/inputmethod/latin/InputTestsBase.java
index 0a1c4e9..da1fb6f 100644
--- a/tests/src/com/android/inputmethod/latin/InputTestsBase.java
+++ b/tests/src/com/android/inputmethod/latin/InputTestsBase.java
@@ -44,10 +44,12 @@
private static final String PREF_DEBUG_MODE = "debug_mode";
- // The message that sets the underline is posted with a 100 ms delay
+ // The message that sets the underline is posted with a 200 ms delay
protected static final int DELAY_TO_WAIT_FOR_UNDERLINE = 200;
- // The message that sets predictions is posted with a 100 ms delay
+ // The message that sets predictions is posted with a 200 ms delay
protected static final int DELAY_TO_WAIT_FOR_PREDICTIONS = 200;
+ // The message that sets auto-corrections is posted within a 100 ms delay.
+ protected static final int DELAY_TO_WAIT_FOR_AUTOCORRECTION = 100;
protected LatinIME mLatinIME;
protected Keyboard mKeyboard;
@@ -221,6 +223,7 @@
protected void type(final String stringToType) {
for (int i = 0; i < stringToType.length(); i = stringToType.offsetByCodePoints(i, 1)) {
type(stringToType.codePointAt(i));
+ sleep(DELAY_TO_WAIT_FOR_AUTOCORRECTION);
}
}
diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
index 1fd1b8a..1a20ec5 100644
--- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
@@ -82,14 +82,29 @@
}
}
+ /**
+ * @param checksContents if true, checks whether written words are actually in the dictionary
+ * or not.
+ */
private void addAndWriteRandomWords(final String testFilenameSuffix, final int numberOfWords,
- final Random random) {
+ final Random random, final boolean checksContents) {
final List<String> words = generateWords(numberOfWords, random);
final UserHistoryPredictionDictionary dict =
PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
testFilenameSuffix /* locale */, mPrefs);
// Add random words to the user history dictionary.
addToDict(dict, words);
+ if (checksContents) {
+ try {
+ Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS));
+ } catch (InterruptedException e) {
+ }
+ for (int i = 0; i < 10 && i < numberOfWords; ++i) {
+ final String word = words.get(i);
+ // This may fail as long as we use tryLock on inserting the bigram words
+ assertTrue(dict.isInDictionaryForTests(word));
+ }
+ }
// write to file.
dict.close();
}
@@ -103,7 +118,8 @@
final Random random = new Random(123456);
try {
- addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random);
+ addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random,
+ true /* checksContents */);
} finally {
try {
Log.d(TAG, "waiting for writing ...");
@@ -148,7 +164,8 @@
final int index = i % numberOfLanguages;
// Switch languages to testFilenameSuffixes[index].
addAndWriteRandomWords(testFilenameSuffixes[index],
- numberOfWordsInsertedForEachLanguageSwitch, random);
+ numberOfWordsInsertedForEachLanguageSwitch, random,
+ false /* checksContents */);
}
final long end = System.currentTimeMillis();