Merge "Set EmojiCapable"
diff --git a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
index 702ed20..546fa81 100644
--- a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
+++ b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
@@ -505,10 +505,7 @@
@Override
public void onKeyClick(final Key key) {
- // TODO: Save emoticons to recents
- if (mEmojiCategory.getCurrentCategoryId() != CATEGORY_ID_EMOTICONS) {
- mEmojiKeyboardAdapter.addRecentKey(key);
- }
+ mEmojiKeyboardAdapter.addRecentKey(key);
mEmojiCategory.saveLastTypedCategoryPage();
final int code = key.getCode();
if (code == Constants.CODE_OUTPUT_TEXT) {
diff --git a/java/src/com/android/inputmethod/keyboard/Keyboard.java b/java/src/com/android/inputmethod/keyboard/Keyboard.java
index 23f037f..bc1383a 100644
--- a/java/src/com/android/inputmethod/keyboard/Keyboard.java
+++ b/java/src/com/android/inputmethod/keyboard/Keyboard.java
@@ -155,6 +155,15 @@
return mKeys;
}
+ public Key getKeyFromOutputText(final String outputText) {
+ for (final Key key : getKeys()) {
+ if (outputText.equals(key.getOutputText())) {
+ return key;
+ }
+ }
+ return null;
+ }
+
public Key getKey(final int code) {
if (code == Constants.CODE_UNSPECIFIED) {
return null;
diff --git a/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java b/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java
index f203eb7..2976e23 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java
@@ -18,25 +18,27 @@
import android.content.SharedPreferences;
import android.text.TextUtils;
+import android.util.Log;
import com.android.inputmethod.keyboard.EmojiKeyboardView;
import com.android.inputmethod.keyboard.Key;
import com.android.inputmethod.keyboard.Keyboard;
-import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.CollectionUtils;
+import com.android.inputmethod.latin.utils.StringUtils;
import java.util.ArrayDeque;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
/**
* This is a Keyboard class where you can add keys dynamically shown in a grid layout
*/
public class DynamicGridKeyboard extends Keyboard {
+ private static final String TAG = DynamicGridKeyboard.class.getSimpleName();
private static final int TEMPLATE_KEY_CODE_0 = 0x30;
private static final int TEMPLATE_KEY_CODE_1 = 0x31;
- // Recent codes are saved as an integer array, so we use comma as a separater.
- private static final String RECENT_KEY_SEPARATOR = Constants.STRING_COMMA;
private final SharedPreferences mPrefs;
private final int mLeftPadding;
@@ -84,6 +86,9 @@
}
private void addKey(final Key usedKey, final boolean addFirst) {
+ if (usedKey == null) {
+ return;
+ }
synchronized (mGridKeys) {
mCachedGridKeys = null;
final GridKey key = new GridKey(usedKey);
@@ -109,27 +114,44 @@
}
private void saveRecentKeys() {
- final StringBuilder sb = new StringBuilder();
+ final ArrayList<Object> keys = CollectionUtils.newArrayList();
for (final Key key : mGridKeys) {
- sb.append(key.getCode()).append(RECENT_KEY_SEPARATOR);
+ if (key.getOutputText() != null) {
+ keys.add(key.getOutputText());
+ } else {
+ keys.add(key.getCode());
+ }
}
- Settings.writeEmojiRecentKeys(mPrefs, sb.toString());
+ final String jsonStr = StringUtils.listToJsonStr(keys);
+ Settings.writeEmojiRecentKeys(mPrefs, jsonStr);
+ }
+
+ private static Key getKey(final Collection<DynamicGridKeyboard> keyboards, final Object o) {
+ for (final DynamicGridKeyboard kbd : keyboards) {
+ if (o instanceof Integer) {
+ final int code = (Integer) o;
+ final Key key = kbd.getKey(code);
+ if (key != null) {
+ return key;
+ }
+ } else if (o instanceof String) {
+ final String outputText = (String) o;
+ final Key key = kbd.getKeyFromOutputText(outputText);
+ if (key != null) {
+ return key;
+ }
+ } else {
+ Log.w(TAG, "Invalid object: " + o);
+ }
+ }
+ return null;
}
public void loadRecentKeys(Collection<DynamicGridKeyboard> keyboards) {
final String str = Settings.readEmojiRecentKeys(mPrefs);
- for (String s : str.split(RECENT_KEY_SEPARATOR)) {
- if (TextUtils.isEmpty(s)) {
- continue;
- }
- final int code = Integer.valueOf(s);
- for (DynamicGridKeyboard kbd : keyboards) {
- final Key key = kbd.getKey(code);
- if (key != null) {
- addKeyLast(key);
- break;
- }
- }
+ final List<Object> keys = StringUtils.jsonStrToList(str);
+ for (final Object o : keys) {
+ addKeyLast(getKey(keyboards, o));
}
}
diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
index 55df263..845a9b9 100644
--- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
@@ -58,7 +58,7 @@
final File file = new File(mContext.getFilesDir(), fileName);
final File tempFile = new File(mContext.getFilesDir(), tempFileName);
try {
- final DictEncoder dictEncoder = new Ver3DictEncoder(file);
+ final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile);
writeDictionary(dictEncoder);
tempFile.renameTo(file);
} catch (IOException e) {
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index dacb848..632ee0d 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -19,6 +19,7 @@
import android.text.TextUtils;
import android.util.SparseArray;
+import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.settings.NativeSuggestOptions;
@@ -47,12 +48,13 @@
private long mNativeDict;
private final Locale mLocale;
private final long mDictSize;
+ private final String mDictFilePath;
private final int[] mInputCodePoints = new int[MAX_WORD_LENGTH];
private final int[] mOutputCodePoints = new int[MAX_WORD_LENGTH * MAX_RESULTS];
private final int[] mSpaceIndices = new int[MAX_RESULTS];
private final int[] mOutputScores = new int[MAX_RESULTS];
private final int[] mOutputTypes = new int[MAX_RESULTS];
- private final int[] mOutputAutoCommitFirstWordConfidence = new int[1]; // Only one result
+ private final int[] mOutputAutoCommitFirstWordConfidence = new int[MAX_RESULTS];
private final NativeSuggestOptions mNativeSuggestOptions = new NativeSuggestOptions();
@@ -91,6 +93,7 @@
super(dictType);
mLocale = locale;
mDictSize = length;
+ mDictFilePath = filename;
mNativeSuggestOptions.setUseFullEditDistance(useFullEditDistance);
loadDictionary(filename, offset, length, isUpdatable);
}
@@ -101,9 +104,12 @@
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
boolean isUpdatable);
+ private static native void flushNative(long dict, String filePath);
+ private static native boolean needsToRunGCNative(long dict);
+ private static native void flushWithGCNative(long dict, String filePath);
private static native void closeNative(long dict);
private static native int getProbabilityNative(long dict, int[] word);
- private static native boolean isValidBigramNative(long dict, int[] word0, int[] word1);
+ private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1);
private static native int getSuggestionsNative(long dict, long proximityInfo,
long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times,
int[] pointerIds, int[] inputCodePoints, int inputSize, int commitPoint,
@@ -116,6 +122,8 @@
private static native void addBigramWordsNative(long dict, int[] word0, int[] word1,
int probability);
private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1);
+ private static native int calculateProbabilityNative(long dict, int unigramProbability,
+ int bigramProbability);
// TODO: Move native dict into session
private final void loadDictionary(final String path, final long startOffset,
@@ -186,7 +194,7 @@
// flags too and pass mOutputTypes[j] instead of kind
suggestions.add(new SuggestedWordInfo(new String(mOutputCodePoints, start, len),
score, kind, this /* sourceDict */,
- mSpaceIndices[0] /* indexOfTouchPointOfSecondWord */,
+ mSpaceIndices[j] /* indexOfTouchPointOfSecondWord */,
mOutputAutoCommitFirstWordConfidence[0]));
}
}
@@ -213,12 +221,12 @@
@Override
public boolean isValidWord(final String word) {
- return getFrequency(word) >= 0;
+ return getFrequency(word) != NOT_A_PROBABILITY;
}
@Override
public int getFrequency(final String word) {
- if (word == null) return -1;
+ if (word == null) return NOT_A_PROBABILITY;
int[] codePoints = StringUtils.toCodePointArray(word);
return getProbabilityNative(mNativeDict, codePoints);
}
@@ -226,10 +234,14 @@
// TODO: Add a batch process version (isValidBigramMultiple?) to avoid excessive numbers of jni
// calls when checking for changes in an entire dictionary.
public boolean isValidBigram(final String word0, final String word1) {
- if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) return false;
+ return getBigramProbability(word0, word1) != NOT_A_PROBABILITY;
+ }
+
+ public int getBigramProbability(final String word0, final String word1) {
+ if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) return NOT_A_PROBABILITY;
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
- return isValidBigramNative(mNativeDict, codePoints0, codePoints1);
+ return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
}
// Add a unigram entry to binary dictionary in native code.
@@ -261,6 +273,30 @@
removeBigramWordsNative(mNativeDict, codePoints0, codePoints1);
}
+ @UsedForTesting
+ public void flush() {
+ if (!isValidDictionary()) return;
+ flushNative(mNativeDict, mDictFilePath);
+ }
+
+ @UsedForTesting
+ public void flushWithGC() {
+ if (!isValidDictionary()) return;
+ flushWithGCNative(mNativeDict, mDictFilePath);
+ }
+
+ @UsedForTesting
+ public boolean needsToRunGC() {
+ if (!isValidDictionary()) return false;
+ return needsToRunGCNative(mNativeDict);
+ }
+
+ @UsedForTesting
+ public int calculateProbability(final int unigramProbability, final int bigramProbability) {
+ if (!isValidDictionary()) return NOT_A_PROBABILITY;
+ return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability);
+ }
+
@Override
public boolean shouldAutoCommit(final SuggestedWordInfo candidate) {
// TODO: actually use the confidence rather than use this completely broken heuristic
diff --git a/java/src/com/android/inputmethod/latin/Constants.java b/java/src/com/android/inputmethod/latin/Constants.java
index 6976f5f..58cae7d 100644
--- a/java/src/com/android/inputmethod/latin/Constants.java
+++ b/java/src/com/android/inputmethod/latin/Constants.java
@@ -226,7 +226,6 @@
}
public static final int MAX_INT_BIT_COUNT = 32;
- public static final String STRING_COMMA = ",";
private Constants() {
// This utility class is not publicly instantiable.
diff --git a/java/src/com/android/inputmethod/latin/InputPointers.java b/java/src/com/android/inputmethod/latin/InputPointers.java
index e96a46e..2e638aa 100644
--- a/java/src/com/android/inputmethod/latin/InputPointers.java
+++ b/java/src/com/android/inputmethod/latin/InputPointers.java
@@ -105,6 +105,17 @@
mTimes.append(times, startPos, length);
}
+ /**
+ * Shift to the left by elementCount, discarding elementCount pointers at the start.
+ * @param elementCount how many elements to shift.
+ */
+ public void shift(final int elementCount) {
+ mXCoordinates.shift(elementCount);
+ mYCoordinates.shift(elementCount);
+ mPointerIds.shift(elementCount);
+ mTimes.shift(elementCount);
+ }
+
public void reset() {
final int defaultCapacity = mDefaultCapacity;
mXCoordinates.reset(defaultCapacity);
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index d8a47a3..bfb9044 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -1847,10 +1847,18 @@
@Override
public void onUpdateBatchInput(final InputPointers batchPointers) {
- final SuggestedWordInfo candidate = mSuggestedWords.getAutoCommitCandidate();
- if (null != candidate) {
- if (candidate.mSourceDict.shouldAutoCommit(candidate)) {
- // TODO: implement auto-commit
+ if (mSettings.getCurrent().mPhraseGestureEnabled) {
+ final SuggestedWordInfo candidate = mSuggestedWords.getAutoCommitCandidate();
+ if (null != candidate) {
+ if (candidate.mSourceDict.shouldAutoCommit(candidate)) {
+ final String[] commitParts = candidate.mWord.split(" ", 2);
+ batchPointers.shift(candidate.mIndexOfTouchPointOfSecondWord);
+ promotePhantomSpace();
+ mConnection.commitText(commitParts[0], 0);
+ mSpaceState = SPACE_STATE_PHANTOM;
+ mKeyboardSwitcher.updateShiftState();
+ mWordComposer.setCapitalizedModeAtStartComposingTime(getActualCapsMode());
+ }
}
}
mInputUpdater.onUpdateBatchInput(batchPointers);
@@ -1863,12 +1871,24 @@
if (TextUtils.isEmpty(batchInputText)) {
return;
}
- mWordComposer.setBatchInputWord(batchInputText);
mConnection.beginBatchEdit();
if (SPACE_STATE_PHANTOM == mSpaceState) {
promotePhantomSpace();
}
- mConnection.setComposingText(batchInputText, 1);
+ if (mSettings.getCurrent().mPhraseGestureEnabled) {
+ // Find the last space
+ final int indexOfLastSpace = batchInputText.lastIndexOf(Constants.CODE_SPACE) + 1;
+ if (0 != indexOfLastSpace) {
+ mConnection.commitText(batchInputText.substring(0, indexOfLastSpace), 1);
+ showSuggestionStrip(suggestedWords.getSuggestedWordsForLastWordOfPhraseGesture());
+ }
+ final String lastWord = batchInputText.substring(indexOfLastSpace);
+ mWordComposer.setBatchInputWord(lastWord);
+ mConnection.setComposingText(lastWord, 1);
+ } else {
+ mWordComposer.setBatchInputWord(batchInputText);
+ mConnection.setComposingText(batchInputText, 1);
+ }
mExpectingUpdateSelection = true;
mConnection.endBatchEdit();
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
@@ -2665,6 +2685,13 @@
return prevWord;
}
+ private boolean isResumableWord(final String word, final SettingsValues settings) {
+ final int firstCodePoint = word.codePointAt(0);
+ return settings.isWordCodePoint(firstCodePoint)
+ && Constants.CODE_SINGLE_QUOTE != firstCodePoint
+ && Constants.CODE_DASH != firstCodePoint;
+ }
+
/**
* Check if the cursor is touching a word. If so, restart suggestions on this word, else
* do nothing.
@@ -2694,6 +2721,7 @@
if (numberOfCharsInWordBeforeCursor > mLastSelectionStart) return;
final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList();
final String typedWord = range.mWord.toString();
+ if (!isResumableWord(typedWord, currentSettings)) return;
int i = 0;
for (final SuggestionSpan span : range.getSuggestionSpansAtWord()) {
for (final String s : span.getSuggestions()) {
diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java
index 7815f4d..1684d47 100644
--- a/java/src/com/android/inputmethod/latin/Suggest.java
+++ b/java/src/com/android/inputmethod/latin/Suggest.java
@@ -460,7 +460,7 @@
private static final SuggestedWordInfoComparator sSuggestedWordInfoComparator =
new SuggestedWordInfoComparator();
- private static SuggestedWordInfo getTransformedSuggestedWordInfo(
+ /* package for test */ static SuggestedWordInfo getTransformedSuggestedWordInfo(
final SuggestedWordInfo wordInfo, final Locale locale, final boolean isAllUpperCase,
final boolean isFirstCharCapitalized, final int trailingSingleQuotesCount) {
final StringBuilder sb = new StringBuilder(wordInfo.mWord.length());
@@ -471,7 +471,12 @@
} else {
sb.append(wordInfo.mWord);
}
- for (int i = trailingSingleQuotesCount - 1; i >= 0; --i) {
+ // Appending quotes is here to help people quote words. However, it's not helpful
+ // when they type words with quotes toward the end like "it's" or "didn't", where
+ // it's more likely the user missed the last character (or didn't type it yet).
+ final int quotesToAppend = trailingSingleQuotesCount
+ - (-1 == wordInfo.mWord.indexOf(Constants.CODE_SINGLE_QUOTE) ? 0 : 1);
+ for (int i = quotesToAppend - 1; i >= 0; --i) {
sb.appendCodePoint(Constants.CODE_SINGLE_QUOTE);
}
return new SuggestedWordInfo(sb.toString(), wordInfo.mScore, wordInfo.mKind,
diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java
index 1763705..fed4cdb 100644
--- a/java/src/com/android/inputmethod/latin/SuggestedWords.java
+++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java
@@ -276,4 +276,24 @@
false /* willAutoCorrect */, mIsPunctuationSuggestions, mIsObsoleteSuggestions,
mIsPrediction);
}
+
+ // Creates a new SuggestedWordInfo from the currently suggested words that removes all but the
+ // last word of all suggestions, separated by a space. This is necessary because when we commit
+ // a multiple-word suggestion, the IME only retains the last word as the composing word, and
+ // we should only suggest replacements for this last word.
+ // TODO: make this work with languages without spaces.
+ public SuggestedWords getSuggestedWordsForLastWordOfPhraseGesture() {
+ final ArrayList<SuggestedWordInfo> newSuggestions = CollectionUtils.newArrayList();
+ for (int i = 0; i < mSuggestedWordInfoList.size(); ++i) {
+ final SuggestedWordInfo info = mSuggestedWordInfoList.get(i);
+ final int indexOfLastSpace = info.mWord.lastIndexOf(Constants.CODE_SPACE) + 1;
+ final String lastWord = info.mWord.substring(indexOfLastSpace);
+ newSuggestions.add(new SuggestedWordInfo(lastWord, info.mScore, info.mKind,
+ info.mSourceDict, SuggestedWordInfo.NOT_AN_INDEX,
+ SuggestedWordInfo.NOT_A_CONFIDENCE));
+ }
+ return new SuggestedWords(newSuggestions, mTypedWordValid,
+ mWillAutoCorrect, mIsPunctuationSuggestions, mIsObsoleteSuggestions,
+ mIsPrediction);
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
index f333b0d..70931f8 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
@@ -758,8 +758,15 @@
final FormatOptions formatOptions) {
int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate
+ getNodeHeaderSize(ptNode, formatOptions);
- if (ptNode.mFrequency >= 0) {
- positionOfChildrenPosField += FormatSpec.PTNODE_FREQUENCY_SIZE;
+ if (ptNode.isTerminal()) {
+ // A terminal node has either the terminal id or the frequency.
+ // If positionOfChildrenPosField is incorrect, we may crash when jumping to the children
+ // position.
+ if (formatOptions.mHasTerminalId) {
+ positionOfChildrenPosField += FormatSpec.PTNODE_TERMINAL_ID_SIZE;
+ } else {
+ positionOfChildrenPosField += FormatSpec.PTNODE_FREQUENCY_SIZE;
+ }
}
return null == ptNode.mChildren ? FormatSpec.NO_CHILDREN_ADDRESS
: ptNode.mChildren.mCachedAddressAfterUpdate - positionOfChildrenPosField;
diff --git a/java/src/com/android/inputmethod/latin/utils/ResizableIntArray.java b/java/src/com/android/inputmethod/latin/utils/ResizableIntArray.java
index 4c7739a..7c6fe93 100644
--- a/java/src/com/android/inputmethod/latin/utils/ResizableIntArray.java
+++ b/java/src/com/android/inputmethod/latin/utils/ResizableIntArray.java
@@ -132,6 +132,15 @@
}
}
+ /**
+ * Shift to the left by elementCount, discarding elementCount pointers at the start.
+ * @param elementCount how many elements to shift.
+ */
+ public void shift(final int elementCount) {
+ System.arraycopy(mArray, elementCount, mArray, 0, mLength - elementCount);
+ mLength -= elementCount;
+ }
+
@Override
public String toString() {
final StringBuilder sb = new StringBuilder();
diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
index be41840..121aecf 100644
--- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
@@ -16,16 +16,25 @@
package com.android.inputmethod.latin.utils;
-import android.text.TextUtils;
-
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.settings.SettingsValues;
+import android.text.TextUtils;
+import android.util.JsonReader;
+import android.util.JsonWriter;
+import android.util.Log;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import java.util.Locale;
public final class StringUtils {
+ private static final String TAG = StringUtils.class.getSimpleName();
public static final int CAPITALIZE_NONE = 0; // No caps, or mixed case
public static final int CAPITALIZE_FIRST = 1; // First only
public static final int CAPITALIZE_ALL = 2; // All caps
@@ -390,4 +399,67 @@
}
return bytes;
}
+
+ public static List<Object> jsonStrToList(String s) {
+ final ArrayList<Object> retval = CollectionUtils.newArrayList();
+ final JsonReader reader = new JsonReader(new StringReader(s));
+ try {
+ reader.beginArray();
+ while(reader.hasNext()) {
+ reader.beginObject();
+ while (reader.hasNext()) {
+ final String name = reader.nextName();
+ if (name.equals(Integer.class.getSimpleName())) {
+ retval.add(reader.nextInt());
+ } else if (name.equals(String.class.getSimpleName())) {
+ retval.add(reader.nextString());
+ } else {
+ Log.w(TAG, "Invalid name: " + name);
+ reader.skipValue();
+ }
+ }
+ reader.endObject();
+ }
+ reader.endArray();
+ return retval;
+ } catch (IOException e) {
+ } finally {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ }
+ }
+ return Collections.<Object>emptyList();
+ }
+
+ public static String listToJsonStr(List<Object> list) {
+ if (list == null || list.isEmpty()) {
+ return "";
+ }
+ final StringWriter sw = new StringWriter();
+ final JsonWriter writer = new JsonWriter(sw);
+ try {
+ writer.beginArray();
+ for (final Object o : list) {
+ writer.beginObject();
+ if (o instanceof Integer) {
+ writer.name(Integer.class.getSimpleName()).value((Integer)o);
+ } else if (o instanceof String) {
+ writer.name(String.class.getSimpleName()).value((String)o);
+ }
+ writer.endObject();
+ }
+ writer.endArray();
+ return sw.toString();
+ } catch (IOException e) {
+ } finally {
+ try {
+ if (writer != null) {
+ writer.close();
+ }
+ } catch (IOException e) {
+ }
+ }
+ return "";
+ }
}
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 6a36612..7f47493 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -46,8 +46,7 @@
sourceDirChars[sourceDirUtf8Length] = '\0';
DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy =
DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy(
- sourceDirChars, static_cast<int>(sourceDirUtf8Length),
- static_cast<int>(dictOffset), static_cast<int>(dictSize),
+ sourceDirChars, static_cast<int>(dictOffset), static_cast<int>(dictSize),
isUpdatable == JNI_TRUE);
if (!dictionaryStructureWithBufferPolicy) {
return 0;
@@ -59,6 +58,35 @@
return reinterpret_cast<jlong>(dictionary);
}
+static void latinime_BinaryDictionary_flush(JNIEnv *env, jclass clazz, jlong dict,
+ jstring filePath) {
+ Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+ if (!dictionary) return;
+ const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
+ char filePathChars[filePathUtf8Length + 1];
+ env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
+ filePathChars[filePathUtf8Length] = '\0';
+ dictionary->flush(filePathChars);
+}
+
+static bool latinime_BinaryDictionary_needsToRunGC(JNIEnv *env, jclass clazz,
+ jlong dict) {
+ Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+ if (!dictionary) return false;
+ return dictionary->needsToRunGC();
+}
+
+static void latinime_BinaryDictionary_flushWithGC(JNIEnv *env, jclass clazz, jlong dict,
+ jstring filePath) {
+ Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+ if (!dictionary) return;
+ const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
+ char filePathChars[filePathUtf8Length + 1];
+ env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
+ filePathChars[filePathUtf8Length] = '\0';
+ dictionary->flushWithGC(filePathChars);
+}
+
static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return;
@@ -160,8 +188,8 @@
return dictionary->getProbability(codePoints, wordLength);
}
-static jboolean latinime_BinaryDictionary_isValidBigram(JNIEnv *env, jclass clazz, jlong dict,
- jintArray word0, jintArray word1) {
+static jint latinime_BinaryDictionary_getBigramProbability(JNIEnv *env, jclass clazz,
+ jlong dict, jintArray word0, jintArray word1) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return JNI_FALSE;
const jsize word0Length = env->GetArrayLength(word0);
@@ -170,7 +198,8 @@
int word1CodePoints[word1Length];
env->GetIntArrayRegion(word0, 0, word0Length, word0CodePoints);
env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints);
- return dictionary->isValidBigram(word0CodePoints, word0Length, word1CodePoints, word1Length);
+ return dictionary->getBigramProbability(word0CodePoints, word0Length, word1CodePoints,
+ word1Length);
}
static jfloat latinime_BinaryDictionary_calcNormalizedScore(JNIEnv *env, jclass clazz,
@@ -241,6 +270,16 @@
word1Length);
}
+static int latinime_BinaryDictionary_calculateProbabilityNative(JNIEnv *env, jclass clazz,
+ jlong dict, jint unigramProbability, jint bigramProbability) {
+ Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+ if (!dictionary) {
+ return NOT_A_PROBABILITY;
+ }
+ return dictionary->getDictionaryStructurePolicy()->getProbability(unigramProbability,
+ bigramProbability);
+}
+
static const JNINativeMethod sMethods[] = {
{
const_cast<char *>("openNative"),
@@ -253,6 +292,21 @@
reinterpret_cast<void *>(latinime_BinaryDictionary_close)
},
{
+ const_cast<char *>("flushNative"),
+ const_cast<char *>("(JLjava/lang/String;)V"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_flush)
+ },
+ {
+ const_cast<char *>("needsToRunGCNative"),
+ const_cast<char *>("(J)Z"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_needsToRunGC)
+ },
+ {
+ const_cast<char *>("flushWithGCNative"),
+ const_cast<char *>("(JLjava/lang/String;)V"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_flushWithGC)
+ },
+ {
const_cast<char *>("getSuggestionsNative"),
const_cast<char *>("(JJJ[I[I[I[I[III[I[I[I[I[I[I[I)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_getSuggestions)
@@ -263,9 +317,9 @@
reinterpret_cast<void *>(latinime_BinaryDictionary_getProbability)
},
{
- const_cast<char *>("isValidBigramNative"),
- const_cast<char *>("(J[I[I)Z"),
- reinterpret_cast<void *>(latinime_BinaryDictionary_isValidBigram)
+ const_cast<char *>("getBigramProbabilityNative"),
+ const_cast<char *>("(J[I[I)I"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_getBigramProbability)
},
{
const_cast<char *>("calcNormalizedScoreNative"),
@@ -291,6 +345,11 @@
const_cast<char *>("removeBigramWordsNative"),
const_cast<char *>("(J[I[I)V"),
reinterpret_cast<void *>(latinime_BinaryDictionary_removeBigramWords)
+ },
+ {
+ const_cast<char *>("calculateProbabilityNative"),
+ const_cast<char *>("(JII)I"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_calculateProbabilityNative)
}
};
diff --git a/native/jni/src/suggest/core/dicnode/dic_node.h b/native/jni/src/suggest/core/dicnode/dic_node.h
index 3770153..41ef9d2 100644
--- a/native/jni/src/suggest/core/dicnode/dic_node.h
+++ b/native/jni/src/suggest/core/dicnode/dic_node.h
@@ -143,7 +143,7 @@
dicNode->mDicNodeState.mDicNodeStatePrevWord.getPrevWordLength(),
dicNode->getOutputWordBuf(),
dicNode->mDicNodeProperties.getDepth(),
- dicNode->mDicNodeState.mDicNodeStatePrevWord.mPrevSpacePositions,
+ dicNode->mDicNodeState.mDicNodeStatePrevWord.getSecondWordFirstInputIndex(),
mDicNodeState.mDicNodeStateInput.getInputIndex(0) /* lastInputIndex */);
PROF_NODE_COPY(&dicNode->mProfiler, mProfiler);
}
@@ -321,8 +321,13 @@
DUMP_WORD_AND_SCORE("OUTPUT");
}
- void outputSpacePositionsResult(int *spaceIndices) const {
- mDicNodeState.mDicNodeStatePrevWord.outputSpacePositions(spaceIndices);
+ int getSecondWordFirstInputIndex(const ProximityInfoState *const pInfoState) const {
+ const int inputIndex = mDicNodeState.mDicNodeStatePrevWord.getSecondWordFirstInputIndex();
+ if (inputIndex == NOT_AN_INDEX) {
+ return NOT_AN_INDEX;
+ } else {
+ return pInfoState->getInputIndexOfSampledPoint(inputIndex);
+ }
}
bool hasMultipleWords() const {
@@ -573,7 +578,11 @@
}
}
- AK_FORCE_INLINE void updateInputIndexG(DicNode_InputStateG *inputStateG) {
+ AK_FORCE_INLINE void updateInputIndexG(const DicNode_InputStateG *const inputStateG) {
+ if (mDicNodeState.mDicNodeStatePrevWord.getPrevWordCount() == 1 && isFirstLetter()) {
+ mDicNodeState.mDicNodeStatePrevWord.setSecondWordFirstInputIndex(
+ inputStateG->mInputIndex);
+ }
mDicNodeState.mDicNodeStateInput.updateInputIndexG(inputStateG->mPointerId,
inputStateG->mInputIndex, inputStateG->mPrevCodePoint,
inputStateG->mTerminalDiffCost, inputStateG->mRawLength);
diff --git a/native/jni/src/suggest/core/dicnode/internal/dic_node_state_prevword.h b/native/jni/src/suggest/core/dicnode/internal/dic_node_state_prevword.h
index b7af970..b898620 100644
--- a/native/jni/src/suggest/core/dicnode/internal/dic_node_state_prevword.h
+++ b/native/jni/src/suggest/core/dicnode/internal/dic_node_state_prevword.h
@@ -22,6 +22,7 @@
#include "defines.h"
#include "suggest/core/dicnode/dic_node_utils.h"
+#include "suggest/core/layout/proximity_info_state.h"
namespace latinime {
@@ -29,9 +30,8 @@
public:
AK_FORCE_INLINE DicNodeStatePrevWord()
: mPrevWordCount(0), mPrevWordLength(0), mPrevWordStart(0), mPrevWordProbability(0),
- mPrevWordNodePos(NOT_A_DICT_POS) {
+ mPrevWordNodePos(NOT_A_DICT_POS), mSecondWordFirstInputIndex(NOT_AN_INDEX) {
memset(mPrevWord, 0, sizeof(mPrevWord));
- memset(mPrevSpacePositions, 0, sizeof(mPrevSpacePositions));
}
virtual ~DicNodeStatePrevWord() {}
@@ -42,7 +42,7 @@
mPrevWordStart = 0;
mPrevWordProbability = -1;
mPrevWordNodePos = NOT_A_DICT_POS;
- memset(mPrevSpacePositions, 0, sizeof(mPrevSpacePositions));
+ mSecondWordFirstInputIndex = NOT_AN_INDEX;
}
void init(const int prevWordNodePos) {
@@ -51,7 +51,7 @@
mPrevWordStart = 0;
mPrevWordProbability = -1;
mPrevWordNodePos = prevWordNodePos;
- memset(mPrevSpacePositions, 0, sizeof(mPrevSpacePositions));
+ mSecondWordFirstInputIndex = NOT_AN_INDEX;
}
// Init by copy
@@ -61,14 +61,14 @@
mPrevWordStart = prevWord->mPrevWordStart;
mPrevWordProbability = prevWord->mPrevWordProbability;
mPrevWordNodePos = prevWord->mPrevWordNodePos;
+ mSecondWordFirstInputIndex = prevWord->mSecondWordFirstInputIndex;
memcpy(mPrevWord, prevWord->mPrevWord, prevWord->mPrevWordLength * sizeof(mPrevWord[0]));
- memcpy(mPrevSpacePositions, prevWord->mPrevSpacePositions, sizeof(mPrevSpacePositions));
}
void init(const int16_t prevWordCount, const int16_t prevWordProbability,
const int prevWordNodePos, const int *const src0, const int16_t length0,
- const int *const src1, const int16_t length1, const int *const prevSpacePositions,
- const int lastInputIndex) {
+ const int *const src1, const int16_t length1,
+ const int prevWordSecondWordFirstInputIndex, const int lastInputIndex) {
mPrevWordCount = min(prevWordCount, static_cast<int16_t>(MAX_RESULTS));
mPrevWordProbability = prevWordProbability;
mPrevWordNodePos = prevWordNodePos;
@@ -80,8 +80,7 @@
mPrevWord[twoWordsLen] = KEYCODE_SPACE;
mPrevWordStart = length0;
mPrevWordLength = static_cast<int16_t>(twoWordsLen + 1);
- memcpy(mPrevSpacePositions, prevSpacePositions, sizeof(mPrevSpacePositions));
- mPrevSpacePositions[mPrevWordCount - 1] = lastInputIndex;
+ mSecondWordFirstInputIndex = prevWordSecondWordFirstInputIndex;
}
void truncate(const int offset) {
@@ -96,11 +95,12 @@
mPrevWordLength = newPrevWordLength;
}
- void outputSpacePositions(int *spaceIndices) const {
- // Convert uint16_t to int
- for (int i = 0; i < MAX_RESULTS; i++) {
- spaceIndices[i] = mPrevSpacePositions[i];
- }
+ void setSecondWordFirstInputIndex(const int inputIndex) {
+ mSecondWordFirstInputIndex = inputIndex;
+ }
+
+ int getSecondWordFirstInputIndex() const {
+ return mSecondWordFirstInputIndex;
}
// TODO: remove
@@ -138,8 +138,6 @@
// TODO: Move to private
int mPrevWord[MAX_WORD_LENGTH];
- // TODO: Move to private
- int mPrevSpacePositions[MAX_RESULTS];
private:
// Caution!!!
@@ -150,6 +148,7 @@
int16_t mPrevWordStart;
int16_t mPrevWordProbability;
int mPrevWordNodePos;
+ int mSecondWordFirstInputIndex;
};
} // namespace latinime
#endif // LATINIME_DIC_NODE_STATE_PREVWORD_H
diff --git a/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp b/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp
index 425b076..5ba71c1 100644
--- a/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp
@@ -150,24 +150,26 @@
return mDictionaryStructurePolicy->getBigramsPositionOfNode(pos);
}
-bool BigramDictionary::isValidBigram(const int *word0, int length0, const int *word1,
+int BigramDictionary::getBigramProbability(const int *word0, int length0, const int *word1,
int length1) const {
int pos = getBigramListPositionForWord(word0, length0, false /* forceLowerCaseSearch */);
// getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams
- if (NOT_A_DICT_POS == pos) return false;
+ if (NOT_A_DICT_POS == pos) return NOT_A_PROBABILITY;
int nextWordPos = mDictionaryStructurePolicy->getTerminalNodePositionOfWord(word1, length1,
false /* forceLowerCaseSearch */);
- if (NOT_A_DICT_POS == nextWordPos) return false;
+ if (NOT_A_DICT_POS == nextWordPos) return NOT_A_PROBABILITY;
BinaryDictionaryBigramsIterator bigramsIt(
mDictionaryStructurePolicy->getBigramsStructurePolicy(), pos);
while (bigramsIt.hasNext()) {
bigramsIt.next();
if (bigramsIt.getBigramPos() == nextWordPos) {
- return true;
+ return mDictionaryStructurePolicy->getProbability(
+ mDictionaryStructurePolicy->getUnigramProbabilityOfPtNode(nextWordPos),
+ bigramsIt.getProbability());
}
}
- return false;
+ return NOT_A_PROBABILITY;
}
// TODO: Move functions related to bigram to here
diff --git a/native/jni/src/suggest/core/dictionary/bigram_dictionary.h b/native/jni/src/suggest/core/dictionary/bigram_dictionary.h
index 99b964c..8af7ee7 100644
--- a/native/jni/src/suggest/core/dictionary/bigram_dictionary.h
+++ b/native/jni/src/suggest/core/dictionary/bigram_dictionary.h
@@ -29,7 +29,7 @@
int getPredictions(const int *word, int length, int *outBigramCodePoints,
int *outBigramProbability, int *outputTypes) const;
- bool isValidBigram(const int *word1, int length1, const int *word2, int length2) const;
+ int getBigramProbability(const int *word1, int length1, const int *word2, int length2) const;
~BigramDictionary();
private:
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp
index 29fe7ab..ec1b63a 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp
@@ -93,8 +93,9 @@
return getDictionaryStructurePolicy()->getUnigramProbabilityOfPtNode(pos);
}
-bool Dictionary::isValidBigram(const int *word0, int length0, const int *word1, int length1) const {
- return mBigramDictionary->isValidBigram(word0, length0, word1, length1);
+int Dictionary::getBigramProbability(const int *word0, int length0, const int *word1,
+ int length1) const {
+ return mBigramDictionary->getBigramProbability(word0, length0, word1, length1);
}
void Dictionary::addUnigramWord(const int *const word, const int length, const int probability) {
@@ -112,6 +113,18 @@
mDictionaryStructureWithBufferPolicy->removeBigramWords(word0, length0, word1, length1);
}
+void Dictionary::flush(const char *const filePath) {
+ mDictionaryStructureWithBufferPolicy->flush(filePath);
+}
+
+void Dictionary::flushWithGC(const char *const filePath) {
+ mDictionaryStructureWithBufferPolicy->flushWithGC(filePath);
+}
+
+bool Dictionary::needsToRunGC() {
+ return mDictionaryStructureWithBufferPolicy->needsToRunGC();
+}
+
void Dictionary::logDictionaryInfo(JNIEnv *const env) const {
const int BUFFER_SIZE = 16;
int dictionaryIdCodePointBuffer[BUFFER_SIZE];
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h
index 0afe5a5..9744474 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.h
+++ b/native/jni/src/suggest/core/dictionary/dictionary.h
@@ -67,7 +67,7 @@
int getProbability(const int *word, int length) const;
- bool isValidBigram(const int *word0, int length0, const int *word1, int length1) const;
+ int getBigramProbability(const int *word0, int length0, const int *word1, int length1) const;
void addUnigramWord(const int *const word, const int length, const int probability);
@@ -77,6 +77,12 @@
void removeBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1);
+ void flush(const char *const filePath);
+
+ void flushWithGC(const char *const filePath);
+
+ bool needsToRunGC();
+
const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
return mDictionaryStructureWithBufferPolicy;
}
diff --git a/native/jni/src/suggest/core/layout/proximity_info_state.h b/native/jni/src/suggest/core/layout/proximity_info_state.h
index 01bf818..c94060f 100644
--- a/native/jni/src/suggest/core/layout/proximity_info_state.h
+++ b/native/jni/src/suggest/core/layout/proximity_info_state.h
@@ -130,6 +130,10 @@
return mSampledInputYs[index];
}
+ int getInputIndexOfSampledPoint(const int sampledIndex) const {
+ return mSampledInputIndice[sampledIndex];
+ }
+
bool hasSpaceProximity(const int index) const;
int getLengthCache(const int index) const {
diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
index c8cbbcf..24d1b8b 100644
--- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
+++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
@@ -74,6 +74,12 @@
virtual bool removeBigramWords(const int *const word0, const int length0,
const int *const word1, const int length1) = 0;
+ virtual void flush(const char *const filePath) = 0;
+
+ virtual void flushWithGC(const char *const filePath) = 0;
+
+ virtual bool needsToRunGC() const = 0;
+
protected:
DictionaryStructureWithBufferPolicy() {}
diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.h b/native/jni/src/suggest/core/session/dic_traverse_session.h
index e2ef5fc..e0b1c67 100644
--- a/native/jni/src/suggest/core/session/dic_traverse_session.h
+++ b/native/jni/src/suggest/core/session/dic_traverse_session.h
@@ -113,7 +113,9 @@
if (usedPointerCount != 1) {
return false;
}
- *pointerId = usedPointerId;
+ if (pointerId) {
+ *pointerId = usedPointerId;
+ }
return true;
}
diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp
index e788e91..0c925be 100644
--- a/native/jni/src/suggest/core/suggest.cpp
+++ b/native/jni/src/suggest/core/suggest.cpp
@@ -117,7 +117,7 @@
* Outputs the final list of suggestions (i.e., terminal nodes).
*/
int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequencies,
- int *outputCodePoints, int *spaceIndices, int *outputTypes) const {
+ int *outputCodePoints, int *outputIndicesToPartialCommit, int *outputTypes) const {
#if DEBUG_EVALUATE_MOST_PROBABLE_STRING
const int terminalSize = 0;
#else
@@ -139,6 +139,7 @@
SCORING->getMostProbableString(traverseSession, terminalSize, languageWeight,
&outputCodePoints[0], &outputTypes[0], &frequencies[0]);
if (hasMostProbableString) {
+ outputIndicesToPartialCommit[outputWordIndex] = NOT_AN_INDEX;
++outputWordIndex;
}
@@ -160,6 +161,9 @@
|| (traverseSession->getInputSize()
>= MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT
&& terminals[0].hasMultipleWords())) : false;
+ // TODO: have partial commit work even with multiple pointers.
+ const bool outputSecondWordFirstLetterInputIndex =
+ traverseSession->isOnlyOnePointerUsed(0 /* pointerId */);
// Output suggestion results here
for (int terminalIndex = 0; terminalIndex < terminalSize && outputWordIndex < MAX_RESULTS;
++terminalIndex) {
@@ -194,18 +198,21 @@
terminalDicNode->isExactMatch()
|| (forceCommitMultiWords && terminalDicNode->hasMultipleWords())
|| (isValidWord && SCORING->doesAutoCorrectValidWord()));
- maxScore = max(maxScore, finalScore);
-
- // TODO: Implement a smarter auto-commit method for handling multi-word suggestions.
- // Index for top typing suggestion should be 0.
- if (isValidWord && outputWordIndex == 0) {
- terminalDicNode->outputSpacePositionsResult(spaceIndices);
+ if (maxScore < finalScore && isValidWord) {
+ maxScore = finalScore;
}
// Don't output invalid words. However, we still need to submit their shortcuts if any.
if (isValidWord) {
outputTypes[outputWordIndex] = Dictionary::KIND_CORRECTION | outputTypeFlags;
frequencies[outputWordIndex] = finalScore;
+ if (outputSecondWordFirstLetterInputIndex) {
+ outputIndicesToPartialCommit[outputWordIndex] =
+ terminalDicNode->getSecondWordFirstInputIndex(
+ traverseSession->getProximityInfoState(0));
+ } else {
+ outputIndicesToPartialCommit[outputWordIndex] = NOT_AN_INDEX;
+ }
// Populate the outputChars array with the suggested word.
const int startIndex = outputWordIndex * MAX_WORD_LENGTH;
terminalDicNode->outputResult(&outputCodePoints[startIndex]);
@@ -220,8 +227,19 @@
// Shortcut is not supported for multiple words suggestions.
// TODO: Check shortcuts during traversal for multiple words suggestions.
const bool sameAsTyped = TRAVERSAL->sameAsTyped(traverseSession, terminalDicNode);
- outputWordIndex = ShortcutUtils::outputShortcuts(&shortcutIt, outputWordIndex,
- finalScore, outputCodePoints, frequencies, outputTypes, sameAsTyped);
+ const int updatedOutputWordIndex = ShortcutUtils::outputShortcuts(&shortcutIt,
+ outputWordIndex, finalScore, outputCodePoints, frequencies, outputTypes,
+ sameAsTyped);
+ const int secondWordFirstInputIndex = terminalDicNode->getSecondWordFirstInputIndex(
+ traverseSession->getProximityInfoState(0));
+ for (int i = outputWordIndex; i < updatedOutputWordIndex; ++i) {
+ if (outputSecondWordFirstLetterInputIndex) {
+ outputIndicesToPartialCommit[i] = secondWordFirstInputIndex;
+ } else {
+ outputIndicesToPartialCommit[i] = NOT_AN_INDEX;
+ }
+ }
+ outputWordIndex = updatedOutputWordIndex;
}
DicNode::managedDelete(terminalDicNode);
}
diff --git a/native/jni/src/suggest/core/suggest.h b/native/jni/src/suggest/core/suggest.h
index 875cbe4..b240196 100644
--- a/native/jni/src/suggest/core/suggest.h
+++ b/native/jni/src/suggest/core/suggest.h
@@ -55,7 +55,7 @@
void createNextWordDicNode(DicTraverseSession *traverseSession, DicNode *dicNode,
const bool spaceSubstitution) const;
int outputSuggestions(DicTraverseSession *traverseSession, int *frequencies,
- int *outputCodePoints, int *outputIndices, int *outputTypes) const;
+ int *outputCodePoints, int *outputIndicesToPartialCommit, int *outputTypes) const;
void initializeSearch(DicTraverseSession *traverseSession, int commitPoint) const;
void expandCurrentDicNodes(DicTraverseSession *traverseSession) const;
void processTerminalDicNode(DicTraverseSession *traverseSession, DicNode *dicNode) const;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
index b8a5f27..4c44d22 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
@@ -54,11 +54,13 @@
}
}
-bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos) {
+bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos,
+ int *outBigramsCount) {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos);
if (usesAdditionalBuffer) {
*fromPos -= mBuffer->getOriginalBufferSize();
}
+ *outBigramsCount = 0;
BigramListReadWriteUtils::BigramFlags flags;
do {
// The buffer address can be changed after calling buffer writing methods.
@@ -91,6 +93,7 @@
toPos)) {
return false;
}
+ (*outBigramsCount)++;
} while(BigramListReadWriteUtils::hasNext(flags));
if (usesAdditionalBuffer) {
*fromPos += mBuffer->getOriginalBufferSize();
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
index c45e26a..dafb62d 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
@@ -45,8 +45,9 @@
void skipAllBigrams(int *const pos) const;
// Copy bigrams from the bigram list that starts at fromPos to toPos and advance these
- // positions after bigram lists. This method skips invalid bigram entries.
- bool copyAllBigrams(int *const fromPos, int *const toPos);
+ // positions after bigram lists. This method skips invalid bigram entries and write the valid
+ // bigram entry count to outBigramsCount.
+ bool copyAllBigrams(int *const fromPos, int *const toPos, int *outBigramsCount);
bool addNewBigramEntryToBigramList(const int bigramPos, const int probability, int *const pos);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp
index 434858d..ff80dd2 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp
@@ -27,12 +27,12 @@
namespace latinime {
/* static */ DictionaryStructureWithBufferPolicy *DictionaryStructureWithBufferPolicyFactory
- ::newDictionaryStructureWithBufferPolicy(const char *const path, const int pathLength,
- const int bufOffset, const int size, const bool isUpdatable) {
+ ::newDictionaryStructureWithBufferPolicy(const char *const path, const int bufOffset,
+ const int size, const bool isUpdatable) {
// Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
// impl classes of DictionaryStructureWithBufferPolicy.
- const MmappedBuffer *const mmapedBuffer = MmappedBuffer::openBuffer(path, pathLength, bufOffset,
- size, isUpdatable);
+ const MmappedBuffer *const mmapedBuffer = MmappedBuffer::openBuffer(path, bufOffset, size,
+ isUpdatable);
if (!mmapedBuffer) {
return 0;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h
index 1cb7a89..8cebc3b 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h
@@ -27,8 +27,7 @@
class DictionaryStructureWithBufferPolicyFactory {
public:
static DictionaryStructureWithBufferPolicy *newDictionaryStructureWithBufferPolicy(
- const char *const path, const int pathLength, const int bufOffset, const int size,
- const bool isUpdatable);
+ const char *const path, const int bufOffset, const int size, const bool isUpdatable);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
index cccc090..2198a13 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
@@ -243,4 +243,31 @@
return writingHelper.removeBigramWords(word0Pos, word1Pos);
}
+void DynamicPatriciaTriePolicy::flush(const char *const filePath) {
+ if (!mBuffer->isUpdatable()) {
+ AKLOGI("Warning: flush() is called for non-updatable dictionary.");
+ return;
+ }
+ DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
+ &mBigramListPolicy, &mShortcutListPolicy);
+ writingHelper.writeToDictFile(filePath, mBuffer->getBuffer(), mHeaderPolicy.getSize());
+}
+
+void DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) {
+ if (!mBuffer->isUpdatable()) {
+ AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
+ return;
+ }
+ // TODO: Implement.
+}
+
+bool DynamicPatriciaTriePolicy::needsToRunGC() const {
+ if (!mBuffer->isUpdatable()) {
+ AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
+ return false;
+ }
+ // TODO: Implement.
+ return false;
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
index 50c7240..2cbb0ff 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
@@ -85,6 +85,12 @@
bool removeBigramWords(const int *const word0, const int length0, const int *const word1,
const int length1);
+ void flush(const char *const filePath);
+
+ void flushWithGC(const char *const filePath);
+
+ bool needsToRunGC() const;
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
index b80b9b3..a67c0d9 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
@@ -16,6 +16,9 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"
+#include <cstdio>
+#include <cstring>
+
#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h"
@@ -27,6 +30,8 @@
namespace latinime {
const int DynamicPatriciaTrieWritingHelper::CHILDREN_POSITION_FIELD_SIZE = 3;
+const char *const DynamicPatriciaTrieWritingHelper::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
+ ".tmp";
bool DynamicPatriciaTrieWritingHelper::addUnigramWord(
DynamicPatriciaTrieReadingHelper *const readingHelper,
@@ -125,11 +130,45 @@
bool DynamicPatriciaTrieWritingHelper::removeBigramWords(const int word0Pos, const int word1Pos) {
DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy);
nodeReader.fetchNodeInfoFromBuffer(word0Pos);
- if (nodeReader.isDeleted() || nodeReader.getBigramsPos() == NOT_A_DICT_POS) {
+ if (nodeReader.getBigramsPos() == NOT_A_DICT_POS) {
return false;
}
- // TODO: Implement.
- return false;
+ return mBigramPolicy->removeBigram(nodeReader.getBigramsPos(), word1Pos);
+}
+
+void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileName,
+ const uint8_t *const headerBuf, const int headerSize) {
+ const int tmpFileNameBufSize = strlen(fileName)
+ + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1;
+ char tmpFileName[tmpFileNameBufSize];
+ snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName,
+ TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
+ FILE *const file = fopen(tmpFileName, "wb");
+ if (!file) {
+ return;
+ }
+ // Write header.
+ if (fwrite(headerBuf, headerSize, 1, file) < 1) {
+ fclose(file);
+ remove(tmpFileName);
+ return;
+ }
+ // Write data in original buffer.
+ if (fwrite(mBuffer->getBuffer(false /* usesAdditionalBuffer */),
+ mBuffer->getOriginalBufferSize(), 1, file) < 1) {
+ fclose(file);
+ remove(tmpFileName);
+ return;
+ }
+ // Write data in additional buffer.
+ if (fwrite(mBuffer->getBuffer(true /* usesAdditionalBuffer */),
+ mBuffer->getTailPosition() - mBuffer->getOriginalBufferSize(), 1, file) < 1) {
+ fclose(file);
+ remove(tmpFileName);
+ return;
+ }
+ fclose(file);
+ rename(tmpFileName, fileName);
}
bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition(
@@ -193,13 +232,9 @@
const int originalBigramListPos, const int originalShortcutListPos,
int *const writingPos) {
const int nodePos = *writingPos;
- // Create node flags and write them.
- const PatriciaTrieReadingUtils::NodeFlags nodeFlags =
- PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord,
- probability != NOT_A_PROBABILITY, originalShortcutListPos != NOT_A_DICT_POS,
- originalBigramListPos != NOT_A_DICT_POS, codePointCount > 1,
- 3 /* childrenPositionFieldSize */);
- if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, nodeFlags,
+ // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the
+ // PtNode writing.
+ if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, 0 /* nodeFlags */,
writingPos)) {
return false;
}
@@ -212,7 +247,7 @@
// Write code points
if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mBuffer, codePoints,
codePointCount, writingPos)) {
- return false;;
+ return false;
}
// Write probability when the probability is a valid probability, which means this node is
// terminal.
@@ -235,12 +270,25 @@
}
}
// Copy bigram list when the originalBigramListPos is valid dictionary position.
+ int bigramCount = 0;
if (originalBigramListPos != NOT_A_DICT_POS) {
int fromPos = originalBigramListPos;
- if (!mBigramPolicy->copyAllBigrams(&fromPos, writingPos)) {
+ if (!mBigramPolicy->copyAllBigrams(&fromPos, writingPos, &bigramCount)) {
return false;
}
}
+ // Create node flags and write them.
+ PatriciaTrieReadingUtils::NodeFlags nodeFlags =
+ PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord,
+ probability != NOT_A_PROBABILITY /* isTerminal */,
+ originalShortcutListPos != NOT_A_DICT_POS /* hasShortcutTargets */,
+ bigramCount > 0 /* hasBigrams */, codePointCount > 1 /* hasMultipleChars */,
+ CHILDREN_POSITION_FIELD_SIZE);
+ int flagsFieldPos = nodePos;
+ if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, nodeFlags,
+ &flagsFieldPos)) {
+ return false;
+ }
return true;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
index 20e35ab..faf7a4e 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
@@ -17,6 +17,8 @@
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H
+#include <stdint.h>
+
#include "defines.h"
namespace latinime {
@@ -46,10 +48,14 @@
// Remove a bigram relation from word0Pos to word1Pos.
bool removeBigramWords(const int word0Pos, const int word1Pos);
+ void writeToDictFile(const char *const fileName, const uint8_t *const headerBuf,
+ const int headerSize);
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper);
static const int CHILDREN_POSITION_FIELD_SIZE;
+ static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE;
BufferWithExtendableBuffer *const mBuffer;
DynamicBigramListPolicy *const mBigramPolicy;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
index 75d9762..cee3e4a 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
@@ -96,6 +96,22 @@
return false;
}
+ void flush(const char *const filePath) {
+ // This method should not be called for non-updatable dictionary.
+ AKLOGI("Warning: flush() is called for non-updatable dictionary.");
+ }
+
+ void flushWithGC(const char *const filePath) {
+ // This method should not be called for non-updatable dictionary.
+ AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
+ }
+
+ bool needsToRunGC() const {
+ // This method should not be called for non-updatable dictionary.
+ AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
+ return false;
+ }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h
index 6febd78..6b69116 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h
@@ -29,8 +29,8 @@
class MmappedBuffer {
public:
- static MmappedBuffer* openBuffer(const char *const path, const int pathLength,
- const int bufferOffset, const int bufferSize, const bool isUpdatable) {
+ static MmappedBuffer* openBuffer(const char *const path, const int bufferOffset,
+ const int bufferSize, const bool isUpdatable) {
const int openMode = isUpdatable ? O_RDWR : O_RDONLY;
const int mmapFd = open(path, openMode);
if (mmapFd < 0) {
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
index f9dd35a..d8105ba 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
@@ -151,7 +151,7 @@
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
for (int i = 0; i < wordCount; ++i) {
final String word = CodePointUtils.generateWord(random, codePointSet);
- probabilityMap.put(word, random.nextInt() & 0xFF);
+ probabilityMap.put(word, random.nextInt(0xFF));
}
for (String word : probabilityMap.keySet()) {
binaryDictionary.addUnigramWord(word, probabilityMap.get(word));
@@ -163,8 +163,6 @@
}
public void testAddBigramWords() {
- // TODO: Add a test to check the frequency of the bigram score which uses current value
- // calculated in the native code
File dictFile = null;
try {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
@@ -179,6 +177,130 @@
final int unigramProbability = 100;
final int bigramProbability = 10;
+ final int updatedBigramProbability = 15;
+ binaryDictionary.addUnigramWord("aaa", unigramProbability);
+ binaryDictionary.addUnigramWord("abb", unigramProbability);
+ binaryDictionary.addUnigramWord("bcc", unigramProbability);
+ binaryDictionary.addBigramWords("aaa", "abb", bigramProbability);
+ binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability);
+ binaryDictionary.addBigramWords("abb", "aaa", bigramProbability);
+ binaryDictionary.addBigramWords("abb", "bcc", bigramProbability);
+
+ final int probability = binaryDictionary.calculateProbability(unigramProbability,
+ bigramProbability);
+ assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
+ assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc"));
+ assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa"));
+ assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc"));
+ assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb"));
+ assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc"));
+ assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa"));
+ assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc"));
+
+ binaryDictionary.addBigramWords("aaa", "abb", updatedBigramProbability);
+ final int updatedProbability = binaryDictionary.calculateProbability(unigramProbability,
+ updatedBigramProbability);
+ assertEquals(updatedProbability, binaryDictionary.getBigramProbability("aaa", "abb"));
+
+ assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa"));
+ assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc"));
+ assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa"));
+ assertEquals(Dictionary.NOT_A_PROBABILITY,
+ binaryDictionary.getBigramProbability("bcc", "aaa"));
+ assertEquals(Dictionary.NOT_A_PROBABILITY,
+ binaryDictionary.getBigramProbability("bcc", "bbc"));
+ assertEquals(Dictionary.NOT_A_PROBABILITY,
+ binaryDictionary.getBigramProbability("aaa", "aaa"));
+
+ // Testing bigram link.
+ binaryDictionary.addUnigramWord("abcde", unigramProbability);
+ binaryDictionary.addUnigramWord("fghij", unigramProbability);
+ binaryDictionary.addBigramWords("abcde", "fghij", bigramProbability);
+ binaryDictionary.addUnigramWord("fgh", unigramProbability);
+ binaryDictionary.addUnigramWord("abc", unigramProbability);
+ binaryDictionary.addUnigramWord("f", unigramProbability);
+ assertEquals(probability, binaryDictionary.getBigramProbability("abcde", "fghij"));
+ assertEquals(Dictionary.NOT_A_PROBABILITY,
+ binaryDictionary.getBigramProbability("abcde", "fgh"));
+ binaryDictionary.addBigramWords("abcde", "fghij", updatedBigramProbability);
+ assertEquals(updatedProbability, binaryDictionary.getBigramProbability("abcde", "fghij"));
+
+ dictFile.delete();
+ }
+
+ public void testRandomlyAddBigramWords() {
+ final int wordCount = 100;
+ final int bigramCount = 1000;
+ final int codePointSetSize = 50;
+ final int seed = 11111;
+
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ } catch (UnsupportedFormatException e) {
+ fail("UnsupportedFormatException while writing an initial dictionary : " + e);
+ }
+ BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ final ArrayList<String> words = new ArrayList<String>();
+ // Test a word that isn't contained within the dictionary.
+ final Random random = new Random(seed);
+ final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
+ final int[] unigramProbabilities = new int[wordCount];
+ for (int i = 0; i < wordCount; ++i) {
+ final String word = CodePointUtils.generateWord(random, codePointSet);
+ words.add(word);
+ final int unigramProbability = random.nextInt(0xFF);
+ unigramProbabilities[i] = unigramProbability;
+ binaryDictionary.addUnigramWord(word, unigramProbability);
+ }
+
+ final int[][] probabilities = new int[wordCount][wordCount];
+
+ for (int i = 0; i < wordCount; ++i) {
+ for (int j = 0; j < wordCount; ++j) {
+ probabilities[i][j] = Dictionary.NOT_A_PROBABILITY;
+ }
+ }
+
+ for (int i = 0; i < bigramCount; i++) {
+ final int word0Index = random.nextInt(wordCount);
+ final int word1Index = random.nextInt(wordCount);
+ final String word0 = words.get(word0Index);
+ final String word1 = words.get(word1Index);
+ final int bigramProbability = random.nextInt(0xF);
+ probabilities[word0Index][word1Index] = binaryDictionary.calculateProbability(
+ unigramProbabilities[word1Index], bigramProbability);
+ binaryDictionary.addBigramWords(word0, word1, bigramProbability);
+ }
+
+ for (int i = 0; i < words.size(); i++) {
+ for (int j = 0; j < words.size(); j++) {
+ assertEquals(probabilities[i][j],
+ binaryDictionary.getBigramProbability(words.get(i), words.get(j)));
+ }
+ }
+
+ dictFile.delete();
+ }
+
+ public void testRemoveBigramWords() {
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ } catch (UnsupportedFormatException e) {
+ fail("UnsupportedFormatException while writing an initial dictionary : " + e);
+ }
+ BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ final int unigramProbability = 100;
+ final int bigramProbability = 10;
binaryDictionary.addUnigramWord("aaa", unigramProbability);
binaryDictionary.addUnigramWord("abb", unigramProbability);
binaryDictionary.addUnigramWord("bcc", unigramProbability);
@@ -192,20 +314,28 @@
assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa"));
assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc"));
- assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa"));
- assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc"));
- assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa"));
+ binaryDictionary.removeBigramWords("aaa", "abb");
+ assertEquals(false, binaryDictionary.isValidBigram("aaa", "abb"));
+ binaryDictionary.addBigramWords("aaa", "abb", bigramProbability);
+ assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
+
+
+ binaryDictionary.removeBigramWords("aaa", "bcc");
+ assertEquals(false, binaryDictionary.isValidBigram("aaa", "bcc"));
+ binaryDictionary.removeBigramWords("abb", "aaa");
+ assertEquals(false, binaryDictionary.isValidBigram("abb", "aaa"));
+ binaryDictionary.removeBigramWords("abb", "bcc");
+ assertEquals(false, binaryDictionary.isValidBigram("abb", "bcc"));
+
+ binaryDictionary.removeBigramWords("aaa", "abb");
+ // Test remove non-existing bigram operation.
+ binaryDictionary.removeBigramWords("aaa", "abb");
+ binaryDictionary.removeBigramWords("bcc", "aaa");
dictFile.delete();
}
- public void testRandomlyAddBigramWords() {
- // TODO: Add a test to check the frequency of the bigram score which uses current value
- // calculated in the native code
- final int wordCount = 100;
- final int bigramCount = 1000;
- final int codePointSetSize = 50;
- final int seed = 11111;
+ public void testFlushDictionary() {
File dictFile = null;
try {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
@@ -218,35 +348,39 @@
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- final ArrayList<String> words = new ArrayList<String>();
- // Test a word that isn't contained within the dictionary.
- final Random random = new Random(seed);
- final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
- final int unigramProbability = 100;
- final int bigramProbability = 10;
- for (int i = 0; i < wordCount; ++i) {
- final String word = CodePointUtils.generateWord(random, codePointSet);
- words.add(word);
- binaryDictionary.addUnigramWord(word, unigramProbability);
- }
+ final int probability = 100;
+ binaryDictionary.addUnigramWord("aaa", probability);
+ binaryDictionary.addUnigramWord("abcd", probability);
+ // Close without flushing.
+ binaryDictionary.close();
- final boolean[][] bigramRelations = new boolean[wordCount][wordCount];
- for (int i = 0; i < bigramCount; i++) {
- final int word0Index = random.nextInt(wordCount);
- final int word1Index = random.nextInt(wordCount);
- final String word0 = words.get(word0Index);
- final String word1 = words.get(word1Index);
+ binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- bigramRelations[word0Index][word1Index] = true;
- binaryDictionary.addBigramWords(word0, word1, bigramProbability);
- }
+ assertEquals(-1, binaryDictionary.getFrequency("aaa"));
+ assertEquals(-1, binaryDictionary.getFrequency("abcd"));
- for (int i = 0; i < words.size(); i++) {
- for (int j = 0; j < words.size(); j++) {
- assertEquals(bigramRelations[i][j],
- binaryDictionary.isValidBigram(words.get(i), words.get(j)));
- }
- }
+ binaryDictionary.addUnigramWord("aaa", probability);
+ binaryDictionary.addUnigramWord("abcd", probability);
+ binaryDictionary.flush();
+ binaryDictionary.close();
+
+ binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+
+ assertEquals(probability, binaryDictionary.getFrequency("aaa"));
+ assertEquals(probability, binaryDictionary.getFrequency("abcd"));
+ binaryDictionary.addUnigramWord("bcde", probability);
+ binaryDictionary.flush();
+ binaryDictionary.close();
+
+ binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ assertEquals(probability, binaryDictionary.getFrequency("bcde"));
+ binaryDictionary.close();
dictFile.delete();
}
diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTests.java b/tests/src/com/android/inputmethod/latin/InputLogicTests.java
index fe92be6..cc2569f 100644
--- a/tests/src/com/android/inputmethod/latin/InputLogicTests.java
+++ b/tests/src/com/android/inputmethod/latin/InputLogicTests.java
@@ -134,6 +134,13 @@
assertEquals("simple auto-correct", EXPECTED_RESULT, mEditText.getText().toString());
}
+ public void testAutoCorrectWithQuote() {
+ final String STRING_TO_TYPE = "didn' ";
+ final String EXPECTED_RESULT = "didn't ";
+ type(STRING_TO_TYPE);
+ assertEquals("auto-correct with quote", EXPECTED_RESULT, mEditText.getText().toString());
+ }
+
public void testAutoCorrectWithPeriod() {
final String STRING_TO_TYPE = "tgis.";
final String EXPECTED_RESULT = "this.";
diff --git a/tests/src/com/android/inputmethod/latin/InputPointersTests.java b/tests/src/com/android/inputmethod/latin/InputPointersTests.java
index f0b6acc..5095f96 100644
--- a/tests/src/com/android/inputmethod/latin/InputPointersTests.java
+++ b/tests/src/com/android/inputmethod/latin/InputPointersTests.java
@@ -244,4 +244,20 @@
expecteds[i + expectedPos], actuals[i + actualPos]);
}
}
+
+ public void testShift() {
+ final InputPointers src = new InputPointers(DEFAULT_CAPACITY);
+ final int limit = 100;
+ final int shiftAmount = 20;
+ for (int i = 0; i < limit; i++) {
+ src.addPointer(i, i * 2, i * 3, i * 4);
+ }
+ src.shift(shiftAmount);
+ for (int i = 0; i < limit - shiftAmount; ++i) {
+ assertEquals("xCoordinates at " + i, i + shiftAmount, src.getXCoordinates()[i]);
+ assertEquals("yCoordinates at " + i, (i + shiftAmount) * 2, src.getYCoordinates()[i]);
+ assertEquals("pointerIds at " + i, (i + shiftAmount) * 3, src.getPointerIds()[i]);
+ assertEquals("times at " + i, (i + shiftAmount) * 4, src.getTimes()[i]);
+ }
+ }
}
diff --git a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java
index 4cf8333..a594baf 100644
--- a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java
+++ b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java
@@ -64,4 +64,37 @@
assertEquals("0", wordsWithoutTyped.getWord(0));
assertEquals(SuggestedWordInfo.KIND_CORRECTION, wordsWithoutTyped.getInfo(0).mKind);
}
+
+ // Helper for testGetTransformedWordInfo
+ private SuggestedWordInfo createWordInfo(final String s) {
+ // Use 100 as the frequency because the numerical value does not matter as
+ // long as it's > 1 and < INT_MAX.
+ return new SuggestedWordInfo(s, 100,
+ SuggestedWordInfo.KIND_TYPED, null /* sourceDict */,
+ SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */,
+ SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */);
+ }
+
+ // Helper for testGetTransformedWordInfo
+ private SuggestedWordInfo transformWordInfo(final String info,
+ final int trailingSingleQuotesCount) {
+ return Suggest.getTransformedSuggestedWordInfo(createWordInfo(info),
+ Locale.ENGLISH, false /* isAllUpperCase */, false /* isFirstCharCapitalized */,
+ trailingSingleQuotesCount);
+ }
+
+ public void testGetTransformedSuggestedWordInfo() {
+ SuggestedWordInfo result = transformWordInfo("word", 0);
+ assertEquals(result.mWord, "word");
+ result = transformWordInfo("word", 1);
+ assertEquals(result.mWord, "word'");
+ result = transformWordInfo("word", 3);
+ assertEquals(result.mWord, "word'''");
+ result = transformWordInfo("didn't", 0);
+ assertEquals(result.mWord, "didn't");
+ result = transformWordInfo("didn't", 1);
+ assertEquals(result.mWord, "didn't");
+ result = transformWordInfo("didn't", 3);
+ assertEquals(result.mWord, "didn't''");
+ }
}
diff --git a/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java b/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java
index cfff61e..cad80d5 100644
--- a/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java
+++ b/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java
@@ -340,4 +340,18 @@
expecteds[i + expectedPos], actuals[i + actualPos]);
}
}
+
+ public void testShift() {
+ final ResizableIntArray src = new ResizableIntArray(DEFAULT_CAPACITY);
+ final int limit = DEFAULT_CAPACITY * 10;
+ final int shiftAmount = 20;
+ for (int i = 0; i < limit; ++i) {
+ src.add(i, i);
+ assertEquals("length after add at " + i, i + 1, src.getLength());
+ }
+ src.shift(shiftAmount);
+ for (int i = 0; i < limit - shiftAmount; ++i) {
+ assertEquals("value at " + i, i + shiftAmount, src.get(i));
+ }
+ }
}
diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
index c6fa943..4e396a1 100644
--- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
+++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
@@ -21,6 +21,8 @@
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.SmallTest;
+import java.util.Arrays;
+import java.util.List;
import java.util.Locale;
@SmallTest
@@ -268,4 +270,14 @@
final String bytesStr2 = StringUtils.byteArrayToHexString(bytes2);
assertTrue(bytesStr.equals(bytesStr2));
}
+
+ public void testJsonStringUtils() {
+ final Object[] objs = new Object[] { 1, "aaa", "bbb", 3 };
+ final List<Object> objArray = Arrays.asList(objs);
+ final String str = StringUtils.listToJsonStr(objArray);
+ final List<Object> newObjArray = StringUtils.jsonStrToList(str);
+ for (int i = 0; i < objs.length; ++i) {
+ assertEquals(objs[i], newObjArray.get(i));
+ }
+ }
}