Merge "Fix punctuation test for tablet"
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index 013f922..c450a1d 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -17,6 +17,7 @@
package com.android.inputmethod.latin;
import android.text.TextUtils;
+import android.util.Log;
import android.util.SparseArray;
import com.android.inputmethod.annotations.UsedForTesting;
@@ -29,6 +30,7 @@
import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.settings.NativeSuggestOptions;
import com.android.inputmethod.latin.utils.CollectionUtils;
+import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.JniUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam;
import com.android.inputmethod.latin.utils.StringUtils;
@@ -84,6 +86,7 @@
private final Locale mLocale;
private final long mDictSize;
private final String mDictFilePath;
+ private final boolean mIsUpdatable;
private final int[] mInputCodePoints = new int[MAX_WORD_LENGTH];
private final int[] mOutputCodePoints = new int[MAX_WORD_LENGTH * MAX_RESULTS];
private final int[] mSpaceIndices = new int[MAX_RESULTS];
@@ -130,6 +133,7 @@
mLocale = locale;
mDictSize = length;
mDictFilePath = filename;
+ mIsUpdatable = isUpdatable;
mNativeSuggestOptions.setUseFullEditDistance(useFullEditDistance);
loadDictionary(filename, offset, length, isUpdatable);
}
@@ -177,6 +181,7 @@
int bigramProbability);
private static native int setCurrentTimeForTestNative(int currentTime);
private static native String getPropertyNative(long dict, String query);
+ private static native boolean isCorruptedNative(long dict);
public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
final Locale locale, final Map<String, String> attributeMap) {
@@ -198,6 +203,22 @@
mNativeDict = openNative(path, startOffset, length, isUpdatable);
}
+ // TODO: Check isCorrupted() for main dictionaries.
+ public boolean isCorrupted() {
+ if (!isValidDictionary()) {
+ return false;
+ }
+ if (!isCorruptedNative(mNativeDict)) {
+ return false;
+ }
+ // TODO: Record the corruption.
+ Log.e(TAG, "BinaryDictionary (" + mDictFilePath + ") is corrupted.");
+ Log.e(TAG, "locale: " + mLocale);
+ Log.e(TAG, "dict size: " + mDictSize);
+ Log.e(TAG, "updatable: " + mIsUpdatable);
+ return true;
+ }
+
@UsedForTesting
public DictionaryHeader getHeader() throws UnsupportedFormatException {
if (mNativeDict == 0) {
@@ -444,7 +465,7 @@
// only be called for actual files. Right now it's only called by the flush() family of
// functions, which require an updatable dictionary, so it's okay. But beware.
loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */,
- dictFile.length(), true /* isUpdatable */);
+ dictFile.length(), mIsUpdatable);
}
public void flush() {
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index 230739d..f9ab941 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -276,22 +276,26 @@
return attributeMap;
}
+ private void removeBinaryDictionaryLocked() {
+ if (mBinaryDictionary != null) {
+ mBinaryDictionary.close();
+ }
+ if (mDictFile.exists() && !FileUtils.deleteRecursively(mDictFile)) {
+ Log.e(TAG, "Can't remove a file: " + mDictFile.getName());
+ }
+ mBinaryDictionary = null;
+ }
+
protected void clear() {
- final File dictFile = mDictFile;
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
if (mDictionaryWriter == null) {
- if (mBinaryDictionary != null) {
- mBinaryDictionary.close();
- }
- if (dictFile.exists() && !FileUtils.deleteRecursively(dictFile)) {
- Log.e(TAG, "Can't remove a file: " + dictFile.getName());
- }
- BinaryDictionary.createEmptyDictFile(dictFile.getAbsolutePath(),
+ removeBinaryDictionaryLocked();
+ BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
mBinaryDictionary = new BinaryDictionary(
- dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(),
+ mDictFile.getAbsolutePath(), 0 /* offset */, mDictFile.length(),
true /* useFullEditDistance */, mLocale, mDictType, mIsUpdatable);
} else {
mDictionaryWriter.clear();
@@ -469,6 +473,9 @@
proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
sessionId);
holder.set(binarySuggestion);
+ if (mBinaryDictionary.isCorrupted()) {
+ removeBinaryDictionaryLocked();
+ }
}
});
return holder.get(null, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);
diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
index eeb5bf5..045d06f 100644
--- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
+++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
@@ -490,6 +490,7 @@
handler.showGesturePreviewAndSuggestionStrip(
SuggestedWords.EMPTY, false /* dismissGestureFloatingPreviewText */);
handler.cancelUpdateSuggestionStrip();
+ ++mAutoCommitSequenceNumber;
mConnection.beginBatchEdit();
if (mWordComposer.isComposingWord()) {
if (settingsValues.mIsInternal) {
@@ -587,6 +588,7 @@
public void onEndBatchInput(final SettingsValues settingValues,
final InputPointers batchPointers) {
mInputLogicHandler.onEndBatchInput(batchPointers, mAutoCommitSequenceNumber);
+ ++mAutoCommitSequenceNumber;
}
// TODO: remove this argument
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
index 71e120c..bf776cf 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
@@ -120,16 +120,10 @@
// used only for testing.
private final DictionaryBufferFactory mBufferFactory;
protected DictBuffer mDictBuffer;
- private final BinaryDictionary mBinaryDictionary;
/* package */ Ver2DictDecoder(final File file, final int factoryFlag) {
mDictionaryBinaryFile = file;
mDictBuffer = null;
- // dictType is not being used in dicttool. Passing an empty string.
- mBinaryDictionary = new BinaryDictionary(file.getAbsolutePath(),
- 0 /* offset */, file.length() /* length */, true /* useFullEditDistance */,
- null /* locale */, "" /* dictType */, false /* isUpdatable */);
-
if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) {
mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory();
} else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) {
@@ -144,10 +138,6 @@
/* package */ Ver2DictDecoder(final File file, final DictionaryBufferFactory factory) {
mDictionaryBinaryFile = file;
mBufferFactory = factory;
- // dictType is not being used in dicttool. Passing an empty string.
- mBinaryDictionary = new BinaryDictionary(file.getAbsolutePath(),
- 0 /* offset */, file.length() /* length */, true /* useFullEditDistance */,
- null /* locale */, "" /* dictType */, false /* isUpdatable */);
}
@Override
@@ -172,7 +162,13 @@
@Override
public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException {
- final DictionaryHeader header = mBinaryDictionary.getHeader();
+ // dictType is not being used in dicttool. Passing an empty string.
+ final BinaryDictionary binaryDictionary = new BinaryDictionary(
+ mDictionaryBinaryFile.getAbsolutePath(), 0 /* offset */,
+ mDictionaryBinaryFile.length() /* length */, true /* useFullEditDistance */,
+ null /* locale */, "" /* dictType */, false /* isUpdatable */);
+ final DictionaryHeader header = binaryDictionary.getHeader();
+ binaryDictionary.close();
if (header == null) {
throw new IOException("Cannot read the dictionary header.");
}
@@ -254,6 +250,11 @@
@Override
public FusionDictionary readDictionaryBinary(final boolean deleteDictIfBroken)
throws FileNotFoundException, IOException, UnsupportedFormatException {
+ // dictType is not being used in dicttool. Passing an empty string.
+ final BinaryDictionary binaryDictionary = new BinaryDictionary(
+ mDictionaryBinaryFile.getAbsolutePath(), 0 /* offset */,
+ mDictionaryBinaryFile.length() /* length */, true /* useFullEditDistance */,
+ null /* locale */, "" /* dictType */, false /* isUpdatable */);
final DictionaryHeader header = readHeader();
final FusionDictionary fusionDict =
new FusionDictionary(new FusionDictionary.PtNodeArray(), header.mDictionaryOptions);
@@ -261,11 +262,11 @@
final ArrayList<WordProperty> wordProperties = CollectionUtils.newArrayList();
do {
final BinaryDictionary.GetNextWordPropertyResult result =
- mBinaryDictionary.getNextWordProperty(token);
+ binaryDictionary.getNextWordProperty(token);
final WordProperty wordProperty = result.mWordProperty;
if (wordProperty == null) {
+ binaryDictionary.close();
if (deleteDictIfBroken) {
- mBinaryDictionary.close();
mDictionaryBinaryFile.delete();
}
return null;
@@ -294,6 +295,7 @@
fusionDict.setBigram(word0, bigram.mWord, bigram.mProbabilityInfo);
}
}
+ binaryDictionary.close();
return fusionDict;
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
index 88fff38..afe8231 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
@@ -35,7 +35,6 @@
private static final String TAG = Ver4DictDecoder.class.getSimpleName();
final File mDictDirectory;
- final BinaryDictionary mBinaryDictionary;
@UsedForTesting
/* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) {
@@ -45,24 +44,32 @@
@UsedForTesting
/* package */ Ver4DictDecoder(final File dictDirectory, final DictionaryBufferFactory factory) {
mDictDirectory = dictDirectory;
- // dictType is not being used in dicttool. Passing an empty string.
- mBinaryDictionary = new BinaryDictionary(dictDirectory.getAbsolutePath(),
- 0 /* offset */, 0 /* length */, true /* useFullEditDistance */, null /* locale */,
- "" /* dictType */, true /* isUpdatable */);
+
}
@Override
public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException {
- final DictionaryHeader header = mBinaryDictionary.getHeader();
+ // dictType is not being used in dicttool. Passing an empty string.
+ final BinaryDictionary binaryDictionary= new BinaryDictionary(
+ mDictDirectory.getAbsolutePath(), 0 /* offset */, 0 /* length */,
+ true /* useFullEditDistance */, null /* locale */,
+ "" /* dictType */, true /* isUpdatable */);
+ final DictionaryHeader header = binaryDictionary.getHeader();
+ binaryDictionary.close();
if (header == null) {
throw new IOException("Cannot read the dictionary header.");
}
- return mBinaryDictionary.getHeader();
+ return header;
}
@Override
public FusionDictionary readDictionaryBinary(final boolean deleteDictIfBroken)
throws FileNotFoundException, IOException, UnsupportedFormatException {
+ // dictType is not being used in dicttool. Passing an empty string.
+ final BinaryDictionary binaryDictionary = new BinaryDictionary(
+ mDictDirectory.getAbsolutePath(), 0 /* offset */, 0 /* length */,
+ true /* useFullEditDistance */, null /* locale */,
+ "" /* dictType */, true /* isUpdatable */);
final DictionaryHeader header = readHeader();
final FusionDictionary fusionDict =
new FusionDictionary(new FusionDictionary.PtNodeArray(), header.mDictionaryOptions);
@@ -70,11 +77,11 @@
final ArrayList<WordProperty> wordProperties = CollectionUtils.newArrayList();
do {
final BinaryDictionary.GetNextWordPropertyResult result =
- mBinaryDictionary.getNextWordProperty(token);
+ binaryDictionary.getNextWordProperty(token);
final WordProperty wordProperty = result.mWordProperty;
if (wordProperty == null) {
+ binaryDictionary.close();
if (deleteDictIfBroken) {
- mBinaryDictionary.close();
FileUtils.deleteRecursively(mDictDirectory);
}
return null;
@@ -103,6 +110,7 @@
fusionDict.setBigram(word0, bigram.mWord, bigram.mProbabilityInfo);
}
}
+ binaryDictionary.close();
return fusionDict;
}
}
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 1e00cd8..bb54cbd 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -529,6 +529,14 @@
return TimeKeeper::peekCurrentTime();
}
+static bool latinime_BinaryDictionary_isCorruptedNative(JNIEnv *env, jclass clazz, jlong dict) {
+ Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+ if (!dictionary) {
+ return false;
+ }
+ return dictionary->getDictionaryStructurePolicy()->isCorrupted();
+}
+
static const JNINativeMethod sMethods[] = {
{
const_cast<char *>("createEmptyDictFileNative"),
@@ -642,6 +650,11 @@
const_cast<char *>("getPropertyNative"),
const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"),
reinterpret_cast<void *>(latinime_BinaryDictionary_getProperty)
+ },
+ {
+ const_cast<char *>("isCorruptedNative"),
+ const_cast<char *>("(J)Z"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_isCorruptedNative)
}
};
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp
index 212f2ef..84a6ccf 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp
@@ -87,9 +87,24 @@
int lastCandidatePtNodePos = 0;
// Let's loop through PtNodes in this PtNode array searching for either the terminal
// or one of its ascendants.
+ if (pos < 0 || pos >= mDictBufferSize) {
+ AKLOGE("PtNode array position is invalid. pos: %d, dict size: %d",
+ pos, mDictBufferSize);
+ mIsCorrupted = true;
+ ASSERT(false);
+ *outUnigramProbability = NOT_A_PROBABILITY;
+ return 0;
+ }
for (int ptNodeCount = PatriciaTrieReadingUtils::getPtNodeArraySizeAndAdvancePosition(
mDictRoot, &pos); ptNodeCount > 0; --ptNodeCount) {
const int startPos = pos;
+ if (pos < 0 || pos >= mDictBufferSize) {
+ AKLOGE("PtNode position is invalid. pos: %d, dict size: %d", pos, mDictBufferSize);
+ mIsCorrupted = true;
+ ASSERT(false);
+ *outUnigramProbability = NOT_A_PROBABILITY;
+ return 0;
+ }
const PatriciaTrieReadingUtils::NodeFlags flags =
PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(mDictRoot, &pos);
const int character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTests.java b/tests/src/com/android/inputmethod/latin/InputLogicTests.java
index 1c714e7..ab97513 100644
--- a/tests/src/com/android/inputmethod/latin/InputLogicTests.java
+++ b/tests/src/com/android/inputmethod/latin/InputLogicTests.java
@@ -351,6 +351,38 @@
}
// TODO: Add some tests for non-BMP characters
+ public void testAutoCorrectByUserHistory() {
+ final String WORD_TO_BE_CORRECTED = "qpmx";
+ final String NOT_CORRECTED_RESULT = "qpmx ";
+ final String DESIRED_WORD = "qpmz";
+ final String CORRECTED_RESULT = "qpmz ";
+ final int typeCountNotToAutocorrect = 3;
+ final int typeCountToAutoCorrect = 16;
+ int startIndex = 0;
+ int endIndex = 0;
+
+ for (int i = 0; i < typeCountNotToAutocorrect; i++) {
+ type(DESIRED_WORD);
+ type(Constants.CODE_SPACE);
+ }
+ startIndex = mEditText.getText().length();
+ type(WORD_TO_BE_CORRECTED);
+ type(Constants.CODE_SPACE);
+ endIndex = mEditText.getText().length();
+ assertEquals("not auto-corrected by user history", NOT_CORRECTED_RESULT,
+ mEditText.getText().subSequence(startIndex, endIndex).toString());
+ for (int i = typeCountNotToAutocorrect; i < typeCountToAutoCorrect; i++) {
+ type(DESIRED_WORD);
+ type(Constants.CODE_SPACE);
+ }
+ startIndex = mEditText.getText().length();
+ type(WORD_TO_BE_CORRECTED);
+ type(Constants.CODE_SPACE);
+ endIndex = mEditText.getText().length();
+ assertEquals("auto-corrected by user history",
+ CORRECTED_RESULT, mEditText.getText().subSequence(startIndex, endIndex).toString());
+ }
+
public void testPredictionsAfterSpace() {
final String WORD_TO_TYPE = "Barack ";
type(WORD_TO_TYPE);