Merge "Fix to follow the field naming conventions."
diff --git a/java/res/values/strings.xml b/java/res/values/strings.xml
index 1eff2f9..aae5b0b 100644
--- a/java/res/values/strings.xml
+++ b/java/res/values/strings.xml
@@ -388,7 +388,7 @@
be a QWERTY, or AZERTY, or any other disposition that only offers Latin characters, so
you wouldn't be able to type, say, Arabic on it. Please translate it in a way that "alphabet"
would be understood to mean specifically the Latin alphabet, rather than any other
-alphabet. [CHAR LIMIT=25] -->
+alphabet. [CHAR LIMIT=29] -->
<string name="subtype_no_language">No language (Alphabet)</string>
<!-- This string is displayed in the description for a keyboard type. It refers specifically to
the Latin alphabet, as opposed to Cyrillic, Arabic, Hebrew or other scripts.
diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
index 8be04c1..47151bf 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
@@ -37,10 +37,9 @@
* An in memory dictionary for memorizing entries and writing a binary dictionary.
*/
public class DictionaryWriter extends AbstractDictionaryWriter {
- // TODO: Regenerate version 3 binary dictionary.
- private static final int BINARY_DICT_VERSION = 2;
+ private static final int BINARY_DICT_VERSION = 3;
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
- new FormatSpec.FormatOptions(BINARY_DICT_VERSION);
+ new FormatSpec.FormatOptions(BINARY_DICT_VERSION, true /* supportsDynamicUpdate */);
private FusionDictionary mFusionDictionary;
diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java
index b69e3f8..35920f8 100644
--- a/java/src/com/android/inputmethod/latin/RichInputConnection.java
+++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java
@@ -654,9 +654,11 @@
+ "\"" + periodSpace + "\" just before the cursor.");
return false;
}
+ // Double-space results in ". ". A backspace to cancel this should result in a single
+ // space in the text field, so we replace ". " with a single space.
deleteSurroundingText(2, 0);
- final String doubleSpace = " ";
- commitText(doubleSpace, 1);
+ final String singleSpace = " ";
+ commitText(singleSpace, 1);
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
ResearchLogger.richInputConnection_revertDoubleSpacePeriod();
}
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp
index 5d14a05..0e8d72f 100644
--- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp
@@ -61,8 +61,7 @@
if (ByteArrayUtils::readUint16(dict, 4) == 2) {
return VERSION_2;
} else if (ByteArrayUtils::readUint16(dict, 4) == 3) {
- // TODO: Support version 3 dictionary.
- return UNKNOWN_VERSION;
+ return VERSION_3;
} else {
return UNKNOWN_VERSION;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
index 20cda91..7ac635a 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
@@ -27,7 +27,9 @@
const uint8_t *const dictRoot = mBinaryDictionaryInfo->getDictRoot();
int pos = nodePos;
mFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictRoot, &pos);
- mParentPos = DynamicPatriciaTrieReadingUtils::getParentPosAndAdvancePosition(dictRoot, &pos);
+ const int parentPos =
+ DynamicPatriciaTrieReadingUtils::getParentPosAndAdvancePosition(dictRoot, &pos);
+ mParentPos = (parentPos != 0) ? mNodePos + parentPos : NOT_A_DICT_POS;
if (outCodePoints != 0) {
mCodePointCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition(
dictRoot, mFlags, maxCodePointCount, outCodePoints, &pos);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h
index b668aab..71558ed 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h
@@ -42,7 +42,7 @@
// Reads node information from dictionary buffer and updates members with the information.
AK_FORCE_INLINE void fetchNodeInfoFromBuffer(const int nodePos) {
- fetchNodeInfoFromBufferAndGetNodeCodePoints(mNodePos , 0 /* maxCodePointCount */,
+ fetchNodeInfoFromBufferAndGetNodeCodePoints(nodePos , 0 /* maxCodePointCount */,
0 /* outCodePoints */);
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
index 9a180e6..3df5056 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
@@ -27,6 +27,8 @@
namespace latinime {
const DynamicPatriciaTriePolicy DynamicPatriciaTriePolicy::sInstance;
+// To avoid infinite loop caused by invalid or malicious forward links.
+const int DynamicPatriciaTriePolicy::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000;
void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode,
const BinaryDictionaryInfo *const binaryDictionaryInfo,
@@ -37,14 +39,23 @@
DynamicPatriciaTrieNodeReader nodeReader(binaryDictionaryInfo);
int mergedNodeCodePoints[MAX_WORD_LENGTH];
int nextPos = dicNode->getChildrenPos();
+ int totalChildCount = 0;
do {
const int childCount = PatriciaTrieReadingUtils::getGroupCountAndAdvancePosition(
binaryDictionaryInfo->getDictRoot(), &nextPos);
+ totalChildCount += childCount;
+ if (childCount <= 0 || totalChildCount > MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP) {
+ // Invalid dictionary.
+ AKLOGI("Invalid dictionary. childCount: %d, totalChildCount: %d, MAX: %d",
+ childCount, totalChildCount, MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP);
+ ASSERT(false);
+ return;
+ }
for (int i = 0; i < childCount; i++) {
nodeReader.fetchNodeInfoFromBufferAndGetNodeCodePoints(nextPos, MAX_WORD_LENGTH,
mergedNodeCodePoints);
if (!nodeReader.isDeleted() && !nodeFilter->isFilteredOut(mergedNodeCodePoints[0])) {
- // Push child note when the node is not deleted and not filtered out.
+ // Push child node when the node is not deleted and not filtered out.
childDicNodes->pushLeavingChild(dicNode, nodeReader.getNodePos(),
nodeReader.getChildrenPos(), nodeReader.getProbability(),
nodeReader.isTerminal(), nodeReader.hasChildren(),
@@ -55,13 +66,17 @@
}
nextPos = DynamicPatriciaTrieReadingUtils::getForwardLinkPosition(
binaryDictionaryInfo->getDictRoot(), nextPos);
- } while(DynamicPatriciaTrieReadingUtils::isValidForwardLinkPosition(nextPos));
+ } while (DynamicPatriciaTrieReadingUtils::isValidForwardLinkPosition(nextPos));
}
int DynamicPatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
const BinaryDictionaryInfo *const binaryDictionaryInfo,
const int nodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const {
+ if (nodePos == NOT_A_VALID_WORD_POS) {
+ *outUnigramProbability = NOT_A_PROBABILITY;
+ return 0;
+ }
// This method traverses parent nodes from the terminal by following parent pointers; thus,
// node code points are stored in the buffer in the reverse order.
int reverseCodePoints[maxCodePointCount];
@@ -79,7 +94,7 @@
reverseCodePoints[codePointCount++] = mergedNodeCodePoints[i];
}
// Then, follow parent pos toward the root node.
- while (nodeReader.getParentPos() != getRootPosition()) {
+ while (nodeReader.getParentPos() != NOT_A_DICT_POS) {
// codePointCount must be incremented at least once in each iteration to ensure preventing
// infinite loop.
if (nodeReader.isDeleted() || codePointCount > maxCodePointCount
@@ -106,12 +121,85 @@
int DynamicPatriciaTriePolicy::getTerminalNodePositionOfWord(
const BinaryDictionaryInfo *const binaryDictionaryInfo, const int *const inWord,
const int length, const bool forceLowerCaseSearch) const {
- // TODO: Implement.
- return NOT_A_DICT_POS;
+ int searchCodePoints[length];
+ for (int i = 0; i < length; ++i) {
+ searchCodePoints[i] = forceLowerCaseSearch ? CharUtils::toLowerCase(inWord[i]) : inWord[i];
+ }
+ int mergedNodeCodePoints[MAX_WORD_LENGTH];
+ int currentLength = 0;
+ int pos = getRootPosition();
+ DynamicPatriciaTrieNodeReader nodeReader(binaryDictionaryInfo);
+ while (currentLength <= length) {
+ // When foundMatchedNode becomes true, currentLength is increased at least once.
+ bool foundMatchedNode = false;
+ int totalChildCount = 0;
+ do {
+ const int childCount = PatriciaTrieReadingUtils::getGroupCountAndAdvancePosition(
+ binaryDictionaryInfo->getDictRoot(), &pos);
+ totalChildCount += childCount;
+ if (childCount <= 0 || totalChildCount > MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP) {
+ // Invalid dictionary.
+ AKLOGI("Invalid dictionary. childCount: %d, totalChildCount: %d, MAX: %d",
+ childCount, totalChildCount, MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP);
+ ASSERT(false);
+ return NOT_A_VALID_WORD_POS;
+ }
+ for (int i = 0; i < childCount; i++) {
+ nodeReader.fetchNodeInfoFromBufferAndGetNodeCodePoints(pos, MAX_WORD_LENGTH,
+ mergedNodeCodePoints);
+ if (nodeReader.isDeleted() || nodeReader.getCodePointCount() <= 0) {
+ // Skip deleted or empty node.
+ pos = nodeReader.getSiblingNodePos();
+ continue;
+ }
+ bool matched = true;
+ for (int j = 0; j < nodeReader.getCodePointCount(); ++j) {
+ if (mergedNodeCodePoints[j] != searchCodePoints[currentLength + j]) {
+ // Different code point is found.
+ matched = false;
+ break;
+ }
+ }
+ if (matched) {
+ currentLength += nodeReader.getCodePointCount();
+ if (length == currentLength) {
+ // Terminal position is found.
+ return nodeReader.getNodePos();
+ }
+ if (!nodeReader.hasChildren()) {
+ return NOT_A_VALID_WORD_POS;
+ }
+ foundMatchedNode = true;
+ // Advance to the children nodes.
+ pos = nodeReader.getChildrenPos();
+ break;
+ }
+ // Try next sibling node.
+ pos = nodeReader.getSiblingNodePos();
+ }
+ if (foundMatchedNode) {
+ break;
+ }
+ // If the matched node is not found in the current node group, try to follow the
+ // forward link.
+ pos = DynamicPatriciaTrieReadingUtils::getForwardLinkPosition(
+ binaryDictionaryInfo->getDictRoot(), pos);
+ } while (DynamicPatriciaTrieReadingUtils::isValidForwardLinkPosition(pos));
+ if (!foundMatchedNode) {
+ // Matched node is not found.
+ return NOT_A_VALID_WORD_POS;
+ }
+ }
+ // If we already traversed the tree further than the word is long, there means
+ // there was no match (or we would have found it).
+ return NOT_A_VALID_WORD_POS;
}
int DynamicPatriciaTriePolicy::getUnigramProbability(
const BinaryDictionaryInfo *const binaryDictionaryInfo, const int nodePos) const {
+ if (nodePos == NOT_A_VALID_WORD_POS) {
+ return NOT_A_PROBABILITY;
+ }
DynamicPatriciaTrieNodeReader nodeReader(binaryDictionaryInfo);
nodeReader.fetchNodeInfoFromBuffer(nodePos);
if (nodeReader.isDeleted() || nodeReader.isBlacklisted() || nodeReader.isNotAWord()) {
@@ -123,6 +211,9 @@
int DynamicPatriciaTriePolicy::getShortcutPositionOfNode(
const BinaryDictionaryInfo *const binaryDictionaryInfo,
const int nodePos) const {
+ if (nodePos == NOT_A_VALID_WORD_POS) {
+ return NOT_A_DICT_POS;
+ }
DynamicPatriciaTrieNodeReader nodeReader(binaryDictionaryInfo);
nodeReader.fetchNodeInfoFromBuffer(nodePos);
if (nodeReader.isDeleted()) {
@@ -134,6 +225,9 @@
int DynamicPatriciaTriePolicy::getBigramsPositionOfNode(
const BinaryDictionaryInfo *const binaryDictionaryInfo,
const int nodePos) const {
+ if (nodePos == NOT_A_VALID_WORD_POS) {
+ return NOT_A_DICT_POS;
+ }
DynamicPatriciaTrieNodeReader nodeReader(binaryDictionaryInfo);
nodeReader.fetchNodeInfoFromBuffer(nodePos);
if (nodeReader.isDeleted()) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
index 39dfb86..6a79771 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
@@ -61,6 +61,7 @@
private:
DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTriePolicy);
static const DynamicPatriciaTriePolicy sInstance;
+ static const int MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP;
DynamicPatriciaTriePolicy() {}
~DynamicPatriciaTriePolicy() {}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h
index f44c265..5398d7e 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h
@@ -39,8 +39,7 @@
static AK_FORCE_INLINE int getParentPosAndAdvancePosition(const uint8_t *const buffer,
int *const pos) {
- const int base = *pos;
- return base + ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos);
+ return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos);
}
static int readChildrenPositionAndAdvancePosition(const uint8_t *const buffer,
diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTests.java b/tests/src/com/android/inputmethod/latin/InputLogicTests.java
index 9140197..d27a7a9 100644
--- a/tests/src/com/android/inputmethod/latin/InputLogicTests.java
+++ b/tests/src/com/android/inputmethod/latin/InputLogicTests.java
@@ -179,7 +179,7 @@
public void testCancelDoubleSpace() {
final String STRING_TO_TYPE = "this ";
- final String EXPECTED_RESULT = "this ";
+ final String EXPECTED_RESULT = "this ";
type(STRING_TO_TYPE);
type(Constants.CODE_DELETE);
assertEquals("double space make a period", EXPECTED_RESULT, mEditText.getText().toString());