Merge "Fix: release lock in UserHistoryDictionary."
diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
new file mode 100644
index 0000000..ebbcedc
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin;
+
+import android.content.Context;
+import android.util.Log;
+
+import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+// TODO: Quit extending Dictionary after implementing dynamic binary dictionary.
+abstract public class AbstractDictionaryWriter extends Dictionary {
+ /** Used for Log actions from this class */
+ private static final String TAG = AbstractDictionaryWriter.class.getSimpleName();
+
+ private final Context mContext;
+
+ public AbstractDictionaryWriter(final Context context, final String dictType) {
+ super(dictType);
+ mContext = context;
+ }
+
+ abstract public void clear();
+
+ abstract public void addUnigramWord(final String word, final String shortcutTarget,
+ final int frequency, final boolean isNotAWord);
+
+ abstract public void addBigramWords(final String word0, final String word1,
+ final int frequency, final boolean isValid);
+
+ abstract public void removeBigramWords(final String word0, final String word1);
+
+ abstract protected void writeBinaryDictionary(final FileOutputStream out)
+ throws IOException, UnsupportedFormatException;
+
+ public void write(final String fileName) {
+ final String tempFileName = fileName + ".temp";
+ final File file = new File(mContext.getFilesDir(), fileName);
+ final File tempFile = new File(mContext.getFilesDir(), tempFileName);
+ FileOutputStream out = null;
+ try {
+ out = new FileOutputStream(tempFile);
+ writeBinaryDictionary(out);
+ out.flush();
+ out.close();
+ tempFile.renameTo(file);
+ } catch (IOException e) {
+ Log.e(TAG, "IO exception while writing file", e);
+ } catch (UnsupportedFormatException e) {
+ Log.e(TAG, "Unsupported format", e);
+ } finally {
+ if (out != null) {
+ try {
+ out.close();
+ } catch (IOException e) {
+ // ignore
+ }
+ }
+ }
+ }
+}
diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
index 110be9d..c99d0e2 100644
--- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
@@ -109,7 +109,6 @@
@Override
public void loadDictionaryAsync() {
- clearFusionDictionary();
loadDeviceAccountsEmailAddresses();
loadDictionaryAsyncForUri(ContactsContract.Profile.CONTENT_URI);
// TODO: Switch this URL to the newer ContactsContract too
@@ -236,6 +235,11 @@
}
@Override
+ protected boolean needsToReloadBeforeWriting() {
+ return true;
+ }
+
+ @Override
protected boolean hasContentChanged() {
final long startTime = SystemClock.uptimeMillis();
final int contactCount = getContactCount();
diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
new file mode 100644
index 0000000..8be04c1
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin;
+
+import android.content.Context;
+
+import com.android.inputmethod.keyboard.ProximityInfo;
+import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
+import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.FormatSpec;
+import com.android.inputmethod.latin.makedict.FusionDictionary;
+import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
+import com.android.inputmethod.latin.utils.CollectionUtils;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+/**
+ * An in memory dictionary for memorizing entries and writing a binary dictionary.
+ */
+public class DictionaryWriter extends AbstractDictionaryWriter {
+ // TODO: Regenerate version 3 binary dictionary.
+ private static final int BINARY_DICT_VERSION = 2;
+ private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
+ new FormatSpec.FormatOptions(BINARY_DICT_VERSION);
+
+ private FusionDictionary mFusionDictionary;
+
+ public DictionaryWriter(final Context context, final String dictType) {
+ super(context, dictType);
+ clear();
+ }
+
+ @Override
+ public void clear() {
+ final HashMap<String, String> attributes = CollectionUtils.newHashMap();
+ mFusionDictionary = new FusionDictionary(new Node(),
+ new FusionDictionary.DictionaryOptions(attributes, false, false));
+ }
+
+ /**
+ * Adds a word unigram to the fusion dictionary.
+ */
+ // TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries,
+ // considering performance regression.
+ @Override
+ public void addUnigramWord(final String word, final String shortcutTarget, final int frequency,
+ final boolean isNotAWord) {
+ if (shortcutTarget == null) {
+ mFusionDictionary.add(word, frequency, null, isNotAWord);
+ } else {
+ // TODO: Do this in the subclass, with this class taking an arraylist.
+ final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList();
+ shortcutTargets.add(new WeightedString(shortcutTarget, frequency));
+ mFusionDictionary.add(word, frequency, shortcutTargets, isNotAWord);
+ }
+ }
+
+ @Override
+ public void addBigramWords(final String word0, final String word1, final int frequency,
+ final boolean isValid) {
+ mFusionDictionary.setBigram(word0, word1, frequency);
+ }
+
+ @Override
+ public void removeBigramWords(final String word0, final String word1) {
+ // This class don't support removing bigram words.
+ }
+
+ @Override
+ protected void writeBinaryDictionary(final FileOutputStream out)
+ throws IOException, UnsupportedFormatException {
+ BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
+ }
+
+ @Override
+ public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
+ final String prevWord, final ProximityInfo proximityInfo,
+ boolean blockOffensiveWords) {
+ // This class doesn't support suggestion.
+ return null;
+ }
+
+ @Override
+ public boolean isValidWord(String word) {
+ // This class doesn't support dictionary retrieval.
+ return false;
+ }
+}
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index f4c4c65..a563c76 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -22,17 +22,9 @@
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
-import com.android.inputmethod.latin.makedict.FormatSpec;
-import com.android.inputmethod.latin.makedict.FusionDictionary;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
-import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
-import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -76,8 +68,8 @@
*/
private BinaryDictionary mBinaryDictionary;
- /** The expandable fusion dictionary used to generate the binary dictionary. */
- private FusionDictionary mFusionDictionary;
+ /** The in-memory dictionary used to generate the binary dictionary. */
+ private AbstractDictionaryWriter mDictionaryWriter;
/**
* The name of this dictionary, used as the filename for storing the binary dictionary. Multiple
@@ -92,11 +84,6 @@
/** Controls access to the local binary dictionary for this instance. */
private final DictionaryController mLocalDictionaryController = new DictionaryController();
- // TODO: Regenerate version 3 binary dictionary.
- private static final int BINARY_DICT_VERSION = 2;
- private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
- new FormatSpec.FormatOptions(BINARY_DICT_VERSION);
-
/**
* Abstract method for loading the unigrams and bigrams of a given dictionary in a background
* thread.
@@ -138,7 +125,7 @@
mContext = context;
mBinaryDictionary = null;
mSharedDictionaryController = getSharedDictionaryController(filename);
- clearFusionDictionary();
+ mDictionaryWriter = new DictionaryWriter(context, dictType);
}
protected static String getFilenameWithLocale(final String name, final String localeStr) {
@@ -157,47 +144,51 @@
mBinaryDictionary.close();
mBinaryDictionary = null;
}
+ mDictionaryWriter.close();
} finally {
mLocalDictionaryController.writeLock().unlock();
}
}
/**
- * Clears the fusion dictionary on the Java side. Note: Does not modify the binary dictionary on
- * the native side.
+ * Adds a word unigram to the dictionary. Used for loading a dictionary.
*/
- public void clearFusionDictionary() {
- final HashMap<String, String> attributes = CollectionUtils.newHashMap();
- mFusionDictionary = new FusionDictionary(new Node(),
- new FusionDictionary.DictionaryOptions(attributes, false, false));
+ protected void addWord(final String word, final String shortcutTarget,
+ final int frequency, final boolean isNotAWord) {
+ mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
}
/**
- * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes
- * are done to update the binary dictionary.
+ * Sets a word bigram in the dictionary. Used for loading a dictionary.
*/
- // TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries,
- // considering performance regression.
- protected void addWord(final String word, final String shortcutTarget, final int frequency,
- final boolean isNotAWord) {
- if (shortcutTarget == null) {
- mFusionDictionary.add(word, frequency, null, isNotAWord);
- } else {
- // TODO: Do this in the subclass, with this class taking an arraylist.
- final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList();
- shortcutTargets.add(new WeightedString(shortcutTarget, frequency));
- mFusionDictionary.add(word, frequency, shortcutTargets, isNotAWord);
+ protected void setBigram(final String prevWord, final String word, final int frequency) {
+ mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */);
+ }
+
+ /**
+ * Dynamically adds a word unigram to the dictionary.
+ */
+ protected void addWordDynamically(final String word, final String shortcutTarget,
+ final int frequency, final boolean isNotAWord) {
+ mLocalDictionaryController.writeLock().lock();
+ try {
+ mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
+ } finally {
+ mLocalDictionaryController.writeLock().unlock();
}
}
/**
- * Sets a word bigram in the fusion dictionary. Call updateBinaryDictionary when all changes are
- * done to update the binary dictionary.
+ * Dynamically sets a word bigram in the dictionary.
*/
- // TODO: Create "cache dictionary" to cache fresh bigrams for frequently updated dictionaries,
- // considering performance regression.
- protected void setBigram(final String prevWord, final String word, final int frequency) {
- mFusionDictionary.setBigram(prevWord, word, frequency);
+ protected void setBigramDynamically(final String prevWord, final String word,
+ final int frequency) {
+ mLocalDictionaryController.writeLock().lock();
+ try {
+ mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */);
+ } finally {
+ mLocalDictionaryController.writeLock().unlock();
+ }
}
@Override
@@ -207,9 +198,23 @@
asyncReloadDictionaryIfRequired();
if (mLocalDictionaryController.readLock().tryLock()) {
try {
+ final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
+ mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo,
+ blockOffensiveWords);
if (mBinaryDictionary != null) {
- return mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo,
- blockOffensiveWords);
+ final ArrayList<SuggestedWordInfo> binarySuggestion =
+ mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo,
+ blockOffensiveWords);
+ if (inMemDictSuggestion == null) {
+ return binarySuggestion;
+ } else if (binarySuggestion == null) {
+ return inMemDictSuggestion;
+ } else {
+ binarySuggestion.addAll(binarySuggestion);
+ return binarySuggestion;
+ }
+ } else {
+ return inMemDictSuggestion;
}
} finally {
mLocalDictionaryController.readLock().unlock();
@@ -240,22 +245,6 @@
return mBinaryDictionary.isValidWord(word);
}
- protected boolean isValidBigram(final String word1, final String word2) {
- if (mBinaryDictionary == null) return false;
- return mBinaryDictionary.isValidBigram(word1, word2);
- }
-
- protected boolean isValidBigramInner(final String word1, final String word2) {
- if (mLocalDictionaryController.readLock().tryLock()) {
- try {
- return isValidBigramLocked(word1, word2);
- } finally {
- mLocalDictionaryController.readLock().unlock();
- }
- }
- return false;
- }
-
protected boolean isValidBigramLocked(final String word1, final String word2) {
if (mBinaryDictionary == null) return false;
return mBinaryDictionary.isValidBigram(word1, word2);
@@ -274,7 +263,7 @@
* Loads the current binary dictionary from internal storage. Assumes the dictionary file
* exists.
*/
- protected void loadBinaryDictionary() {
+ private void loadBinaryDictionary() {
if (DEBUG) {
Log.d(TAG, "Loading binary dictionary: " + mFilename + " request="
+ mSharedDictionaryController.mLastUpdateRequestTime + " update="
@@ -306,6 +295,12 @@
}
/**
+ * Abstract method for checking if it is required to reload the dictionary before writing
+ * a binary dictionary.
+ */
+ abstract protected boolean needsToReloadBeforeWriting();
+
+ /**
* Generates and writes a new binary dictionary based on the contents of the fusion dictionary.
*/
private void generateBinaryDictionary() {
@@ -314,33 +309,11 @@
+ mSharedDictionaryController.mLastUpdateRequestTime + " update="
+ mSharedDictionaryController.mLastUpdateTime);
}
-
- loadDictionaryAsync();
-
- final String tempFileName = mFilename + ".temp";
- final File file = new File(mContext.getFilesDir(), mFilename);
- final File tempFile = new File(mContext.getFilesDir(), tempFileName);
- FileOutputStream out = null;
- try {
- out = new FileOutputStream(tempFile);
- BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
- out.flush();
- out.close();
- tempFile.renameTo(file);
- clearFusionDictionary();
- } catch (IOException e) {
- Log.e(TAG, "IO exception while writing file", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Unsupported format", e);
- } finally {
- if (out != null) {
- try {
- out.close();
- } catch (IOException e) {
- // ignore
- }
- }
+ if (needsToReloadBeforeWriting()) {
+ mDictionaryWriter.clear();
+ loadDictionaryAsync();
}
+ mDictionaryWriter.write(mFilename);
}
/**
diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
index ba84c1a..ab8f348 100644
--- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
@@ -240,7 +240,6 @@
private void addWords(final Cursor cursor) {
final boolean hasShortcutColumn = Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN;
- clearFusionDictionary();
if (cursor == null) return;
if (cursor.moveToFirst()) {
final int indexWord = cursor.getColumnIndex(Words.WORD);
@@ -267,4 +266,9 @@
protected boolean hasContentChanged() {
return true;
}
+
+ @Override
+ protected boolean needsToReloadBeforeWriting() {
+ return true;
+ }
}
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 2b8dbbc..6e1b80e 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -201,7 +201,7 @@
static jint latinime_BinaryDictionary_getProbability(JNIEnv *env, jclass clazz, jlong dict,
jintArray word) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
- if (!dictionary) return 0;
+ if (!dictionary) return NOT_A_PROBABILITY;
const jsize wordLength = env->GetArrayLength(word);
int codePoints[wordLength];
env->GetIntArrayRegion(word, 0, wordLength, codePoints);
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp
index f48386b..5d14a05 100644
--- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp
@@ -31,9 +31,9 @@
// The versions of Latin IME that only handle format version 1 only test for the magic
// number, so we had to change it so that version 2 files would be rejected by older
// implementations. On this occasion, we made the magic number 32 bits long.
-const uint32_t BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE;
+const uint32_t BinaryDictionaryFormatUtils::HEADER_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE;
// Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12
-const int BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MINIMUM_SIZE = 12;
+const int BinaryDictionaryFormatUtils::HEADER_VERSION_2_MINIMUM_SIZE = 12;
/* static */ BinaryDictionaryFormatUtils::FORMAT_VERSION
BinaryDictionaryFormatUtils::detectFormatVersion(const uint8_t *const dict,
@@ -46,25 +46,28 @@
}
const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0);
switch (magicNumber) {
- case FORMAT_VERSION_2_MAGIC_NUMBER:
- // Version 2 dictionaries are at least 12 bytes long.
- // If this dictionary has the version 2 magic number but is less than 12 bytes long,
- // then it's an unknown format and we need to avoid confidently reading the next bytes.
- if (dictSize < FORMAT_VERSION_2_MINIMUM_SIZE) {
+ case HEADER_VERSION_2_MAGIC_NUMBER:
+ // Version 2 header are at least 12 bytes long.
+ // If this header has the version 2 magic number but is less than 12 bytes long,
+ // then it's an unknown format and we need to avoid confidently reading the next bytes.
+ if (dictSize < HEADER_VERSION_2_MINIMUM_SIZE) {
+ return UNKNOWN_VERSION;
+ }
+ // Version 2 header is as follows:
+ // Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE
+ // Version number (2 bytes)
+ // Options (2 bytes)
+ // Header size (4 bytes) : integer, big endian
+ if (ByteArrayUtils::readUint16(dict, 4) == 2) {
+ return VERSION_2;
+ } else if (ByteArrayUtils::readUint16(dict, 4) == 3) {
+ // TODO: Support version 3 dictionary.
+ return UNKNOWN_VERSION;
+ } else {
+ return UNKNOWN_VERSION;
+ }
+ default:
return UNKNOWN_VERSION;
- }
- // Format 2 header is as follows:
- // Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE
- // Version number (2 bytes) 0x00 0x02
- // Options (2 bytes)
- // Header size (4 bytes) : integer, big endian
- if (ByteArrayUtils::readUint16(dict, 4) == 2) {
- return VERSION_2;
- } else {
- return UNKNOWN_VERSION;
- }
- default:
- return UNKNOWN_VERSION;
}
}
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h
index 80067b2..830684c 100644
--- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h
@@ -33,9 +33,9 @@
*/
class BinaryDictionaryFormatUtils {
public:
- // TODO: Support version 3 format.
enum FORMAT_VERSION {
- VERSION_2 = 1,
+ VERSION_2,
+ VERSION_3,
UNKNOWN_VERSION
};
@@ -45,8 +45,8 @@
DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormatUtils);
static const int DICTIONARY_MINIMUM_SIZE;
- static const uint32_t FORMAT_VERSION_2_MAGIC_NUMBER;
- static const int FORMAT_VERSION_2_MINIMUM_SIZE;
+ static const uint32_t HEADER_VERSION_2_MAGIC_NUMBER;
+ static const int HEADER_VERSION_2_MINIMUM_SIZE;
};
} // namespace latinime
#endif /* LATINIME_BINARY_DICTIONARY_FORMAT_UTILS_H */
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp
index c4c4bed..a57b0f8 100644
--- a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp
@@ -26,10 +26,10 @@
const int BinaryDictionaryHeaderReadingUtils::MAX_OPTION_KEY_LENGTH = 256;
-const int BinaryDictionaryHeaderReadingUtils::VERSION_2_MAGIC_NUMBER_SIZE = 4;
-const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_VERSION_SIZE = 2;
-const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_FLAG_SIZE = 2;
-const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_HEADER_SIZE_SIZE = 4;
+const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_MAGIC_NUMBER_SIZE = 4;
+const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_DICTIONARY_VERSION_SIZE = 2;
+const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_FLAG_SIZE = 2;
+const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_SIZE_FIELD_SIZE = 4;
const BinaryDictionaryHeaderReadingUtils::DictionaryFlags
BinaryDictionaryHeaderReadingUtils::NO_FLAGS = 0;
@@ -45,13 +45,13 @@
/* static */ int BinaryDictionaryHeaderReadingUtils::getHeaderSize(
const BinaryDictionaryInfo *const binaryDictionaryInfo) {
- switch (binaryDictionaryInfo->getFormat()) {
- case BinaryDictionaryFormatUtils::VERSION_2:
+ switch (getHeaderVersion(binaryDictionaryInfo->getFormat())) {
+ case HEADER_VERSION_2:
// See the format of the header in the comment in
// BinaryDictionaryFormatUtils::detectFormatVersion()
return ByteArrayUtils::readUint32(binaryDictionaryInfo->getDictBuf(),
- VERSION_2_MAGIC_NUMBER_SIZE + VERSION_2_DICTIONARY_VERSION_SIZE
- + VERSION_2_DICTIONARY_FLAG_SIZE);
+ VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE
+ + VERSION_2_HEADER_FLAG_SIZE);
default:
return S_INT_MAX;
}
@@ -60,10 +60,10 @@
/* static */ BinaryDictionaryHeaderReadingUtils::DictionaryFlags
BinaryDictionaryHeaderReadingUtils::getFlags(
const BinaryDictionaryInfo *const binaryDictionaryInfo) {
- switch (binaryDictionaryInfo->getFormat()) {
- case BinaryDictionaryFormatUtils::VERSION_2:
+ switch (getHeaderVersion(binaryDictionaryInfo->getFormat())) {
+ case HEADER_VERSION_2:
return ByteArrayUtils::readUint16(binaryDictionaryInfo->getDictBuf(),
- VERSION_2_MAGIC_NUMBER_SIZE + VERSION_2_DICTIONARY_VERSION_SIZE);
+ VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE);
default:
return NO_FLAGS;
}
@@ -73,11 +73,15 @@
/* static */ bool BinaryDictionaryHeaderReadingUtils::readHeaderValue(
const BinaryDictionaryInfo *const binaryDictionaryInfo,
const char *const key, int *outValue, const int outValueSize) {
- if (outValueSize <= 0 || !hasHeaderAttributes(binaryDictionaryInfo->getFormat())) {
+ if (outValueSize <= 0) {
return false;
}
const int headerSize = getHeaderSize(binaryDictionaryInfo);
int pos = getHeaderOptionsPosition(binaryDictionaryInfo->getFormat());
+ if (pos == NOT_A_DICT_POS) {
+ // The header doesn't have header options.
+ return false;
+ }
while (pos < headerSize) {
if(ByteArrayUtils::compareStringInBufferWithCharArray(
binaryDictionaryInfo->getDictBuf(), key, headerSize - pos, &pos) == 0) {
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h
index 94b9e12..6174822 100644
--- a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h
@@ -48,27 +48,15 @@
return (flags & FRENCH_LIGATURE_PROCESSING_FLAG) != 0;
}
- static AK_FORCE_INLINE bool hasHeaderAttributes(
- const BinaryDictionaryFormatUtils::FORMAT_VERSION format) {
- // Only format 2 and above have header attributes as {key,value} string pairs.
- switch (format) {
- case BinaryDictionaryFormatUtils::VERSION_2:
- return true;
- break;
- default:
- return false;
- }
- }
-
static AK_FORCE_INLINE int getHeaderOptionsPosition(
- const BinaryDictionaryFormatUtils::FORMAT_VERSION format) {
- switch (format) {
- case BinaryDictionaryFormatUtils::VERSION_2:
- return VERSION_2_MAGIC_NUMBER_SIZE + VERSION_2_DICTIONARY_VERSION_SIZE
- + VERSION_2_DICTIONARY_FLAG_SIZE + VERSION_2_DICTIONARY_HEADER_SIZE_SIZE;
+ const BinaryDictionaryFormatUtils::FORMAT_VERSION dictionaryFormat) {
+ switch (getHeaderVersion(dictionaryFormat)) {
+ case HEADER_VERSION_2:
+ return VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE
+ + VERSION_2_HEADER_FLAG_SIZE + VERSION_2_HEADER_SIZE_FIELD_SIZE;
break;
default:
- return 0;
+ return NOT_A_DICT_POS;
}
}
@@ -82,10 +70,15 @@
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryHeaderReadingUtils);
- static const int VERSION_2_MAGIC_NUMBER_SIZE;
- static const int VERSION_2_DICTIONARY_VERSION_SIZE;
- static const int VERSION_2_DICTIONARY_FLAG_SIZE;
- static const int VERSION_2_DICTIONARY_HEADER_SIZE_SIZE;
+ enum HEADER_VERSION {
+ HEADER_VERSION_2,
+ UNKNOWN_HEADER_VERSION
+ };
+
+ static const int VERSION_2_HEADER_MAGIC_NUMBER_SIZE;
+ static const int VERSION_2_HEADER_DICTIONARY_VERSION_SIZE;
+ static const int VERSION_2_HEADER_FLAG_SIZE;
+ static const int VERSION_2_HEADER_SIZE_FIELD_SIZE;
static const DictionaryFlags NO_FLAGS;
// Flags for special processing
@@ -95,6 +88,18 @@
static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG;
static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG;
static const DictionaryFlags CONTAINS_BIGRAMS_FLAG;
+
+ static HEADER_VERSION getHeaderVersion(
+ const BinaryDictionaryFormatUtils::FORMAT_VERSION formatVersion) {
+ switch(formatVersion) {
+ case BinaryDictionaryFormatUtils::VERSION_2:
+ // Fall through
+ case BinaryDictionaryFormatUtils::VERSION_3:
+ return HEADER_VERSION_2;
+ default:
+ return UNKNOWN_HEADER_VERSION;
+ }
+ }
};
}
#endif /* LATINIME_DICTIONARY_HEADER_READING_UTILS_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_policy_factory.h b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_policy_factory.h
index c0e24fa..70dad67 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_policy_factory.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_policy_factory.h
@@ -32,6 +32,9 @@
switch (dictionaryFormat) {
case BinaryDictionaryFormatUtils::VERSION_2:
return PatriciaTriePolicy::getInstance();
+ case BinaryDictionaryFormatUtils::VERSION_3:
+ // TODO: support version 3 dictionaries.
+ return 0;
default:
ASSERT(false);
return 0;
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
index cca81a0..87acafe 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
@@ -51,7 +51,7 @@
@LargeTest
public class BinaryDictIOTests extends AndroidTestCase {
private static final String TAG = BinaryDictIOTests.class.getSimpleName();
- private static final int MAX_UNIGRAMS = 100;
+ private static final int DEFAULT_MAX_UNIGRAMS = 100;
private static final int UNIGRAM_FREQ = 10;
private static final int BIGRAM_FREQ = 50;
private static final int TOLERANCE_OF_BIGRAM_FREQ = 5;
@@ -73,13 +73,15 @@
new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */);
public BinaryDictIOTests() {
- super();
+ this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS);
+ }
- final long time = System.currentTimeMillis();
- Log.e(TAG, "Testing dictionary: seed is " + time);
- final Random random = new Random(time);
+ public BinaryDictIOTests(final long seed, final int maxUnigrams) {
+ super();
+ Log.e(TAG, "Testing dictionary: seed is " + seed);
+ final Random random = new Random(seed);
sWords.clear();
- generateWords(MAX_UNIGRAMS, random);
+ generateWords(maxUnigrams, random);
for (int i = 0; i < sWords.size(); ++i) {
sChainBigrams.put(i, new ArrayList<Integer>());
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java
index 7b311c3..cacee52 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java
@@ -72,15 +72,21 @@
return command;
}
- private void execute(final String[] arguments) {
+ /**
+ * Executes the specified command with the specified arguments.
+ * @param arguments the arguments passed to dicttool.
+ * @return 0 for success, an error code otherwise (always 1 at the moment)
+ */
+ private int execute(final String[] arguments) {
final Command command = getCommand(arguments);
try {
command.run();
+ return 0;
} catch (Exception e) {
System.out.println("Exception while processing command "
+ command.getClass().getSimpleName() + " : " + e);
e.printStackTrace();
- return;
+ return 1;
}
}
@@ -89,6 +95,7 @@
help();
return;
}
- new Dicttool().execute(arguments);
+ // Exit with the success/error code from #execute() as status.
+ System.exit(new Dicttool().execute(arguments));
}
}
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java
index df5ea35..972b6e7 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java
@@ -16,10 +16,12 @@
package com.android.inputmethod.latin.dicttool;
+import com.android.inputmethod.latin.makedict.BinaryDictIOTests;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtilsTests;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest;
import com.android.inputmethod.latin.makedict.FusionDictionaryTest;
+import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
@@ -36,7 +38,8 @@
BinaryDictOffdeviceUtilsTests.class,
FusionDictionaryTest.class,
BinaryDictInputOutputTest.class,
- BinaryDictIOUtilsTests.class
+ BinaryDictIOUtilsTests.class,
+ BinaryDictIOTests.class
};
private ArrayList<Method> mAllTestMethods = new ArrayList<Method>();
private ArrayList<String> mUsedTestMethods = new ArrayList<String>();
@@ -86,12 +89,19 @@
for (final Method m : mAllTestMethods) {
final Class<?> declaringClass = m.getDeclaringClass();
if (!mUsedTestMethods.isEmpty() && !mUsedTestMethods.contains(m.getName())) continue;
- final Object instance;
- if (BinaryDictIOUtilsTests.class == declaringClass) {
- instance = new BinaryDictIOUtilsTests(mSeed, mMaxUnigrams);
- } else {
- instance = declaringClass.newInstance();
+ // Some of the test classes expose a two-argument constructor, taking a long as a
+ // seed for Random, and an int for a vocabulary size to test the dictionary with. They
+ // correspond respectively to the -s and -m numerical arguments to this command, which
+ // are stored in mSeed and mMaxUnigrams. If the two-arguments constructor is present,
+ // then invoke it; otherwise, invoke the default constructor.
+ Constructor<?> twoArgsConstructor = null;
+ try {
+ twoArgsConstructor = declaringClass.getDeclaredConstructor(Long.TYPE, Integer.TYPE);
+ } catch (NoSuchMethodException e) {
+ // No constructor with two args
}
+ final Object instance = null == twoArgsConstructor ? declaringClass.newInstance()
+ : twoArgsConstructor.newInstance(mSeed, mMaxUnigrams);
m.invoke(instance);
}
}