Merge "Update dictionaries"
diff --git a/java/res/values-land/dimens.xml b/java/res/values-land/dimens.xml
index 42a746b..b874d48 100644
--- a/java/res/values-land/dimens.xml
+++ b/java/res/values-land/dimens.xml
@@ -77,7 +77,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">10%p</fraction>
<fraction name="emoji_keyboard_row_height">50%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">60%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">54%p</fraction>
<integer name="emoji_keyboard_max_key_count">20</integer>
</resources>
diff --git a/java/res/values-sw600dp-land/dimens.xml b/java/res/values-sw600dp-land/dimens.xml
index 730b7d8..d067265 100644
--- a/java/res/values-sw600dp-land/dimens.xml
+++ b/java/res/values-sw600dp-land/dimens.xml
@@ -66,7 +66,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">10%p</fraction>
<fraction name="emoji_keyboard_row_height">33%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">95%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">85%p</fraction>
<integer name="emoji_keyboard_max_key_count">30</integer>
</resources>
diff --git a/java/res/values-sw600dp/dimens.xml b/java/res/values-sw600dp/dimens.xml
index 2bcf2fa..591355d 100644
--- a/java/res/values-sw600dp/dimens.xml
+++ b/java/res/values-sw600dp/dimens.xml
@@ -92,7 +92,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">12.5%p</fraction>
<fraction name="emoji_keyboard_row_height">33%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">85%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">76%p</fraction>
<integer name="emoji_keyboard_max_key_count">24</integer>
</resources>
diff --git a/java/res/values-sw768dp-land/dimens.xml b/java/res/values-sw768dp-land/dimens.xml
index 1e2e1c6..664630b 100644
--- a/java/res/values-sw768dp-land/dimens.xml
+++ b/java/res/values-sw768dp-land/dimens.xml
@@ -67,7 +67,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">7.69%p</fraction>
<fraction name="emoji_keyboard_row_height">33%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">75%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">68%p</fraction>
<integer name="emoji_keyboard_max_key_count">39</integer>
</resources>
diff --git a/java/res/values-sw768dp/dimens.xml b/java/res/values-sw768dp/dimens.xml
index f62a536..1fd933a 100644
--- a/java/res/values-sw768dp/dimens.xml
+++ b/java/res/values-sw768dp/dimens.xml
@@ -92,7 +92,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">10%p</fraction>
<fraction name="emoji_keyboard_row_height">33%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">85%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">76%p</fraction>
<integer name="emoji_keyboard_max_key_count">30</integer>
</resources>
diff --git a/java/res/values/dimens.xml b/java/res/values/dimens.xml
index 4e3b2f5..2d626db 100644
--- a/java/res/values/dimens.xml
+++ b/java/res/values/dimens.xml
@@ -118,7 +118,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">14.2857%p</fraction>
<fraction name="emoji_keyboard_row_height">33%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">90%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">81%p</fraction>
<integer name="emoji_keyboard_max_key_count">21</integer>
<dimen name="emoji_category_page_id_height">3dp</dimen>
diff --git a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
index 61dc56e..eb48d01 100644
--- a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
+++ b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
@@ -718,12 +718,14 @@
@Override
public void run() {
+ int repeatCount = 1;
int timeCount = 0;
while (timeCount < MAX_REPEAT_COUNT_TIME && !mAborted) {
if (timeCount > mKeyRepeatStartTimeout) {
- pressDelete();
+ pressDelete(repeatCount);
}
timeCount += mKeyRepeatInterval;
+ ++repeatCount;
try {
Thread.sleep(mKeyRepeatInterval);
} catch (InterruptedException e) {
@@ -736,9 +738,9 @@
}
}
- public void pressDelete() {
+ public void pressDelete(int repeatCount) {
mKeyboardActionListener.onPressKey(
- Constants.CODE_DELETE, 0 /* repeatCount */, true /* isSinglePointer */);
+ Constants.CODE_DELETE, repeatCount, true /* isSinglePointer */);
mKeyboardActionListener.onCodeInput(
Constants.CODE_DELETE, NOT_A_COORDINATE, NOT_A_COORDINATE);
mKeyboardActionListener.onReleaseKey(
@@ -754,7 +756,7 @@
switch(event.getAction()) {
case MotionEvent.ACTION_DOWN:
v.setBackgroundColor(mDeleteKeyPressedBackgroundColor);
- pressDelete();
+ pressDelete(0 /* repeatCount */);
startRepeat();
return true;
case MotionEvent.ACTION_UP:
diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
index cc1ffd1..74edd87 100644
--- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
+++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
@@ -315,7 +315,7 @@
}
public boolean isShowingEmojiKeyboard() {
- return mEmojiKeyboardView.getVisibility() == View.VISIBLE;
+ return mEmojiKeyboardView != null && mEmojiKeyboardView.getVisibility() == View.VISIBLE;
}
public boolean isShowingMoreKeysPanel() {
diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
index 845a9b9..4a0ce37 100644
--- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
@@ -25,6 +25,7 @@
import java.io.File;
import java.io.IOException;
+import java.util.Map;
// TODO: Quit extending Dictionary after implementing dynamic binary dictionary.
abstract public class AbstractDictionaryWriter extends Dictionary {
@@ -50,16 +51,16 @@
abstract public void removeBigramWords(final String word0, final String word1);
- abstract protected void writeDictionary(final DictEncoder dictEncoder)
- throws IOException, UnsupportedFormatException;
+ abstract protected void writeDictionary(final DictEncoder dictEncoder,
+ final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException;
- public void write(final String fileName) {
+ public void write(final String fileName, final Map<String, String> attributeMap) {
final String tempFileName = fileName + ".temp";
final File file = new File(mContext.getFilesDir(), fileName);
final File tempFile = new File(mContext.getFilesDir(), tempFileName);
try {
final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile);
- writeDictionary(dictEncoder);
+ writeDictionary(dictEncoder, attributeMap);
tempFile.renameTo(file);
} catch (IOException e) {
Log.e(TAG, "IO exception while writing file", e);
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index a463651..29c6c04 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -48,6 +48,11 @@
// TODO: Remove this heuristic.
private static final int SPACE_COUNT_FOR_AUTO_COMMIT = 3;
+ @UsedForTesting
+ public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
+ @UsedForTesting
+ public static final String BIGRAM_COUNT_QUERY = "BIGRAM_COUNT";
+
private long mNativeDict;
private final Locale mLocale;
private final long mDictSize;
@@ -110,7 +115,7 @@
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
boolean isUpdatable);
private static native void flushNative(long dict, String filePath);
- private static native boolean needsToRunGCNative(long dict);
+ private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC);
private static native void flushWithGCNative(long dict, String filePath);
private static native void closeNative(long dict);
private static native int getProbabilityNative(long dict, int[] word);
@@ -129,6 +134,7 @@
private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1);
private static native int calculateProbabilityNative(long dict, int unigramProbability,
int bigramProbability);
+ private static native String getPropertyNative(long dict, String query);
@UsedForTesting
public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
@@ -264,7 +270,7 @@
}
private void runGCIfRequired() {
- if (needsToRunGCNative(mNativeDict)) {
+ if (needsToRunGC(true /* mindsBlockByGC */)) {
flushWithGC();
}
}
@@ -320,9 +326,15 @@
reopen();
}
- public boolean needsToRunGC() {
+ /**
+ * Checks whether GC is needed to run or not.
+ * @param mindsBlockByGC Whether to mind operations blocked by GC. We don't need to care about
+ * the blocking in some situations such as in idle time or just before closing.
+ * @return whether GC is needed to run or not.
+ */
+ public boolean needsToRunGC(final boolean mindsBlockByGC) {
if (!isValidDictionary()) return false;
- return needsToRunGCNative(mNativeDict);
+ return needsToRunGCNative(mNativeDict, mindsBlockByGC);
}
@UsedForTesting
@@ -331,6 +343,12 @@
return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability);
}
+ @UsedForTesting
+ public String getPropertyForTests(String query) {
+ if (!isValidDictionary()) return "";
+ return getPropertyNative(mNativeDict, query);
+ }
+
@Override
public boolean shouldAutoCommit(final SuggestedWordInfo candidate) {
// TODO: actually use the confidence rather than use this completely broken heuristic
diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
index 5a453dd..84abfa6 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
@@ -31,6 +31,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Map;
/**
* An in memory dictionary for memorizing entries and writing a binary dictionary.
@@ -84,8 +85,11 @@
}
@Override
- protected void writeDictionary(final DictEncoder dictEncoder)
- throws IOException, UnsupportedFormatException {
+ protected void writeDictionary(final DictEncoder dictEncoder,
+ final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException {
+ for (final Map.Entry<String, String> entry : attributeMap.entrySet()) {
+ mFusionDictionary.addOptionAttribute(entry.getKey(), entry.getValue());
+ }
dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS);
}
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index a578bb9..2d1ca51 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -34,6 +34,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -69,14 +70,15 @@
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE;
/**
- * A static map of time recorders, each of which records the time of accesses to a single binary
- * dictionary file. The key for this map is the filename and the value is the shared dictionary
- * time recorder associated with that filename.
+ * A static map of update controllers, each of which records the time of accesses to a single
+ * binary dictionary file and tracks whether the file is regenerating. The key for this map is
+ * the filename and the value is the shared dictionary time recorder associated with that
+ * filename.
*/
- private static volatile ConcurrentHashMap<String, DictionaryTimeRecorder>
- sFilenameDictionaryTimeRecorderMap = CollectionUtils.newConcurrentHashMap();
+ private static final ConcurrentHashMap<String, DictionaryUpdateController>
+ sFilenameDictionaryUpdateControllerMap = CollectionUtils.newConcurrentHashMap();
- private static volatile ConcurrentHashMap<String, PrioritizedSerialExecutor>
+ private static final ConcurrentHashMap<String, PrioritizedSerialExecutor>
sFilenameExecutorMap = CollectionUtils.newConcurrentHashMap();
/** The application context. */
@@ -103,13 +105,13 @@
private final boolean mIsUpdatable;
// TODO: remove, once dynamic operations is serialized
- /** Records access to the shared binary dictionary file across multiple instances. */
- private final DictionaryTimeRecorder mFilenameDictionaryTimeRecorder;
+ /** Controls updating the shared binary dictionary file across multiple instances. */
+ private final DictionaryUpdateController mFilenameDictionaryUpdateController;
// TODO: remove, once dynamic operations is serialized
- /** Records access to the local binary dictionary for this instance. */
- private final DictionaryTimeRecorder mPerInstanceDictionaryTimeRecorder =
- new DictionaryTimeRecorder();
+ /** Controls updating the local binary dictionary for this instance. */
+ private final DictionaryUpdateController mPerInstanceDictionaryUpdateController =
+ new DictionaryUpdateController();
/* A extension for a binary dictionary file. */
public static final String DICT_FILE_EXTENSION = ".dict";
@@ -131,15 +133,15 @@
protected abstract boolean hasContentChanged();
/**
- * Gets the dictionary time recorder for the given filename.
+ * Gets the dictionary update controller for the given filename.
*/
- private static DictionaryTimeRecorder getDictionaryTimeRecorder(
+ private static DictionaryUpdateController getDictionaryUpdateController(
String filename) {
- DictionaryTimeRecorder recorder = sFilenameDictionaryTimeRecorderMap.get(filename);
+ DictionaryUpdateController recorder = sFilenameDictionaryUpdateControllerMap.get(filename);
if (recorder == null) {
- synchronized(sFilenameDictionaryTimeRecorderMap) {
- recorder = new DictionaryTimeRecorder();
- sFilenameDictionaryTimeRecorderMap.put(filename, recorder);
+ synchronized(sFilenameDictionaryUpdateControllerMap) {
+ recorder = new DictionaryUpdateController();
+ sFilenameDictionaryUpdateControllerMap.put(filename, recorder);
}
}
return recorder;
@@ -189,7 +191,7 @@
mContext = context;
mIsUpdatable = isUpdatable;
mBinaryDictionary = null;
- mFilenameDictionaryTimeRecorder = getDictionaryTimeRecorder(filename);
+ mFilenameDictionaryUpdateController = getDictionaryUpdateController(filename);
// Currently, only dynamic personalization dictionary is updatable.
mDictionaryWriter = getDictionaryWriter(context, dictType, isUpdatable);
}
@@ -234,6 +236,7 @@
HashMap<String, String> attributeMap = new HashMap<String, String>();
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
SUPPORTS_DYNAMIC_UPDATE);
+ attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFilename);
return attributeMap;
}
@@ -347,6 +350,9 @@
final boolean blockOffensiveWords, final int[] additionalFeaturesOptions,
final int sessionId) {
reloadDictionaryIfRequired();
+ if (isRegenerating()) {
+ return null;
+ }
final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList();
final AsyncResultHolder<ArrayList<SuggestedWordInfo>> holder =
new AsyncResultHolder<ArrayList<SuggestedWordInfo>>();
@@ -407,6 +413,9 @@
}
protected boolean isValidWordInner(final String word) {
+ if (isRegenerating()) {
+ return false;
+ }
final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>();
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
@@ -432,7 +441,7 @@
* dictionary exists, this method will generate one.
*/
protected void loadDictionary() {
- mPerInstanceDictionaryTimeRecorder.mLastUpdateRequestTime = SystemClock.uptimeMillis();
+ mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = SystemClock.uptimeMillis();
reloadDictionaryIfRequired();
}
@@ -443,8 +452,8 @@
private void loadBinaryDictionary() {
if (DEBUG) {
Log.d(TAG, "Loading binary dictionary: " + mFilename + " request="
- + mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime + " update="
- + mFilenameDictionaryTimeRecorder.mLastUpdateTime);
+ + mFilenameDictionaryUpdateController.mLastUpdateRequestTime + " update="
+ + mFilenameDictionaryUpdateController.mLastUpdateTime);
}
final File file = new File(mContext.getFilesDir(), mFilename);
@@ -482,13 +491,13 @@
private void writeBinaryDictionary() {
if (DEBUG) {
Log.d(TAG, "Generating binary dictionary: " + mFilename + " request="
- + mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime + " update="
- + mFilenameDictionaryTimeRecorder.mLastUpdateTime);
+ + mFilenameDictionaryUpdateController.mLastUpdateRequestTime + " update="
+ + mFilenameDictionaryUpdateController.mLastUpdateTime);
}
if (needsToReloadBeforeWriting()) {
mDictionaryWriter.clear();
loadDictionaryAsync();
- mDictionaryWriter.write(mFilename);
+ mDictionaryWriter.write(mFilename, getHeaderAttributeMap());
} else {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
@@ -496,14 +505,14 @@
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
} else {
- if (mBinaryDictionary.needsToRunGC()) {
+ if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
mBinaryDictionary.flushWithGC();
} else {
mBinaryDictionary.flush();
}
}
} else {
- mDictionaryWriter.write(mFilename);
+ mDictionaryWriter.write(mFilename, getHeaderAttributeMap());
}
}
}
@@ -517,11 +526,11 @@
*/
protected void setRequiresReload(final boolean requiresRebuild) {
final long time = SystemClock.uptimeMillis();
- mPerInstanceDictionaryTimeRecorder.mLastUpdateRequestTime = time;
- mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime = time;
+ mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = time;
+ mFilenameDictionaryUpdateController.mLastUpdateRequestTime = time;
if (DEBUG) {
Log.d(TAG, "Reload request: " + mFilename + ": request=" + time + " update="
- + mFilenameDictionaryTimeRecorder.mLastUpdateTime);
+ + mFilenameDictionaryUpdateController.mLastUpdateTime);
}
}
@@ -530,14 +539,26 @@
*/
public final void reloadDictionaryIfRequired() {
if (!isReloadRequired()) return;
- reloadDictionary();
+ if (setIsRegeneratingIfNotRegenerating()) {
+ reloadDictionary();
+ }
}
/**
* Returns whether a dictionary reload is required.
*/
private boolean isReloadRequired() {
- return mBinaryDictionary == null || mPerInstanceDictionaryTimeRecorder.isOutOfDate();
+ return mBinaryDictionary == null || mPerInstanceDictionaryUpdateController.isOutOfDate();
+ }
+
+ private boolean isRegenerating() {
+ return mFilenameDictionaryUpdateController.mIsRegenerating.get();
+ }
+
+ // Returns whether the dictionary can be regenerated.
+ private boolean setIsRegeneratingIfNotRegenerating() {
+ return mFilenameDictionaryUpdateController.mIsRegenerating.compareAndSet(
+ false /* expect */ , true /* update */);
}
/**
@@ -550,39 +571,44 @@
getExecutor(mFilename).execute(new Runnable() {
@Override
public void run() {
- final long time = SystemClock.uptimeMillis();
- final boolean dictionaryFileExists = dictionaryFileExists();
- if (mFilenameDictionaryTimeRecorder.isOutOfDate() || !dictionaryFileExists) {
- // If the shared dictionary file does not exist or is out of date, the first
- // instance that acquires the lock will generate a new one.
- if (hasContentChanged() || !dictionaryFileExists) {
- // If the source content has changed or the dictionary does not exist,
- // rebuild the binary dictionary. Empty dictionaries are supported (in the
- // case where loadDictionaryAsync() adds nothing) in order to provide a
- // uniform framework.
- mFilenameDictionaryTimeRecorder.mLastUpdateTime = time;
+ try {
+ final long time = SystemClock.uptimeMillis();
+ final boolean dictionaryFileExists = dictionaryFileExists();
+ if (mFilenameDictionaryUpdateController.isOutOfDate()
+ || !dictionaryFileExists) {
+ // If the shared dictionary file does not exist or is out of date, the
+ // first instance that acquires the lock will generate a new one.
+ if (hasContentChanged() || !dictionaryFileExists) {
+ // If the source content has changed or the dictionary does not exist,
+ // rebuild the binary dictionary. Empty dictionaries are supported (in
+ // the case where loadDictionaryAsync() adds nothing) in order to
+ // provide a uniform framework.
+ mFilenameDictionaryUpdateController.mLastUpdateTime = time;
+ writeBinaryDictionary();
+ loadBinaryDictionary();
+ } else {
+ // If not, the reload request was unnecessary so revert
+ // LastUpdateRequestTime to LastUpdateTime.
+ mFilenameDictionaryUpdateController.mLastUpdateRequestTime =
+ mFilenameDictionaryUpdateController.mLastUpdateTime;
+ }
+ } else if (mBinaryDictionary == null ||
+ mPerInstanceDictionaryUpdateController.mLastUpdateTime
+ < mFilenameDictionaryUpdateController.mLastUpdateTime) {
+ // Otherwise, if the local dictionary is older than the shared dictionary,
+ // load the shared dictionary.
+ loadBinaryDictionary();
+ }
+ if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) {
+ // Binary dictionary is not valid. Regenerate the dictionary file.
+ mFilenameDictionaryUpdateController.mLastUpdateTime = time;
writeBinaryDictionary();
loadBinaryDictionary();
- } else {
- // If not, the reload request was unnecessary so revert
- // LastUpdateRequestTime to LastUpdateTime.
- mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime =
- mFilenameDictionaryTimeRecorder.mLastUpdateTime;
}
- } else if (mBinaryDictionary == null ||
- mPerInstanceDictionaryTimeRecorder.mLastUpdateTime
- < mFilenameDictionaryTimeRecorder.mLastUpdateTime) {
- // Otherwise, if the local dictionary is older than the shared dictionary, load
- // the shared dictionary.
- loadBinaryDictionary();
+ mPerInstanceDictionaryUpdateController.mLastUpdateTime = time;
+ } finally {
+ mFilenameDictionaryUpdateController.mIsRegenerating.set(false);
}
- if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) {
- // Binary dictionary is not valid. Regenerate the dictionary file.
- mFilenameDictionaryTimeRecorder.mLastUpdateTime = time;
- writeBinaryDictionary();
- loadBinaryDictionary();
- }
- mPerInstanceDictionaryTimeRecorder.mLastUpdateTime = time;
}
});
}
@@ -622,14 +648,15 @@
}
/**
- * Time recorder for tracking whether the dictionary is out of date.
+ * For tracking whether the dictionary is out of date and the dictionary is regenerating.
* Can be shared across multiple dictionary instances that access the same filename.
*/
- private static class DictionaryTimeRecorder {
- private volatile long mLastUpdateTime = 0;
- private volatile long mLastUpdateRequestTime = 0;
+ private static class DictionaryUpdateController {
+ public volatile long mLastUpdateTime = 0;
+ public volatile long mLastUpdateRequestTime = 0;
+ public volatile AtomicBoolean mIsRegenerating = new AtomicBoolean();
- private boolean isOutOfDate() {
+ public boolean isOutOfDate() {
return (mLastUpdateRequestTime > mLastUpdateTime);
}
}
@@ -646,7 +673,7 @@
holder.set(mBinaryDictionary.isValidWord(word));
} else {
holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
- .isInDictionaryForTests(word));
+ .isInBigramListForTests(word));
}
}
}
diff --git a/java/src/com/android/inputmethod/latin/InputAttributes.java b/java/src/com/android/inputmethod/latin/InputAttributes.java
index 21b103e..8caf6f1 100644
--- a/java/src/com/android/inputmethod/latin/InputAttributes.java
+++ b/java/src/com/android/inputmethod/latin/InputAttributes.java
@@ -103,6 +103,10 @@
}
}
+ public boolean isTypeNull() {
+ return InputType.TYPE_NULL == mInputType;
+ }
+
public boolean isSameInputType(final EditorInfo editorInfo) {
return editorInfo.inputType == mInputType;
}
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index 270dc4c..0c2625f 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -81,7 +81,7 @@
import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister;
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary;
-import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
+import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.settings.SettingsActivity;
import com.android.inputmethod.latin.settings.SettingsValues;
@@ -179,7 +179,7 @@
private boolean mIsMainDictionaryAvailable;
private UserBinaryDictionary mUserDictionary;
- private UserHistoryPredictionDictionary mUserHistoryPredictionDictionary;
+ private UserHistoryDictionary mUserHistoryDictionary;
private PersonalizationPredictionDictionary mPersonalizationPredictionDictionary;
private PersonalizationDictionary mPersonalizationDictionary;
private boolean mIsUserDictionaryAvailable;
@@ -623,9 +623,9 @@
final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
- mUserHistoryPredictionDictionary = PersonalizationHelper
- .getUserHistoryPredictionDictionary(this, localeStr, prefs);
- newSuggest.setUserHistoryPredictionDictionary(mUserHistoryPredictionDictionary);
+ mUserHistoryDictionary = PersonalizationHelper.getUserHistoryDictionary(
+ this, localeStr, prefs);
+ newSuggest.setUserHistoryDictionary(mUserHistoryDictionary);
mPersonalizationDictionary = PersonalizationHelper
.getPersonalizationDictionary(this, localeStr, prefs);
newSuggest.setPersonalizationDictionary(mPersonalizationDictionary);
@@ -1521,7 +1521,7 @@
mSubtypeState.switchSubtype(token, mRichImm);
}
- private void sendDownUpKeyEventForBackwardCompatibility(final int code) {
+ private void sendDownUpKeyEvent(final int code) {
final long eventTime = SystemClock.uptimeMillis();
mConnection.sendKeyEvent(new KeyEvent(eventTime, eventTime,
KeyEvent.ACTION_DOWN, code, 0, 0, KeyCharacterMap.VIRTUAL_KEYBOARD, 0,
@@ -1538,7 +1538,7 @@
// TODO: Remove this special handling of digit letters.
// For backward compatibility. See {@link InputMethodService#sendKeyChar(char)}.
if (code >= '0' && code <= '9') {
- sendDownUpKeyEventForBackwardCompatibility(code - '0' + KeyEvent.KEYCODE_0);
+ sendDownUpKeyEvent(code - '0' + KeyEvent.KEYCODE_0);
return;
}
@@ -1547,7 +1547,7 @@
// a hardware keyboard event on pressing enter or delete. This is bad for many
// reasons (there are race conditions with commits) but some applications are
// relying on this behavior so we continue to support it for older apps.
- sendDownUpKeyEventForBackwardCompatibility(KeyEvent.KEYCODE_ENTER);
+ sendDownUpKeyEvent(KeyEvent.KEYCODE_ENTER);
} else {
final String text = new String(new int[] { code }, 0, 1);
mConnection.commitText(text, text.length());
@@ -2104,12 +2104,16 @@
}
final int lengthToDelete = Character.isSupplementaryCodePoint(
mConnection.getCodePointBeforeCursor()) ? 2 : 1;
- if (mAppWorkAroundsUtils.isBeforeJellyBean()) {
- // Backward compatibility mode. Before Jelly bean, the keyboard would simulate
- // a hardware keyboard event on pressing enter or delete. This is bad for many
- // reasons (there are race conditions with commits) but some applications are
- // relying on this behavior so we continue to support it for older apps.
- sendDownUpKeyEventForBackwardCompatibility(KeyEvent.KEYCODE_DEL);
+ if (mAppWorkAroundsUtils.isBeforeJellyBean() ||
+ currentSettings.mInputAttributes.isTypeNull()) {
+ // There are two possible reasons to send a key event: either the field has
+ // type TYPE_NULL, in which case the keyboard should send events, or we are
+ // running in backward compatibility mode. Before Jelly bean, the keyboard
+ // would simulate a hardware keyboard event on pressing enter or delete. This
+ // is bad for many reasons (there are race conditions with commits) but some
+ // applications are relying on this behavior so we continue to support it for
+ // older apps, so we retain this behavior if the app has target SDK < JellyBean.
+ sendDownUpKeyEvent(KeyEvent.KEYCODE_DEL);
} else {
mConnection.deleteSurroundingText(lengthToDelete, 0);
}
@@ -2746,9 +2750,8 @@
final SettingsValues currentSettings = mSettings.getCurrent();
if (!currentSettings.mCorrectionEnabled) return null;
- final UserHistoryPredictionDictionary userHistoryPredictionDictionary =
- mUserHistoryPredictionDictionary;
- if (userHistoryPredictionDictionary == null) return null;
+ final UserHistoryDictionary userHistoryDictionary = mUserHistoryDictionary;
+ if (userHistoryDictionary == null) return null;
final String prevWord = mConnection.getNthPreviousWord(currentSettings.mWordSeparators, 2);
final String secondWord;
@@ -2762,8 +2765,7 @@
final int maxFreq = AutoCorrectionUtils.getMaxFrequency(
suggest.getUnigramDictionaries(), suggestion);
if (maxFreq == 0) return null;
- userHistoryPredictionDictionary
- .addToPersonalizationPredictionDictionary(prevWord, secondWord, maxFreq > 0);
+ userHistoryDictionary.addToDictionary(prevWord, secondWord, maxFreq > 0);
return prevWord;
}
@@ -2949,7 +2951,7 @@
}
mConnection.deleteSurroundingText(deleteLength, 0);
if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) {
- mUserHistoryPredictionDictionary.cancelAddingUserHistory(previousWord, committedWord);
+ mUserHistoryDictionary.cancelAddingUserHistory(previousWord, committedWord);
}
final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString;
if (mSettings.getCurrent().mCurrentLanguageHasSpaces) {
diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java
index 6c18c94..9fd1f53 100644
--- a/java/src/com/android/inputmethod/latin/Suggest.java
+++ b/java/src/com/android/inputmethod/latin/Suggest.java
@@ -26,7 +26,7 @@
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary;
-import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
+import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.AutoCorrectionUtils;
import com.android.inputmethod.latin.utils.BoundedTreeSet;
@@ -190,10 +190,8 @@
addOrReplaceDictionaryInternal(Dictionary.TYPE_CONTACTS, contactsDictionary);
}
- public void setUserHistoryPredictionDictionary(
- final UserHistoryPredictionDictionary userHistoryPredictionDictionary) {
- addOrReplaceDictionaryInternal(Dictionary.TYPE_USER_HISTORY,
- userHistoryPredictionDictionary);
+ public void setUserHistoryDictionary(final UserHistoryDictionary userHistoryDictionary) {
+ addOrReplaceDictionaryInternal(Dictionary.TYPE_USER_HISTORY, userHistoryDictionary);
}
public void setPersonalizationPredictionDictionary(
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
index 6cc0bfb..af61f29 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
@@ -385,12 +385,14 @@
nodeSize + size, ptNode.mChildren));
}
nodeSize += getShortcutListSize(ptNode.mShortcutTargets);
- if (null != ptNode.mBigrams) {
- for (WeightedString bigram : ptNode.mBigrams) {
- final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray,
- nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE,
- FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord));
- nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE;
+ if (formatOptions.mVersion < FormatSpec.FIRST_VERSION_WITH_TERMINAL_ID) {
+ if (null != ptNode.mBigrams) {
+ for (WeightedString bigram : ptNode.mBigrams) {
+ final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray,
+ nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE,
+ FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord));
+ nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE;
+ }
}
}
ptNode.mCachedSize = nodeSize;
diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
index bf3d191..411e265 100644
--- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
@@ -77,7 +77,7 @@
* @param newParentAddress the absolute address of the parent.
* @param formatOptions file format options.
*/
- public static void updateParentAddress(final DictBuffer dictBuffer,
+ private static void updateParentAddress(final DictBuffer dictBuffer,
final int ptNodeOriginAddress, final int newParentAddress,
final FormatOptions formatOptions) {
final int originalPosition = dictBuffer.position();
@@ -109,7 +109,7 @@
* @param newParentAddress the address to be written.
* @param formatOptions file format options.
*/
- public static void updateParentAddresses(final DictBuffer dictBuffer,
+ private static void updateParentAddresses(final DictBuffer dictBuffer,
final int ptNodeOriginAddress, final int newParentAddress,
final FormatOptions formatOptions) {
final int originalPosition = dictBuffer.position();
@@ -136,7 +136,7 @@
* @param newChildrenAddress the absolute address of the child.
* @param formatOptions file format options.
*/
- public static void updateChildrenAddress(final DictBuffer dictBuffer,
+ private static void updateChildrenAddress(final DictBuffer dictBuffer,
final int ptNodeOriginAddress, final int newChildrenAddress,
final FormatOptions formatOptions) {
final int originalPosition = dictBuffer.position();
diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
index 849bff0..9481a8c 100644
--- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
+++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
@@ -265,8 +265,12 @@
static final String FREQ_FILE_EXTENSION = ".freq";
// tat = Terminal Address Table
static final String TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat";
+ static final String BIGRAM_FILE_EXTENSION = ".bigram";
+ static final String BIGRAM_LOOKUP_TABLE_FILE_EXTENSION = ".bigram_lookup";
+ static final String BIGRAM_ADDRESS_TABLE_FILE_EXTENSION = ".bigram_index";
static final int FREQUENCY_AND_FLAGS_SIZE = 2;
static final int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3;
+ static final int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4;
static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE;
static final int NO_PARENT_ADDRESS = 0;
@@ -331,9 +335,9 @@
public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE";
public static final String ATTRIBUTE_VALUE_TRUE = "1";
- private static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
- private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
- private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";
+ public static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
+ public static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
+ public static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";
private static final String DICTIONARY_DESCRIPTION_ATTRIBUTE = "description";
public FileHeader(final int headerSize, final DictionaryOptions dictionaryOptions,
final FormatOptions formatOptions) {
diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java
index 0b9cf91..96d057a 100644
--- a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java
+++ b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java
@@ -18,6 +18,9 @@
import com.android.inputmethod.annotations.UsedForTesting;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
@@ -147,4 +150,45 @@
BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4);
}
}
+
+ @UsedForTesting
+ public void writeToFiles(final File lookupTableFile, final File contentFile)
+ throws IOException {
+ FileOutputStream lookupTableOutStream = null;
+ FileOutputStream contentOutStream = null;
+ try {
+ lookupTableOutStream = new FileOutputStream(lookupTableFile);
+ contentOutStream = new FileOutputStream(contentFile);
+ write(lookupTableOutStream, contentOutStream);
+ } finally {
+ if (lookupTableOutStream != null) {
+ lookupTableOutStream.close();
+ }
+ if (contentOutStream != null) {
+ contentOutStream.close();
+ }
+ }
+ }
+
+ private static byte[] readFileToByteArray(final File file) throws IOException {
+ final byte[] contents = new byte[(int) file.length()];
+ FileInputStream inStream = null;
+ try {
+ inStream = new FileInputStream(file);
+ inStream.read(contents);
+ } finally {
+ if (inStream != null) {
+ inStream.close();
+ }
+ }
+ return contents;
+ }
+
+ @UsedForTesting
+ public static SparseTable readFromFiles(final File lookupTableFile, final File contentFile,
+ final int blockSize) throws IOException {
+ final byte[] lookupTable = readFileToByteArray(lookupTableFile);
+ final byte[] content = readFileToByteArray(contentFile);
+ return new SparseTable(lookupTable, content, blockSize);
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
index 4c8ff8e..0aa4319 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
@@ -42,12 +42,15 @@
private static final int FILETYPE_TRIE = 1;
private static final int FILETYPE_FREQUENCY = 2;
private static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3;
+ private static final int FILETYPE_BIGRAM = 4;
private final File mDictDirectory;
private final DictionaryBufferFactory mBufferFactory;
private DictBuffer mDictBuffer;
private DictBuffer mFrequencyBuffer;
private DictBuffer mTerminalAddressTableBuffer;
+ private DictBuffer mBigramBuffer;
+ private SparseTable mBigramAddressTable;
@UsedForTesting
/* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) {
@@ -82,6 +85,9 @@
} else if (fileType == FILETYPE_TERMINAL_ADDRESS_TABLE) {
return new File(mDictDirectory,
mDictDirectory.getName() + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
+ } else if (fileType == FILETYPE_BIGRAM) {
+ return new File(mDictDirectory,
+ mDictDirectory.getName() + FormatSpec.BIGRAM_FILE_EXTENSION);
} else {
throw new RuntimeException("Unsupported kind of file : " + fileType);
}
@@ -94,6 +100,8 @@
mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY));
mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer(
getFile(FILETYPE_TERMINAL_ADDRESS_TABLE));
+ mBigramBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_BIGRAM));
+ loadBigramAddressSparseTable();
}
@Override
@@ -118,6 +126,15 @@
return header;
}
+ private void loadBigramAddressSparseTable() throws IOException {
+ final File lookupIndexFile = new File(mDictDirectory,
+ mDictDirectory.getName() + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION);
+ final File contentFile = new File(mDictDirectory,
+ mDictDirectory.getName() + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION);
+ mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, contentFile,
+ FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE);
+ }
+
protected static class PtNodeReader extends DictDecoder.PtNodeReader {
protected static int readFrequency(final DictBuffer frequencyBuffer, final int terminalId) {
frequencyBuffer.position(terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE + 1);
@@ -191,8 +208,21 @@
final ArrayList<PendingAttribute> bigrams;
if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
bigrams = new ArrayList<PendingAttribute>();
- addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams,
- addressPointer);
+ final int posOfBigrams = mBigramAddressTable.get(terminalId);
+ mBigramBuffer.position(posOfBigrams);
+ while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
+ // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE,
+ // remaining bigram entries are ignored.
+ final int bigramFlags = mBigramBuffer.readUnsignedByte();
+ final int targetTerminalId = mBigramBuffer.readUnsignedInt24();
+ mTerminalAddressTableBuffer.position(
+ targetTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE);
+ final int targetAddress = mTerminalAddressTableBuffer.readUnsignedInt24();
+ bigrams.add(new PendingAttribute(
+ bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY,
+ targetAddress));
+ if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break;
+ }
if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
MakedictLog.d("too many bigrams in a node.");
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
index 4fb8967..4c25faf 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
@@ -26,6 +26,7 @@
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -43,9 +44,13 @@
private byte[] mTrieBuf;
private int mTriePos;
private int mHeaderSize;
+ private SparseTable mBigramAddressTable;
private OutputStream mTrieOutStream;
private OutputStream mFreqOutStream;
private OutputStream mTerminalAddressTableOutStream;
+ private OutputStream mBigramOutStream;
+ private File mDictDir;
+ private String mBaseFilename;
@UsedForTesting
public Ver4DictEncoder(final File dictPlacedDir) {
@@ -55,12 +60,14 @@
private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions)
throws FileNotFoundException, IOException {
final FileHeader header = new FileHeader(0, dictOptions, formatOptions);
- final String filename = header.getId() + "." + header.getVersion();
- final File mDictDir = new File(mDictPlacedDir, filename);
- final File trieFile = new File(mDictDir, filename + FormatSpec.TRIE_FILE_EXTENSION);
- final File freqFile = new File(mDictDir, filename + FormatSpec.FREQ_FILE_EXTENSION);
+ mBaseFilename = header.getId() + "." + header.getVersion();
+ mDictDir = new File(mDictPlacedDir, mBaseFilename);
+ final File trieFile = new File(mDictDir, mBaseFilename + FormatSpec.TRIE_FILE_EXTENSION);
+ final File freqFile = new File(mDictDir, mBaseFilename + FormatSpec.FREQ_FILE_EXTENSION);
final File terminalAddressTableFile = new File(mDictDir,
- filename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
+ mBaseFilename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
+ final File bigramFile = new File(mDictDir,
+ mBaseFilename + FormatSpec.BIGRAM_FILE_EXTENSION);
if (!mDictDir.isDirectory()) {
if (mDictDir.exists()) mDictDir.delete();
mDictDir.mkdirs();
@@ -71,6 +78,7 @@
mTrieOutStream = new FileOutputStream(trieFile);
mFreqOutStream = new FileOutputStream(freqFile);
mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile);
+ mBigramOutStream = new FileOutputStream(bigramFile);
}
private void close() throws IOException {
@@ -84,10 +92,14 @@
if (mTerminalAddressTableOutStream != null) {
mTerminalAddressTableOutStream.close();
}
+ if (mBigramOutStream != null) {
+ mBigramOutStream.close();
+ }
} finally {
mTrieOutStream = null;
mFreqOutStream = null;
mTerminalAddressTableOutStream = null;
+ mBigramOutStream = null;
}
}
@@ -123,6 +135,10 @@
if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes);
writeTerminalData(flatNodes, terminalCount);
+ mBigramAddressTable = new SparseTable(terminalCount,
+ FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE);
+ writeBigrams(flatNodes, dict);
+ writeBigramAddressSparseTable();
final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize;
@@ -230,24 +246,41 @@
shortcutByteSize, FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE);
}
- private void writeBigrams(ArrayList<WeightedString> bigrams, FusionDictionary dict) {
- if (bigrams == null) return;
+ private void writeBigrams(final ArrayList<PtNodeArray> flatNodes, final FusionDictionary dict)
+ throws IOException {
+ final ByteArrayOutputStream bigramBuffer = new ByteArrayOutputStream();
- final Iterator<WeightedString> bigramIterator = bigrams.iterator();
- while (bigramIterator.hasNext()) {
- final WeightedString bigram = bigramIterator.next();
- final PtNode target =
- FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord);
- final int addressOfBigram = target.mCachedAddressAfterUpdate;
- final int unigramFrequencyForThisWord = target.mFrequency;
- final int offset = addressOfBigram
- - (mTriePos + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
- int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(),
- offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
- mTrieBuf[mTriePos++] = (byte) bigramFlags;
- mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf,
- mTriePos, Math.abs(offset));
+ for (final PtNodeArray nodeArray : flatNodes) {
+ for (final PtNode ptNode : nodeArray.mData) {
+ if (ptNode.mBigrams != null) {
+ final int startPos = bigramBuffer.size();
+ mBigramAddressTable.set(ptNode.mTerminalId, startPos);
+ final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator();
+ while (bigramIterator.hasNext()) {
+ final WeightedString bigram = bigramIterator.next();
+ final PtNode target =
+ FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord);
+ final int unigramFrequencyForThisWord = target.mFrequency;
+ final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(
+ bigramIterator.hasNext(), 0, bigram.mFrequency,
+ unigramFrequencyForThisWord, bigram.mWord);
+ BinaryDictEncoderUtils.writeUIntToStream(bigramBuffer, bigramFlags,
+ FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
+ BinaryDictEncoderUtils.writeUIntToStream(bigramBuffer, target.mTerminalId,
+ FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE);
+ }
+ }
+ }
}
+ bigramBuffer.writeTo(mBigramOutStream);
+ }
+
+ private void writeBigramAddressSparseTable() throws IOException {
+ final File lookupIndexFile =
+ new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION);
+ final File contentFile =
+ new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION);
+ mBigramAddressTable.writeToFiles(lookupIndexFile, contentFile);
}
@Override
@@ -267,7 +300,6 @@
}
writeChildrenPosition(ptNode, formatOptions);
writeShortcuts(ptNode.mShortcutTargets);
- writeBigrams(ptNode.mBigrams, dict);
}
private void writeTerminalData(final ArrayList<PtNodeArray> flatNodes,
diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
index 66517a8..1ca91a0 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
@@ -94,6 +94,8 @@
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+ attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName);
+ attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale);
return attributeMap;
}
@@ -117,15 +119,14 @@
}
/**
- * Pair will be added to the personalization prediction dictionary.
+ * Pair will be added to the decaying dictionary.
*
* The first word may be null. That means we don't know the context, in other words,
* it's only a unigram. The first word may also be an empty string : this means start
* context, as in beginning of a sentence for example.
* The second word may not be null (a NullPointerException would be thrown).
*/
- public void addToPersonalizationPredictionDictionary(
- final String word0, final String word1, final boolean isValid) {
+ public void addToDictionary(final String word0, final String word1, final boolean isValid) {
if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
return;
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
index 0af028a..039b253 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
@@ -36,6 +36,7 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Map;
// Currently this class is used to implement dynamic prodiction dictionary.
// TODO: Move to native code.
@@ -79,7 +80,7 @@
public void addUnigramWord(final String word, final String shortcutTarget, final int frequency,
final boolean isNotAWord) {
if (mBigramList.size() > mMaxHistoryBigrams * 2) {
- // Too many entries: just stop adding new vocabrary and wait next refresh.
+ // Too many entries: just stop adding new vocabulary and wait next refresh.
return;
}
mExpandableDictionary.addWord(word, shortcutTarget, frequency);
@@ -90,7 +91,7 @@
public void addBigramWords(final String word0, final String word1, final int frequency,
final boolean isValid, final long lastModifiedTime) {
if (mBigramList.size() > mMaxHistoryBigrams * 2) {
- // Too many entries: just stop adding new vocabrary and wait next refresh.
+ // Too many entries: just stop adding new vocabulary and wait next refresh.
return;
}
if (lastModifiedTime > 0) {
@@ -113,8 +114,8 @@
}
@Override
- protected void writeDictionary(final DictEncoder dictEncoder)
- throws IOException, UnsupportedFormatException {
+ protected void writeDictionary(final DictEncoder dictEncoder,
+ final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException {
UserHistoryDictIOUtils.writeDictionary(dictEncoder,
new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams),
mBigramList, FORMAT_OPTIONS);
@@ -176,8 +177,8 @@
}
@UsedForTesting
- public boolean isInDictionaryForTests(final String word) {
+ public boolean isInBigramListForTests(final String word) {
// TODO: Use native method to determine whether the word is in dictionary or not
- return mBigramList.containsKey(word);
+ return mBigramList.containsKey(word) || mBigramList.getBigrams(null).containsKey(word);
}
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
index c616a29..a86f6e5 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
@@ -110,7 +110,7 @@
if (dictionary == null) {
return;
}
- dictionary.addToPersonalizationPredictionDictionary(word0, word1, isValid);
+ dictionary.addToDictionary(word0, word1, isValid);
}
// Bulk import
@@ -122,8 +122,7 @@
return;
}
for (final PersonalizationLanguageModelParam lmParam : lmParams) {
- dictionary.addToPersonalizationPredictionDictionary(
- lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid);
+ dictionary.addToDictionary(lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid);
}
}
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
index 5f702ee..8c9484b 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
@@ -30,7 +30,7 @@
private static final String TAG = PersonalizationHelper.class.getSimpleName();
private static final boolean DEBUG = false;
- private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>>
+ private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>>
sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap();
private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>>
@@ -41,25 +41,23 @@
sLangPersonalizationPredictionDictCache =
CollectionUtils.newConcurrentHashMap();
- public static UserHistoryPredictionDictionary getUserHistoryPredictionDictionary(
+ public static UserHistoryDictionary getUserHistoryDictionary(
final Context context, final String locale, final SharedPreferences sp) {
synchronized (sLangUserHistoryDictCache) {
if (sLangUserHistoryDictCache.containsKey(locale)) {
- final SoftReference<UserHistoryPredictionDictionary> ref =
+ final SoftReference<UserHistoryDictionary> ref =
sLangUserHistoryDictCache.get(locale);
- final UserHistoryPredictionDictionary dict = ref == null ? null : ref.get();
+ final UserHistoryDictionary dict = ref == null ? null : ref.get();
if (dict != null) {
if (DEBUG) {
- Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale);
+ Log.w(TAG, "Use cached UserHistoryDictionary for " + locale);
}
dict.reloadDictionaryIfRequired();
return dict;
}
}
- final UserHistoryPredictionDictionary dict =
- new UserHistoryPredictionDictionary(context, locale, sp);
- sLangUserHistoryDictCache.put(
- locale, new SoftReference<UserHistoryPredictionDictionary>(dict));
+ final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale, sp);
+ sLangUserHistoryDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict));
return dict;
}
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
similarity index 84%
rename from java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
rename to java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
index 38e308a..a60226d 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
@@ -26,10 +26,10 @@
* Locally gathers stats about the words user types and various other signals like auto-correction
* cancellation or manual picks. This allows the keyboard to adapt to the typist over time.
*/
-public class UserHistoryPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
+public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase {
/* package for tests */ static final String NAME =
- UserHistoryPredictionDictionary.class.getSimpleName();
- /* package */ UserHistoryPredictionDictionary(final Context context, final String locale,
+ UserHistoryDictionary.class.getSimpleName();
+ /* package */ UserHistoryDictionary(final Context context, final String locale,
final SharedPreferences sp) {
super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale));
}
diff --git a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java
index 5dc0b58..201a70d 100644
--- a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java
+++ b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java
@@ -16,8 +16,11 @@
package com.android.inputmethod.latin.utils;
-import java.util.ArrayDeque;
import java.util.Queue;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
/**
* An object that executes submitted tasks using a thread.
@@ -27,19 +30,20 @@
private final Object mLock = new Object();
- // The default value of capacities of task queues.
- private static final int TASK_QUEUE_CAPACITY = 1000;
private final Queue<Runnable> mTasks;
private final Queue<Runnable> mPrioritizedTasks;
private boolean mIsShutdown;
+ private final ThreadPoolExecutor mThreadPoolExecutor;
// The task which is running now.
private Runnable mActive;
public PrioritizedSerialExecutor() {
- mTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY);
- mPrioritizedTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY);
+ mTasks = new ConcurrentLinkedQueue<Runnable>();
+ mPrioritizedTasks = new ConcurrentLinkedQueue<Runnable>();
mIsShutdown = false;
+ mThreadPoolExecutor = new ThreadPoolExecutor(1 /* corePoolSize */, 1 /* maximumPoolSize */,
+ 0 /* keepAliveTime */, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<Runnable>(1));
}
/**
@@ -59,7 +63,16 @@
public void execute(final Runnable r) {
synchronized(mLock) {
if (!mIsShutdown) {
- mTasks.offer(r);
+ mTasks.offer(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ r.run();
+ } finally {
+ scheduleNext();
+ }
+ }
+ });
if (mActive == null) {
scheduleNext();
}
@@ -74,45 +87,36 @@
public void executePrioritized(final Runnable r) {
synchronized(mLock) {
if (!mIsShutdown) {
- mPrioritizedTasks.offer(r);
- if (mActive == null) {
+ mPrioritizedTasks.offer(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ r.run();
+ } finally {
+ scheduleNext();
+ }
+ }
+ });
+ if (mActive == null) {
scheduleNext();
}
}
}
}
- private boolean fetchNextTasks() {
- synchronized(mLock) {
- mActive = mPrioritizedTasks.poll();
- if (mActive == null) {
- mActive = mTasks.poll();
- }
- return mActive != null;
+ private boolean fetchNextTasksLocked() {
+ mActive = mPrioritizedTasks.poll();
+ if (mActive == null) {
+ mActive = mTasks.poll();
}
+ return mActive != null;
}
private void scheduleNext() {
synchronized(mLock) {
- if (!fetchNextTasks()) {
- return;
+ if (fetchNextTasksLocked()) {
+ mThreadPoolExecutor.execute(mActive);
}
- new Thread(new Runnable() {
- @Override
- public void run() {
- try {
- do {
- synchronized(mLock) {
- if (mActive != null) {
- mActive.run();
- }
- }
- } while (fetchNextTasks());
- } finally {
- scheduleNext();
- }
- }
- }).start();
}
}
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 7761ec4..c5ef264 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -113,10 +113,10 @@
}
static bool latinime_BinaryDictionary_needsToRunGC(JNIEnv *env, jclass clazz,
- jlong dict) {
+ jlong dict, jboolean mindsBlockByGC) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return false;
- return dictionary->needsToRunGC();
+ return dictionary->needsToRunGC(mindsBlockByGC == JNI_TRUE);
}
static void latinime_BinaryDictionary_flushWithGC(JNIEnv *env, jclass clazz, jlong dict,
@@ -323,6 +323,24 @@
bigramProbability);
}
+static jstring latinime_BinaryDictionary_getProperty(JNIEnv *env, jclass clazz, jlong dict,
+ jstring query) {
+ Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+ if (!dictionary) {
+ return env->NewStringUTF("");
+ }
+ const jsize queryUtf8Length = env->GetStringUTFLength(query);
+ char queryChars[queryUtf8Length + 1];
+ env->GetStringUTFRegion(query, 0, env->GetStringLength(query), queryChars);
+ queryChars[queryUtf8Length] = '\0';
+ static const int GET_PROPERTY_RESULT_LENGTH = 100;
+ char resultChars[GET_PROPERTY_RESULT_LENGTH];
+ resultChars[0] = '\0';
+ dictionary->getDictionaryStructurePolicy()->getProperty(queryChars, resultChars,
+ GET_PROPERTY_RESULT_LENGTH);
+ return env->NewStringUTF(resultChars);
+}
+
static const JNINativeMethod sMethods[] = {
{
const_cast<char *>("createEmptyDictFileNative"),
@@ -346,7 +364,7 @@
},
{
const_cast<char *>("needsToRunGCNative"),
- const_cast<char *>("(J)Z"),
+ const_cast<char *>("(JZ)Z"),
reinterpret_cast<void *>(latinime_BinaryDictionary_needsToRunGC)
},
{
@@ -398,6 +416,11 @@
const_cast<char *>("calculateProbabilityNative"),
const_cast<char *>("(JII)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_calculateProbabilityNative)
+ },
+ {
+ const_cast<char *>("getPropertyNative"),
+ const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_getProperty)
}
};
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp
index ec1b63a..b1d01ed 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp
@@ -33,6 +33,8 @@
namespace latinime {
+const int Dictionary::HEADER_ATTRIBUTE_BUFFER_SIZE = 32;
+
Dictionary::Dictionary(JNIEnv *env,
DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy)
: mDictionaryStructureWithBufferPolicy(dictionaryStructureWithBufferPolicy),
@@ -121,32 +123,37 @@
mDictionaryStructureWithBufferPolicy->flushWithGC(filePath);
}
-bool Dictionary::needsToRunGC() {
- return mDictionaryStructureWithBufferPolicy->needsToRunGC();
+bool Dictionary::needsToRunGC(const bool mindsBlockByGC) {
+ return mDictionaryStructureWithBufferPolicy->needsToRunGC(mindsBlockByGC);
+}
+
+void Dictionary::getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const {
+ return mDictionaryStructureWithBufferPolicy->getProperty(query, outResult, maxResultLength);
}
void Dictionary::logDictionaryInfo(JNIEnv *const env) const {
- const int BUFFER_SIZE = 16;
- int dictionaryIdCodePointBuffer[BUFFER_SIZE];
- int versionStringCodePointBuffer[BUFFER_SIZE];
- int dateStringCodePointBuffer[BUFFER_SIZE];
+ int dictionaryIdCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
+ int versionStringCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
+ int dateStringCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
const DictionaryHeaderStructurePolicy *const headerPolicy =
getDictionaryStructurePolicy()->getHeaderStructurePolicy();
headerPolicy->readHeaderValueOrQuestionMark("dictionary", dictionaryIdCodePointBuffer,
- BUFFER_SIZE);
+ HEADER_ATTRIBUTE_BUFFER_SIZE);
headerPolicy->readHeaderValueOrQuestionMark("version", versionStringCodePointBuffer,
- BUFFER_SIZE);
- headerPolicy->readHeaderValueOrQuestionMark("date", dateStringCodePointBuffer, BUFFER_SIZE);
+ HEADER_ATTRIBUTE_BUFFER_SIZE);
+ headerPolicy->readHeaderValueOrQuestionMark("date", dateStringCodePointBuffer,
+ HEADER_ATTRIBUTE_BUFFER_SIZE);
- char dictionaryIdCharBuffer[BUFFER_SIZE];
- char versionStringCharBuffer[BUFFER_SIZE];
- char dateStringCharBuffer[BUFFER_SIZE];
- intArrayToCharArray(dictionaryIdCodePointBuffer, BUFFER_SIZE,
- dictionaryIdCharBuffer, BUFFER_SIZE);
- intArrayToCharArray(versionStringCodePointBuffer, BUFFER_SIZE,
- versionStringCharBuffer, BUFFER_SIZE);
- intArrayToCharArray(dateStringCodePointBuffer, BUFFER_SIZE,
- dateStringCharBuffer, BUFFER_SIZE);
+ char dictionaryIdCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
+ char versionStringCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
+ char dateStringCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
+ intArrayToCharArray(dictionaryIdCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE,
+ dictionaryIdCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE);
+ intArrayToCharArray(versionStringCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE,
+ versionStringCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE);
+ intArrayToCharArray(dateStringCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE,
+ dateStringCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE);
LogUtils::logToJava(env,
"Dictionary info: dictionary = %s ; version = %s ; date = %s",
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h
index 9744474..d8a0f3e 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.h
+++ b/native/jni/src/suggest/core/dictionary/dictionary.h
@@ -81,7 +81,10 @@
void flushWithGC(const char *const filePath);
- bool needsToRunGC();
+ bool needsToRunGC(const bool mindsBlockByGC);
+
+ void getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const;
const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
return mDictionaryStructureWithBufferPolicy;
@@ -92,6 +95,8 @@
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
+ static const int HEADER_ATTRIBUTE_BUFFER_SIZE;
+
DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy;
const BigramDictionary *const mBigramDictionary;
const SuggestInterface *const mGestureSuggest;
diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
index b95488e..c7ffef0 100644
--- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
+++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
@@ -78,7 +78,10 @@
virtual void flushWithGC(const char *const filePath) = 0;
- virtual bool needsToRunGC() const = 0;
+ virtual bool needsToRunGC(const bool mindsBlockByGC) const = 0;
+
+ virtual void getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const = 0;
protected:
DictionaryStructureWithBufferPolicy() {}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
index 29307b5..e02f4cb 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
@@ -119,7 +119,7 @@
// Finding useless bigram entries and remove them. Bigram entry is useless when the target PtNode
// has been deleted or is not a valid terminal.
bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(
- int *const bigramListPos) {
+ int *const bigramListPos, int *const outValidBigramEntryCount) {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
if (usesAdditionalBuffer) {
*bigramListPos -= mBuffer->getOriginalBufferSize();
@@ -161,6 +161,8 @@
NOT_A_DICT_POS /* targetOffset */, &bigramEntryPos)) {
return false;
}
+ } else {
+ (*outValidBigramEntryCount) += 1;
}
} while(BigramListReadWriteUtils::hasNext(bigramFlags));
return true;
@@ -169,7 +171,7 @@
// Updates bigram target PtNode positions in the list after the placing step in GC.
bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos,
const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const
- ptNodePositionRelocationMap) {
+ ptNodePositionRelocationMap, int *const outBigramEntryCount) {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
if (usesAdditionalBuffer) {
*bigramListPos -= mBuffer->getOriginalBufferSize();
@@ -211,11 +213,12 @@
return false;
}
} while(BigramListReadWriteUtils::hasNext(bigramFlags));
+ (*outBigramEntryCount) = bigramEntryCount;
return true;
}
bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTargetPos,
- const int probability, int *const bigramListPos) {
+ const int probability, int *const bigramListPos, bool *const outAddedNewBigram) {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
if (usesAdditionalBuffer) {
*bigramListPos -= mBuffer->getOriginalBufferSize();
@@ -243,6 +246,7 @@
}
if (followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos) == bigramTargetPos) {
// Update this bigram entry.
+ *outAddedNewBigram = false;
const BigramListReadWriteUtils::BigramFlags updatedFlags =
BigramListReadWriteUtils::setProbabilityInFlags(bigramFlags, probability);
return BigramListReadWriteUtils::writeBigramEntry(mBuffer, updatedFlags,
@@ -254,12 +258,14 @@
// The current last entry is found.
// First, update the flags of the last entry.
if (!BigramListReadWriteUtils::setHasNextFlag(mBuffer, true /* hasNext */, entryPos)) {
+ *outAddedNewBigram = false;
return false;
}
if (usesAdditionalBuffer) {
*bigramListPos += mBuffer->getOriginalBufferSize();
}
// Then, add a new entry after the last entry.
+ *outAddedNewBigram = true;
return writeNewBigramEntry(bigramTargetPos, probability, bigramListPos);
} while(BigramListReadWriteUtils::hasNext(bigramFlags));
// We return directly from the while loop.
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
index 8ea318a..3ebf699 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
@@ -50,19 +50,20 @@
bool copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite, int *const fromPos,
int *const toPos, int *const outBigramsCount) const;
- bool updateAllBigramEntriesAndDeleteUselessEntries(int *const bigramListPos);
+ bool updateAllBigramEntriesAndDeleteUselessEntries(int *const bigramListPos,
+ int *const outBigramEntryCount);
bool updateAllBigramTargetPtNodePositions(int *const bigramListPos,
const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const
- ptNodePositionRelocationMap);
+ ptNodePositionRelocationMap, int *const outValidBigramEntryCount);
bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability,
- int *const bigramListPos);
+ int *const bigramListPos, bool *const outAddedNewBigram);
bool writeNewBigramEntry(const int bigramTargetPos, const int probability,
int *const writingPos);
- // Return if targetBigramPos is found or not.
+ // Return whether or not targetBigramPos is found.
bool removeBigram(const int bigramListPos, const int bigramTargetPos);
private:
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
index c60e458..5f755c3 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
@@ -41,7 +41,27 @@
return false;
}
} else {
- valueStack.back() += 1;
+ mValueStack.back() += 1;
+ if (node->isTerminal()) {
+ mValidUnigramCount += 1;
+ }
+ }
+ return true;
+}
+
+bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability
+ ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
+ const int *const nodeCodePoints) {
+ if (!node->isDeleted()) {
+ int pos = node->getBigramsPos();
+ if (pos != NOT_A_DICT_POS) {
+ int bigramEntryCount = 0;
+ if (!mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&pos,
+ &bigramEntryCount)) {
+ return false;
+ }
+ mValidBigramEntryCount += bigramEntryCount;
+ }
}
return true;
}
@@ -137,10 +157,15 @@
// Updates bigram target PtNode positions in the bigram list.
int bigramsPos = node->getBigramsPos();
if (bigramsPos != NOT_A_DICT_POS) {
+ int bigramEntryCount;
if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos,
- &mDictPositionRelocationMap->mPtNodePositionRelocationMap)) {
+ &mDictPositionRelocationMap->mPtNodePositionRelocationMap, &bigramEntryCount)) {
return false;
}
+ mBigramCount += bigramEntryCount;
+ }
+ if (node->isTerminal()) {
+ mUnigramCount++;
}
return true;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
index 4256f22..3019988 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
@@ -40,22 +40,22 @@
TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
DynamicPatriciaTrieWritingHelper *const writingHelper,
BufferWithExtendableBuffer *const buffer)
- : mWritingHelper(writingHelper), mBuffer(buffer), valueStack(),
- mChildrenValue(0) {}
+ : mWritingHelper(writingHelper), mBuffer(buffer), mValueStack(),
+ mChildrenValue(0), mValidUnigramCount(0) {}
~TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted() {};
bool onAscend() {
- if (valueStack.empty()) {
+ if (mValueStack.empty()) {
return false;
}
- mChildrenValue = valueStack.back();
- valueStack.pop_back();
+ mChildrenValue = mValueStack.back();
+ mValueStack.pop_back();
return true;
}
bool onDescend(const int ptNodeArrayPos) {
- valueStack.push_back(0);
+ mValueStack.push_back(0);
return true;
}
@@ -64,14 +64,19 @@
bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
const int *const nodeCodePoints);
+ int getValidUnigramCount() const {
+ return mValidUnigramCount;
+ }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(
TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted);
DynamicPatriciaTrieWritingHelper *const mWritingHelper;
BufferWithExtendableBuffer *const mBuffer;
- std::vector<int> valueStack;
+ std::vector<int> mValueStack;
int mChildrenValue;
+ int mValidUnigramCount;
};
// Updates all bigram entries that are held by valid PtNodes. This removes useless bigram
@@ -80,7 +85,7 @@
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
public:
TraversePolicyToUpdateBigramProbability(DynamicBigramListPolicy *const bigramPolicy)
- : mBigramPolicy(bigramPolicy) {}
+ : mBigramPolicy(bigramPolicy), mValidBigramEntryCount(0) {}
bool onAscend() { return true; }
@@ -89,22 +94,17 @@
bool onReadingPtNodeArrayTail() { return true; }
bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
- const int *const nodeCodePoints) {
- if (!node->isDeleted()) {
- int pos = node->getBigramsPos();
- if (pos != NOT_A_DICT_POS) {
- if (!mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&pos)) {
- return false;
- }
- }
- }
- return true;
+ const int *const nodeCodePoints);
+
+ int getValidBigramEntryCount() const {
+ return mValidBigramEntryCount;
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability);
DynamicBigramListPolicy *const mBigramPolicy;
+ int mValidBigramEntryCount;
};
class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
@@ -150,7 +150,8 @@
dictPositionRelocationMap)
: mWritingHelper(writingHelper), mBigramPolicy(bigramPolicy),
mBufferToWrite(bufferToWrite),
- mDictPositionRelocationMap(dictPositionRelocationMap) {};
+ mDictPositionRelocationMap(dictPositionRelocationMap), mUnigramCount(0),
+ mBigramCount(0) {};
bool onAscend() { return true; }
@@ -161,6 +162,14 @@
bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
const int *const nodeCodePoints);
+ int getUnigramCount() const {
+ return mUnigramCount;
+ }
+
+ int getBigramCount() const {
+ return mBigramCount;
+ }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields);
@@ -169,6 +178,8 @@
BufferWithExtendableBuffer *const mBufferToWrite;
const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
mDictPositionRelocationMap;
+ int mUnigramCount;
+ int mBigramCount;
};
private:
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
index 456352c..2fa3111 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
@@ -26,7 +26,8 @@
void DynamicPatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProcessMovedPtNode(
const int ptNodePos, const int maxCodePointCount, int *const outCodePoints) {
if (ptNodePos < 0 || ptNodePos >= mBuffer->getTailPosition()) {
- AKLOGE("Fetching PtNode info form invalid dictionary position: %d, dictionary size: %d",
+ // Reading invalid position because of bug or broken dictionary.
+ AKLOGE("Fetching PtNode info from invalid dictionary position: %d, dictionary size: %d",
ptNodePos, mBuffer->getTailPosition());
ASSERT(false);
invalidatePtNodeInfo();
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
index 42397c1..8c0890e 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
@@ -16,6 +16,9 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h"
+#include <cstdio>
+#include <cstring>
+
#include "defines.h"
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_vector.h"
@@ -28,6 +31,9 @@
namespace latinime {
+const char *const DynamicPatriciaTriePolicy::UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
+const char *const DynamicPatriciaTriePolicy::BIGRAM_COUNT_QUERY = "BIGRAM_COUNT";
+
void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode,
DicNodeVector *const childDicNodes) const {
if (!dicNode->hasChildren()) {
@@ -198,7 +204,16 @@
readingHelper.initWithPtNodeArrayPos(getRootPosition());
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
&mBigramListPolicy, &mShortcutListPolicy);
- return writingHelper.addUnigramWord(&readingHelper, word, length, probability);
+ bool addedNewUnigram = false;
+ if (writingHelper.addUnigramWord(&readingHelper, word, length, probability,
+ &addedNewUnigram)) {
+ if (addedNewUnigram) {
+ mUnigramCount++;
+ }
+ return true;
+ } else {
+ return false;
+ }
}
bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int length0,
@@ -219,7 +234,15 @@
}
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
&mBigramListPolicy, &mShortcutListPolicy);
- return writingHelper.addBigramWords(word0Pos, word1Pos, probability);
+ bool addedNewBigram = false;
+ if (writingHelper.addBigramWords(word0Pos, word1Pos, probability, &addedNewBigram)) {
+ if (addedNewBigram) {
+ mBigramCount++;
+ }
+ return true;
+ } else {
+ return false;
+ }
}
bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0,
@@ -240,7 +263,12 @@
}
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
&mBigramListPolicy, &mShortcutListPolicy);
- return writingHelper.removeBigramWords(word0Pos, word1Pos);
+ if (writingHelper.removeBigramWords(word0Pos, word1Pos)) {
+ mBigramCount--;
+ return true;
+ } else {
+ return false;
+ }
}
void DynamicPatriciaTriePolicy::flush(const char *const filePath) {
@@ -250,7 +278,7 @@
}
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
&mBigramListPolicy, &mShortcutListPolicy);
- writingHelper.writeToDictFile(filePath, &mHeaderPolicy);
+ writingHelper.writeToDictFile(filePath, &mHeaderPolicy, mUnigramCount, mBigramCount);
}
void DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) {
@@ -263,7 +291,7 @@
writingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy);
}
-bool DynamicPatriciaTriePolicy::needsToRunGC() const {
+bool DynamicPatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
if (!mBuffer->isUpdatable()) {
AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
return false;
@@ -272,4 +300,13 @@
return mBufferWithExtendableBuffer.isNearSizeLimit();
}
+void DynamicPatriciaTriePolicy::getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const {
+ if (strncmp(query, UNIGRAM_COUNT_QUERY, maxResultLength) == 0) {
+ snprintf(outResult, maxResultLength, "%d", mUnigramCount);
+ } else if (strncmp(query, BIGRAM_COUNT_QUERY, maxResultLength) == 0) {
+ snprintf(outResult, maxResultLength, "%d", mBigramCount);
+ }
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
index 06d8095..bdb436c 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
@@ -37,7 +37,9 @@
mBufferWithExtendableBuffer(mBuffer->getBuffer() + mHeaderPolicy.getSize(),
mBuffer->getBufferSize() - mHeaderPolicy.getSize()),
mShortcutListPolicy(&mBufferWithExtendableBuffer),
- mBigramListPolicy(&mBufferWithExtendableBuffer, &mShortcutListPolicy) {}
+ mBigramListPolicy(&mBufferWithExtendableBuffer, &mShortcutListPolicy),
+ mUnigramCount(mHeaderPolicy.getUnigramCount()),
+ mBigramCount(mHeaderPolicy.getBigramCount()) {}
~DynamicPatriciaTriePolicy() {
delete mBuffer;
@@ -89,16 +91,24 @@
void flushWithGC(const char *const filePath);
- bool needsToRunGC() const;
+ bool needsToRunGC(const bool mindsBlockByGC) const;
+
+ void getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);
+ static const char*const UNIGRAM_COUNT_QUERY;
+ static const char*const BIGRAM_COUNT_QUERY;
+
const MmappedBuffer *const mBuffer;
const HeaderPolicy mHeaderPolicy;
BufferWithExtendableBuffer mBufferWithExtendableBuffer;
DynamicShortcutListPolicy mShortcutListPolicy;
DynamicBigramListPolicy mBigramListPolicy;
+ int mUnigramCount;
+ int mBigramCount;
};
} // namespace latinime
#endif // LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp
index f4a2ef3..601ee66 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp
@@ -155,6 +155,15 @@
// Read node array size and process empty node arrays. Nodes and arrays are counted up in this
// method to avoid an infinite loop.
void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() {
+ if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) {
+ // Reading invalid position because of a bug or a broken dictionary.
+ AKLOGE("Reading PtNode array info from invalid dictionary position: %d, dict size: %d",
+ mReadingState.mPos, mBuffer->getTailPosition());
+ ASSERT(false);
+ mIsError = true;
+ mReadingState.mPos = NOT_A_DICT_POS;
+ return;
+ }
mReadingState.mPosOfLastPtNodeArrayHead = mReadingState.mPos;
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(mReadingState.mPos);
const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer);
@@ -191,6 +200,15 @@
// Follow the forward link and read the next node array if exists.
void DynamicPatriciaTrieReadingHelper::followForwardLink() {
+ if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) {
+ // Reading invalid position because of bug or broken dictionary.
+ AKLOGE("Reading forward link from invalid dictionary position: %d, dict size: %d",
+ mReadingState.mPos, mBuffer->getTailPosition());
+ ASSERT(false);
+ mIsError = true;
+ mReadingState.mPos = NOT_A_DICT_POS;
+ return;
+ }
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(mReadingState.mPos);
const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer);
if (usesAdditionalBuffer) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h
index c6d8ddc..512a4d8 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h
@@ -240,6 +240,7 @@
static const int MAX_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP;
static const size_t MAX_READING_STATE_STACK_SIZE;
+ // TODO: Introduce error code to track what caused the error.
bool mIsError;
ReadingState mReadingState;
const BufferWithExtendableBuffer *const mBuffer;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
index 578645c..2a2e9bc 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
@@ -36,7 +36,8 @@
bool DynamicPatriciaTrieWritingHelper::addUnigramWord(
DynamicPatriciaTrieReadingHelper *const readingHelper,
- const int *const wordCodePoints, const int codePointCount, const int probability) {
+ const int *const wordCodePoints, const int codePointCount, const int probability,
+ bool *const outAddedNewUnigram) {
int parentPos = NOT_A_DICT_POS;
while (!readingHelper->isEnd()) {
const int matchedCodePointCount = readingHelper->getPrevTotalCodePointCount();
@@ -54,6 +55,7 @@
const int nextIndex = matchedCodePointCount + j;
if (nextIndex >= codePointCount || !readingHelper->isMatchedCodePoint(j,
wordCodePoints[matchedCodePointCount + j])) {
+ *outAddedNewUnigram = true;
return reallocatePtNodeAndAddNewPtNodes(nodeReader,
readingHelper->getMergedNodeCodePoints(), j, probability,
wordCodePoints + matchedCodePointCount,
@@ -63,9 +65,10 @@
// All characters are matched.
if (codePointCount == readingHelper->getTotalCodePointCount()) {
return setPtNodeProbability(nodeReader, probability,
- readingHelper->getMergedNodeCodePoints());
+ readingHelper->getMergedNodeCodePoints(), outAddedNewUnigram);
}
if (!nodeReader->hasChildren()) {
+ *outAddedNewUnigram = true;
return createChildrenPtNodeArrayAndAChildPtNode(nodeReader, probability,
wordCodePoints + readingHelper->getTotalCodePointCount(),
codePointCount - readingHelper->getTotalCodePointCount());
@@ -79,6 +82,7 @@
return false;
}
int pos = readingHelper->getPosOfLastForwardLinkField();
+ *outAddedNewUnigram = true;
return createAndInsertNodeIntoPtNodeArray(parentPos,
wordCodePoints + readingHelper->getPrevTotalCodePointCount(),
codePointCount - readingHelper->getPrevTotalCodePointCount(),
@@ -86,7 +90,7 @@
}
bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const int word1Pos,
- const int probability) {
+ const int probability, bool *const outAddedNewBigram) {
int mMergedNodeCodePoints[MAX_WORD_LENGTH];
DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy);
nodeReader.fetchNodeInfoInBufferFromPtNodePosAndGetNodeCodePoints(word0Pos, MAX_WORD_LENGTH,
@@ -107,9 +111,11 @@
if (nodeReader.getBigramsPos() != NOT_A_DICT_POS) {
// Insert a new bigram entry into the existing bigram list.
int bigramListPos = nodeReader.getBigramsPos();
- return mBigramPolicy->addNewBigramEntryToBigramList(word1Pos, probability, &bigramListPos);
+ return mBigramPolicy->addNewBigramEntryToBigramList(word1Pos, probability, &bigramListPos,
+ outAddedNewBigram);
} else {
// The PtNode doesn't have a bigram list.
+ *outAddedNewBigram = true;
// First, Write a bigram entry at the tail position of the PtNode.
if (!mBigramPolicy->writeNewBigramEntry(word1Pos, probability, &writingPos)) {
return false;
@@ -138,9 +144,12 @@
}
void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileName,
- const HeaderPolicy *const headerPolicy) {
+ const HeaderPolicy *const headerPolicy, const int unigramCount, const int bigramCount) {
BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
- if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) {
+ const int extendedRegionSize = headerPolicy->getExtendedRegionSize() +
+ mBuffer->getUsedAdditionalBufferSize();
+ if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */,
+ unigramCount, bigramCount, extendedRegionSize)) {
return;
}
DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, mBuffer);
@@ -148,13 +157,16 @@
void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
const char *const fileName, const HeaderPolicy *const headerPolicy) {
- BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
- if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */)) {
- return;
- }
BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */,
MAX_DICTIONARY_SIZE);
- if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) {
+ int unigramCount = 0;
+ int bigramCount = 0;
+ if (!runGC(rootPtNodeArrayPos, &newDictBuffer, &unigramCount, &bigramCount)) {
+ return;
+ }
+ BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
+ if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */,
+ unigramCount, bigramCount, 0 /* extendedRegionSize */)) {
return;
}
DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, &newDictBuffer);
@@ -335,9 +347,10 @@
bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability(
const DynamicPatriciaTrieNodeReader *const originalPtNode, const int probability,
- const int *const codePoints) {
+ const int *const codePoints, bool *const outAddedNewUnigram) {
if (originalPtNode->isTerminal()) {
// Overwrites the probability.
+ *outAddedNewUnigram = false;
int probabilityFieldPos = originalPtNode->getProbabilityFieldPos();
if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(mBuffer,
probability, &probabilityFieldPos)) {
@@ -345,6 +358,7 @@
}
} else {
// Make the node terminal and write the probability.
+ *outAddedNewUnigram = true;
int movedPos = mBuffer->getTailPosition();
if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) {
return false;
@@ -460,7 +474,8 @@
}
bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
- BufferWithExtendableBuffer *const bufferToWrite) {
+ BufferWithExtendableBuffer *const bufferToWrite, int *const outUnigramCount,
+ int *const outBigramCount) {
DynamicPatriciaTrieReadingHelper readingHelper(mBuffer, mBigramPolicy, mShortcutPolicy);
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners
@@ -505,6 +520,8 @@
&traversePolicyToUpdateAllPositionFields)) {
return false;
}
+ *outUnigramCount = traversePolicyToUpdateAllPositionFields.getUnigramCount();
+ *outBigramCount = traversePolicyToUpdateAllPositionFields.getBigramCount();
return true;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
index fe1b243..827b609 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
@@ -56,15 +56,18 @@
// Add a word to the dictionary. If the word already exists, update the probability.
bool addUnigramWord(DynamicPatriciaTrieReadingHelper *const readingHelper,
- const int *const wordCodePoints, const int codePointCount, const int probability);
+ const int *const wordCodePoints, const int codePointCount, const int probability,
+ bool *const outAddedNewUnigram);
// Add a bigram relation from word0Pos to word1Pos.
- bool addBigramWords(const int word0Pos, const int word1Pos, const int probability);
+ bool addBigramWords(const int word0Pos, const int word1Pos, const int probability,
+ bool *const outAddedNewBigram);
// Remove a bigram relation from word0Pos to word1Pos.
bool removeBigramWords(const int word0Pos, const int word1Pos);
- void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy);
+ void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy,
+ const int unigramCount, const int bigramCount);
void writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const fileName,
const HeaderPolicy *const headerPolicy);
@@ -107,7 +110,7 @@
const int nodeCodePointCount, const int probability, int *const forwardLinkFieldPos);
bool setPtNodeProbability(const DynamicPatriciaTrieNodeReader *const originalNode,
- const int probability, const int *const codePoints);
+ const int probability, const int *const codePoints, bool *const outAddedNewUnigram);
bool createChildrenPtNodeArrayAndAChildPtNode(
const DynamicPatriciaTrieNodeReader *const parentNode, const int probability,
@@ -122,7 +125,8 @@
const int probabilityOfNewPtNode, const int *const newNodeCodePoints,
const int newNodeCodePointCount);
- bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite);
+ bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite,
+ int *const outUnigramCount, int *const outBigramCount);
};
} // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
index 7bbeaca..9ce9994 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
@@ -16,17 +16,16 @@
#include "suggest/policyimpl/dictionary/header/header_policy.h"
-#include <cstddef>
-#include <cstdio>
-#include <ctime>
-
namespace latinime {
-
// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE";
-const char *const HeaderPolicy::USES_FORGETTING_CURVE_KEY = "USES_FORGETTING_CURVE";
+// TODO: Change attribute string to "IS_DECAYING_DICT".
+const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE";
const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date";
+const char *const HeaderPolicy::UNIGRAM_COUNT_KEY = "UNIGRAM_COUNT";
+const char *const HeaderPolicy::BIGRAM_COUNT_KEY = "BIGRAM_COUNT";
+const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE";
const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100;
const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f;
@@ -55,33 +54,17 @@
}
float HeaderPolicy::readMultipleWordCostMultiplier() const {
- std::vector<int> keyVector;
- HeaderReadWriteUtils::insertCharactersIntoVector(MULTIPLE_WORDS_DEMOTION_RATE_KEY, &keyVector);
const int demotionRate = HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
- &keyVector, DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE);
+ MULTIPLE_WORDS_DEMOTION_RATE_KEY, DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE);
if (demotionRate <= 0) {
return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
}
return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(demotionRate);
}
-bool HeaderPolicy::readUsesForgettingCurveFlag() const {
- std::vector<int> keyVector;
- HeaderReadWriteUtils::insertCharactersIntoVector(USES_FORGETTING_CURVE_KEY, &keyVector);
- return HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, &keyVector,
- false /* defaultValue */);
-}
-
-// Returns current time when the key is not found or the value is invalid.
-int HeaderPolicy::readLastUpdatedTime() const {
- std::vector<int> keyVector;
- HeaderReadWriteUtils::insertCharactersIntoVector(LAST_UPDATED_TIME_KEY, &keyVector);
- return HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, &keyVector,
- time(0) /* defaultValue */);
-}
-
bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
- const bool updatesLastUpdatedTime) const {
+ const bool updatesLastUpdatedTime, const int unigramCount, const int bigramCount,
+ const int extendedRegionSize) const {
int writingPos = 0;
if (!HeaderReadWriteUtils::writeDictionaryVersion(bufferToWrite, mDictFormatVersion,
&writingPos)) {
@@ -97,21 +80,19 @@
&writingPos)) {
return false;
}
+ HeaderReadWriteUtils::AttributeMap attributeMapTowrite(mAttributeMap);
+ HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, UNIGRAM_COUNT_KEY, unigramCount);
+ HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, BIGRAM_COUNT_KEY, bigramCount);
+ HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, EXTENDED_REGION_SIZE_KEY,
+ extendedRegionSize);
if (updatesLastUpdatedTime) {
// Set current time as a last updated time.
- HeaderReadWriteUtils::AttributeMap attributeMapTowrite(mAttributeMap);
- std::vector<int> updatedTimekey;
- HeaderReadWriteUtils::insertCharactersIntoVector(LAST_UPDATED_TIME_KEY, &updatedTimekey);
- HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, &updatedTimekey, time(0));
- if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &attributeMapTowrite,
- &writingPos)) {
- return false;
- }
- } else {
- if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &mAttributeMap,
- &writingPos)) {
- return false;
- }
+ HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, LAST_UPDATED_TIME_KEY,
+ time(0));
+ }
+ if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &attributeMapTowrite,
+ &writingPos)) {
+ return false;
}
// Writes an actual header size.
if (!HeaderReadWriteUtils::writeDictionaryHeaderSize(bufferToWrite, writingPos,
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
index e97c08c..4261667 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
@@ -17,6 +17,7 @@
#ifndef LATINIME_HEADER_POLICY_H
#define LATINIME_HEADER_POLICY_H
+#include <ctime>
#include <stdint.h>
#include "defines.h"
@@ -35,8 +36,16 @@
mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)),
mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
- mUsesForgettingCurve(readUsesForgettingCurveFlag()),
- mLastUpdatedTime(readLastUpdatedTime()) {}
+ mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
+ IS_DECAYING_DICT_KEY, false /* defaultValue */)),
+ mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)),
+ mUnigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ UNIGRAM_COUNT_KEY, 0 /* defaultValue */)),
+ mBigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ BIGRAM_COUNT_KEY, 0 /* defaultValue */)),
+ mExtendedRegionSize(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ EXTENDED_REGION_SIZE_KEY, 0 /* defaultValue */)) {}
// Constructs header information using an attribute map.
HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion,
@@ -44,9 +53,12 @@
: mDictFormatVersion(dictFormatVersion),
mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
attributeMap)), mSize(0), mAttributeMap(*attributeMap),
- mMultiWordCostMultiplier(readUsesForgettingCurveFlag()),
- mUsesForgettingCurve(readUsesForgettingCurveFlag()),
- mLastUpdatedTime(readLastUpdatedTime()) {}
+ mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
+ mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
+ IS_DECAYING_DICT_KEY, false /* defaultValue */)),
+ mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)),
+ mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0) {}
~HeaderPolicy() {}
@@ -70,26 +82,42 @@
return mMultiWordCostMultiplier;
}
- AK_FORCE_INLINE bool usesForgettingCurve() const {
- return mUsesForgettingCurve;
+ AK_FORCE_INLINE bool isDecayingDict() const {
+ return mIsDecayingDict;
}
AK_FORCE_INLINE int getLastUpdatedTime() const {
return mLastUpdatedTime;
}
+ AK_FORCE_INLINE int getUnigramCount() const {
+ return mUnigramCount;
+ }
+
+ AK_FORCE_INLINE int getBigramCount() const {
+ return mBigramCount;
+ }
+
+ AK_FORCE_INLINE int getExtendedRegionSize() const {
+ return mExtendedRegionSize;
+ }
+
void readHeaderValueOrQuestionMark(const char *const key,
int *outValue, int outValueSize) const;
bool writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
- const bool updatesLastUpdatedTime) const;
+ const bool updatesLastUpdatedTime, const int unigramCount,
+ const int bigramCount, const int extendedRegionSize) const;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy);
static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
- static const char *const USES_FORGETTING_CURVE_KEY;
+ static const char *const IS_DECAYING_DICT_KEY;
static const char *const LAST_UPDATED_TIME_KEY;
+ static const char *const UNIGRAM_COUNT_KEY;
+ static const char *const BIGRAM_COUNT_KEY;
+ static const char *const EXTENDED_REGION_SIZE_KEY;
static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE;
static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE;
@@ -98,15 +126,14 @@
const int mSize;
HeaderReadWriteUtils::AttributeMap mAttributeMap;
const float mMultiWordCostMultiplier;
- const bool mUsesForgettingCurve;
+ const bool mIsDecayingDict;
const int mLastUpdatedTime;
+ const int mUnigramCount;
+ const int mBigramCount;
+ const int mExtendedRegionSize;
float readMultipleWordCostMultiplier() const;
- bool readUsesForgettingCurveFlag() const;
-
- int readLastUpdatedTime() const;
-
static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes(
const uint8_t *const dictBuf);
};
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
index 3b1c780..2694ce8 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
@@ -68,18 +68,12 @@
/* static */ HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
- AttributeMap::key_type key;
- insertCharactersIntoVector(REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, &key);
- const bool requiresGermanUmlautProcessing = readBoolAttributeValue(attributeMap, &key,
- false /* defaultValue */);
- key.clear();
- insertCharactersIntoVector(REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY, &key);
- const bool requiresFrenchLigatureProcessing = readBoolAttributeValue(attributeMap, &key,
- false /* defaultValue */);
- key.clear();
- insertCharactersIntoVector(SUPPORTS_DYNAMIC_UPDATE_KEY, &key);
- const bool supportsDynamicUpdate = readBoolAttributeValue(attributeMap, &key,
- false /* defaultValue */);
+ const bool requiresGermanUmlautProcessing = readBoolAttributeValue(attributeMap,
+ REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false /* defaultValue */);
+ const bool requiresFrenchLigatureProcessing = readBoolAttributeValue(attributeMap,
+ REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY, false /* defaultValue */);
+ const bool supportsDynamicUpdate = readBoolAttributeValue(attributeMap,
+ SUPPORTS_DYNAMIC_UPDATE_KEY, false /* defaultValue */);
DictionaryFlags dictflags = NO_FLAGS;
dictflags |= requiresGermanUmlautProcessing ? GERMAN_UMLAUT_PROCESSING_FLAG : 0;
dictflags |= requiresFrenchLigatureProcessing ? FRENCH_LIGATURE_PROCESSING_FLAG : 0;
@@ -160,11 +154,18 @@
}
/* static */ void HeaderReadWriteUtils::setBoolAttribute(AttributeMap *const headerAttributes,
- const AttributeMap::key_type *const key, const bool value) {
+ const char *const key, const bool value) {
setIntAttribute(headerAttributes, key, value ? 1 : 0);
}
/* static */ void HeaderReadWriteUtils::setIntAttribute(AttributeMap *const headerAttributes,
+ const char *const key, const int value) {
+ AttributeMap::key_type keyVector;
+ insertCharactersIntoVector(key, &keyVector);
+ setIntAttributeInner(headerAttributes, &keyVector, value);
+}
+
+/* static */ void HeaderReadWriteUtils::setIntAttributeInner(AttributeMap *const headerAttributes,
const AttributeMap::key_type *const key, const int value) {
AttributeMap::mapped_type valueVector;
char charBuf[LARGEST_INT_DIGIT_COUNT + 1];
@@ -174,7 +175,7 @@
}
/* static */ bool HeaderReadWriteUtils::readBoolAttributeValue(
- const AttributeMap *const headerAttributes, const AttributeMap::key_type *const key,
+ const AttributeMap *const headerAttributes, const char *const key,
const bool defaultValue) {
const int intDefaultValue = defaultValue ? 1 : 0;
const int intValue = readIntAttributeValue(headerAttributes, key, intDefaultValue);
@@ -182,6 +183,14 @@
}
/* static */ int HeaderReadWriteUtils::readIntAttributeValue(
+ const AttributeMap *const headerAttributes, const char *const key,
+ const int defaultValue) {
+ AttributeMap::key_type keyVector;
+ insertCharactersIntoVector(key, &keyVector);
+ return readIntAttributeValueInner(headerAttributes, &keyVector, defaultValue);
+}
+
+/* static */ int HeaderReadWriteUtils::readIntAttributeValueInner(
const AttributeMap *const headerAttributes, const AttributeMap::key_type *const key,
const int defaultValue) {
AttributeMap::const_iterator it = headerAttributes->find(*key);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h
index caa5097..2259683 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h
@@ -76,16 +76,16 @@
* Methods for header attributes.
*/
static void setBoolAttribute(AttributeMap *const headerAttributes,
- const AttributeMap::key_type *const key, const bool value);
+ const char *const key, const bool value);
static void setIntAttribute(AttributeMap *const headerAttributes,
- const AttributeMap::key_type *const key, const int value);
+ const char *const key, const int value);
static bool readBoolAttributeValue(const AttributeMap *const headerAttributes,
- const AttributeMap::key_type *const key, const bool defaultValue);
+ const char *const key, const bool defaultValue);
static int readIntAttributeValue(const AttributeMap *const headerAttributes,
- const AttributeMap::key_type *const key, const int defaultValue);
+ const char *const key, const int defaultValue);
static void insertCharactersIntoVector(const char *const characters,
AttributeMap::key_type *const key);
@@ -112,6 +112,12 @@
static const char *const SUPPORTS_DYNAMIC_UPDATE_KEY;
static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY;
static const char *const REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY;
+
+ static void setIntAttributeInner(AttributeMap *const headerAttributes,
+ const AttributeMap::key_type *const key, const int value);
+
+ static int readIntAttributeValueInner(const AttributeMap *const headerAttributes,
+ const AttributeMap::key_type *const key, const int defaultValue);
};
}
#endif /* LATINIME_HEADER_READ_WRITE_UTILS_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
index f1de914..8d88c68 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
@@ -107,12 +107,20 @@
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
}
- bool needsToRunGC() const {
+ bool needsToRunGC(const bool mindsBlockByGC) const {
// This method should not be called for non-updatable dictionary.
AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
return false;
}
+ void getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const {
+ // getProperty is not supported for this class.
+ if (maxResultLength > 0) {
+ outResult[0] = '\0';
+ }
+ }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
index 17d2e39..9dc3482 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
@@ -42,6 +42,10 @@
return mOriginalBufferSize + mUsedAdditionalBufferSize;
}
+ AK_FORCE_INLINE int getUsedAdditionalBufferSize() const {
+ return mUsedAdditionalBufferSize;
+ }
+
/**
* For reading.
*/
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
index 2e4ec2e..f22e94c 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
@@ -43,7 +43,8 @@
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
HeaderPolicy headerPolicy(FormatUtils::VERSION_3, attributeMap);
- headerPolicy.writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */);
+ headerPolicy.writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */,
+ 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);
BufferWithExtendableBuffer bodyBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(&bodyBuffer, 0 /* rootPos */)) {
return false;
@@ -95,7 +96,7 @@
fclose(file);
return false;
}
- const int additionalBufSize = buffer->getTailPosition() - buffer->getOriginalBufferSize();
+ const int additionalBufSize = buffer->getUsedAdditionalBufferSize();
if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */),
additionalBufSize, 1, file) < 1) {
fclose(file);
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
index 7ed3ee1..6a21522 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
@@ -27,6 +27,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
@@ -605,7 +606,7 @@
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- while(!binaryDictionary.needsToRunGC()) {
+ while(!binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final String word = CodePointUtils.generateWord(random, codePointSet);
words.add(word);
final int unigramProbability = random.nextInt(0xFF);
@@ -625,4 +626,57 @@
dictFile.delete();
}
+
+ public void testUnigramAndBigramCount() {
+ final int flashWithGCIterationCount = 10;
+ final int codePointSetSize = 50;
+ final int unigramCountPerIteration = 1000;
+ final int bigramCountPerIteration = 2000;
+ final int seed = 1123581321;
+
+ final Random random = new Random(seed);
+
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ }
+
+ final ArrayList<String> words = new ArrayList<String>();
+ final HashSet<Pair<String, String>> bigrams = new HashSet<Pair<String, String>>();
+ final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
+
+ BinaryDictionary binaryDictionary;
+ for (int i = 0; i < flashWithGCIterationCount; i++) {
+ binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ for (int j = 0; j < unigramCountPerIteration; j++) {
+ final String word = CodePointUtils.generateWord(random, codePointSet);
+ words.add(word);
+ final int unigramProbability = random.nextInt(0xFF);
+ binaryDictionary.addUnigramWord(word, unigramProbability);
+ }
+ for (int j = 0; j < bigramCountPerIteration; j++) {
+ final String word0 = words.get(random.nextInt(words.size()));
+ final String word1 = words.get(random.nextInt(words.size()));
+ bigrams.add(new Pair<String, String>(word0, word1));
+ final int bigramProbability = random.nextInt(0xF);
+ binaryDictionary.addBigramWords(word0, word1, bigramProbability);
+ }
+ assertEquals(new HashSet<String>(words).size(), Integer.parseInt(
+ binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
+ assertEquals(new HashSet<Pair<String, String>>(bigrams).size(), Integer.parseInt(
+ binaryDictionary.getPropertyForTests(BinaryDictionary.BIGRAM_COUNT_QUERY)));
+ binaryDictionary.flushWithGC();
+ assertEquals(new HashSet<String>(words).size(), Integer.parseInt(
+ binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
+ assertEquals(new HashSet<Pair<String, String>>(bigrams).size(), Integer.parseInt(
+ binaryDictionary.getPropertyForTests(BinaryDictionary.BIGRAM_COUNT_QUERY)));
+ binaryDictionary.close();
+ }
+
+ dictFile.delete();
+ }
}
diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
index d605cdb..ddc9546 100644
--- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
@@ -75,38 +75,33 @@
return new ArrayList<String>(wordSet);
}
- private void addToDict(final UserHistoryPredictionDictionary dict, final List<String> words) {
+ private void addToDict(final UserHistoryDictionary dict, final List<String> words) {
String prevWord = null;
for (String word : words) {
- dict.addToPersonalizationPredictionDictionary(prevWord, word, true);
+ dict.addToDictionary(prevWord, word, true);
prevWord = word;
}
}
/**
- * @param checksContents if true, checks whether written words are actually in the dictionary
+ * @param checkContents if true, checks whether written words are actually in the dictionary
* or not.
*/
private void addAndWriteRandomWords(final String testFilenameSuffix, final int numberOfWords,
- final Random random, final boolean checksContents) {
+ final Random random, final boolean checkContents) {
final List<String> words = generateWords(numberOfWords, random);
- final UserHistoryPredictionDictionary dict =
- PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+ final UserHistoryDictionary dict =
+ PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffix /* locale */, mPrefs);
// Add random words to the user history dictionary.
addToDict(dict, words);
- if (checksContents) {
+ if (checkContents) {
try {
Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS));
} catch (InterruptedException e) {
}
- // Limit word count to check when using a Java on memory dictionary.
- final int wordCountToCheck =
- ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
- numberOfWords : 10;
- for (int i = 0; i < wordCountToCheck; ++i) {
+ for (int i = 0; i < numberOfWords; ++i) {
final String word = words.get(i);
- // This may fail as long as we use tryLock on inserting the bigram words
assertTrue(dict.isInDictionaryForTests(word));
}
}
@@ -127,8 +122,8 @@
true /* checksContents */);
} finally {
try {
- final UserHistoryPredictionDictionary dict =
- PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+ final UserHistoryDictionary dict =
+ PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffix, mPrefs);
Log.d(TAG, "waiting for writing ...");
dict.shutdownExecutorForTests();
@@ -139,7 +134,7 @@
Log.d(TAG, "InterruptedException: " + e);
}
- final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
+ final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFile = new File(getContext().getFilesDir(), fileName);
@@ -164,7 +159,7 @@
// Create filename suffixes for this test.
for (int i = 0; i < numberOfLanguages; i++) {
testFilenameSuffixes[i] = "testSwitchingLanguages" + i;
- final String fileName = UserHistoryPredictionDictionary.NAME + "." +
+ final String fileName = UserHistoryDictionary.NAME + "." +
testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFiles[i] = new File(getContext().getFilesDir(), fileName);
}
@@ -186,8 +181,8 @@
try {
Log.d(TAG, "waiting for writing ...");
for (int i = 0; i < numberOfLanguages; i++) {
- final UserHistoryPredictionDictionary dict =
- PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+ final UserHistoryDictionary dict =
+ PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffixes[i], mPrefs);
dict.shutdownExecutorForTests();
while (!dict.isTerminatedForTests()) {
@@ -215,8 +210,8 @@
10000 : 1000;
final Random random = new Random(123456);
- UserHistoryPredictionDictionary dict =
- PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+ UserHistoryDictionary dict =
+ PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffix, mPrefs);
try {
addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random,
@@ -232,7 +227,7 @@
} catch (InterruptedException e) {
Log.d(TAG, "InterruptedException: ", e);
}
- final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
+ final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFile = new File(getContext().getFilesDir(), fileName);
if (dictFile != null) {
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
index 6c4cbcf..bd06e9f 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
@@ -80,17 +80,17 @@
}
/**
- * Returns a decrypted/uncompressed binary dictionary.
+ * Returns a decrypted/uncompressed dictionary.
*
- * This will decrypt/uncompress any number of times as necessary until it finds the binary
+ * This will decrypt/uncompress any number of times as necessary until it finds the
* dictionary signature, and copy the decoded file to a temporary place.
- * If this is not a binary dictionary, the method returns null.
+ * If this is not a dictionary, the method returns null.
*/
- public static DecoderChainSpec getRawBinaryDictionaryOrNull(final File src) {
- return getRawBinaryDictionaryOrNullInternal(new DecoderChainSpec(), src, 0);
+ public static DecoderChainSpec getRawDictionaryOrNull(final File src) {
+ return getRawDictionaryOrNullInternal(new DecoderChainSpec(), src, 0);
}
- private static DecoderChainSpec getRawBinaryDictionaryOrNullInternal(
+ private static DecoderChainSpec getRawDictionaryOrNullInternal(
final DecoderChainSpec spec, final File src, final int depth) {
// Unfortunately the decoding scheme we use can consider any data to be encrypted
// and will product some output, meaning it's not possible to reliably detect encrypted
@@ -98,7 +98,8 @@
// over and over, ending in a stack overflow. Hence we limit the depth at which we try
// decoding the file.
if (depth > MAX_DECODE_DEPTH) return null;
- if (BinaryDictDecoderUtils.isBinaryDictionary(src)) {
+ if (BinaryDictDecoderUtils.isBinaryDictionary(src)
+ || CombinedInputOutput.isCombinedDictionary(src.getAbsolutePath())) {
spec.mFile = src;
return spec;
}
@@ -106,7 +107,7 @@
final File uncompressedFile = tryGetUncompressedFile(src);
if (null != uncompressedFile) {
final DecoderChainSpec newSpec =
- getRawBinaryDictionaryOrNullInternal(spec, uncompressedFile, depth + 1);
+ getRawDictionaryOrNullInternal(spec, uncompressedFile, depth + 1);
if (null == newSpec) return null;
return newSpec.addStep(COMPRESSION);
}
@@ -114,7 +115,7 @@
final File decryptedFile = tryGetDecryptedFile(src);
if (null != decryptedFile) {
final DecoderChainSpec newSpec =
- getRawBinaryDictionaryOrNullInternal(spec, decryptedFile, depth + 1);
+ getRawDictionaryOrNullInternal(spec, decryptedFile, depth + 1);
if (null == newSpec) return null;
return newSpec.addStep(ENCRYPTION);
}
@@ -175,15 +176,16 @@
return XmlDictInputOutput.readDictionaryXml(
new BufferedInputStream(new FileInputStream(file)),
null /* shortcuts */, null /* bigrams */);
- } else if (CombinedInputOutput.isCombinedDictionary(filename)) {
- if (report) System.out.println("Format : Combined format");
- return CombinedInputOutput.readDictionaryCombined(
- new BufferedInputStream(new FileInputStream(file)));
} else {
- final DecoderChainSpec decodedSpec = getRawBinaryDictionaryOrNull(file);
+ final DecoderChainSpec decodedSpec = getRawDictionaryOrNull(file);
if (null == decodedSpec) {
crash(filename, new RuntimeException(
filename + " does not seem to be a dictionary file"));
+ } else if (CombinedInputOutput.isCombinedDictionary(
+ decodedSpec.mFile.getAbsolutePath())){
+ if (report) System.out.println("Format : Combined format");
+ return CombinedInputOutput.readDictionaryCombined(
+ new BufferedInputStream(new FileInputStream(decodedSpec.mFile)));
} else {
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile,
DictDecoder.USE_BYTEARRAY);
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java
index 9274dcd..dff3387 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java
@@ -79,7 +79,7 @@
throw new RuntimeException("Too many/too few arguments for command " + COMMAND);
}
final BinaryDictOffdeviceUtils.DecoderChainSpec decodedSpec =
- BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(new File(mArgs[0]));
+ BinaryDictOffdeviceUtils.getRawDictionaryOrNull(new File(mArgs[0]));
if (null == decodedSpec) {
System.out.println(mArgs[0] + " does not seem to be a dictionary");
return;
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
index 1eff497..1baeb7a 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
@@ -64,7 +64,7 @@
// Test for an actually compressed dictionary and its contents
final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec =
- BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(dst);
+ BinaryDictOffdeviceUtils.getRawDictionaryOrNull(dst);
for (final String step : decodeSpec.mDecoderSpec) {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
}
@@ -90,7 +90,7 @@
// Test that a random data file actually fails
assertNull("Wrongly identified data file",
- BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(dst));
+ BinaryDictOffdeviceUtils.getRawDictionaryOrNull(dst));
final File gzDst = File.createTempFile("testGetRawDict", ".tmp");
gzDst.deleteOnExit();
@@ -103,6 +103,6 @@
// Test that a compressed random data file actually fails
assertNull("Wrongly identified data file",
- BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(gzDst));
+ BinaryDictOffdeviceUtils.getRawDictionaryOrNull(gzDst));
}
}