Merge "Import translations. DO NOT MERGE"
diff --git a/java/res/drawable-hdpi/btn_keyboard_key_popup_selected_holo.9.png b/java/res/drawable-hdpi/btn_keyboard_key_popup_selected_holo.9.png
index de14931..10f8e97 100644
--- a/java/res/drawable-hdpi/btn_keyboard_key_popup_selected_holo.9.png
+++ b/java/res/drawable-hdpi/btn_keyboard_key_popup_selected_holo.9.png
Binary files differ
diff --git a/java/res/drawable-mdpi/btn_keyboard_key_popup_selected_holo.9.png b/java/res/drawable-mdpi/btn_keyboard_key_popup_selected_holo.9.png
index 5b202e2..ee0aae2 100644
--- a/java/res/drawable-mdpi/btn_keyboard_key_popup_selected_holo.9.png
+++ b/java/res/drawable-mdpi/btn_keyboard_key_popup_selected_holo.9.png
Binary files differ
diff --git a/java/res/drawable-xhdpi/btn_keyboard_key_popup_selected_holo.9.png b/java/res/drawable-xhdpi/btn_keyboard_key_popup_selected_holo.9.png
index af9187e..891d000 100644
--- a/java/res/drawable-xhdpi/btn_keyboard_key_popup_selected_holo.9.png
+++ b/java/res/drawable-xhdpi/btn_keyboard_key_popup_selected_holo.9.png
Binary files differ
diff --git a/java/res/drawable-xxhdpi/btn_keyboard_key_popup_selected_holo.9.png b/java/res/drawable-xxhdpi/btn_keyboard_key_popup_selected_holo.9.png
index c62051f..0cbb2ec 100644
--- a/java/res/drawable-xxhdpi/btn_keyboard_key_popup_selected_holo.9.png
+++ b/java/res/drawable-xxhdpi/btn_keyboard_key_popup_selected_holo.9.png
Binary files differ
diff --git a/java/res/layout/emoji_keyboard_view.xml b/java/res/layout/emoji_keyboard_view.xml
index 6a953a7..4566a5a 100644
--- a/java/res/layout/emoji_keyboard_view.xml
+++ b/java/res/layout/emoji_keyboard_view.xml
@@ -40,7 +40,7 @@
<TabWidget
android:id="@android:id/tabs"
android:layout_width="match_parent"
- android:layout_height="wrap_content"
+ android:layout_height="match_parent"
android:background="@drawable/tab_selected"
android:divider="@null"
android:tabStripEnabled="true"
@@ -61,11 +61,16 @@
android:visibility="gone" />
</FrameLayout>
</TabHost>
+ <View
+ android:layout_width="2dip"
+ android:layout_height="match_parent"
+ android:background="@drawable/suggestions_strip_divider" />
<ImageButton
android:id="@+id/emoji_keyboard_delete"
android:layout_width="0dip"
android:layout_weight="12.5"
android:layout_height="match_parent"
+ android:background="@color/emoji_key_background_color"
android:src="@drawable/sym_keyboard_delete_holo_dark" />
</LinearLayout>
<android.support.v4.view.ViewPager
diff --git a/java/res/values-hy/donottranslate.xml b/java/res/values-hy/donottranslate.xml
new file mode 100644
index 0000000..4a6d188
--- /dev/null
+++ b/java/res/values-hy/donottranslate.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+/*
+**
+** Copyright 2013, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+-->
+<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <!-- Same list as in English, but add armenian period and comma: -->
+ <!-- U+055D: "՝" ARMENIAN COMMA -->
+ <!-- U+0589: "։" ARMENIAN FULL STOP -->
+ <!-- Symbols that are normally followed by a space (used to add an auto-space after these) -->
+ <string name="symbols_followed_by_space">.,;:!?)]}&։՝</string>
+ <!-- Symbols that separate words. Adding armenian period and comma. -->
+ <!-- Don't remove the enclosing double quotes, they protect whitespace (not just U+0020) -->
+ <string name="symbols_word_separators">"	 \n"()[]{}*&<>+=|.,;:!?/_\"։՝</string>
+</resources>
diff --git a/java/res/values/colors.xml b/java/res/values/colors.xml
index ea7400d..3803cb7 100644
--- a/java/res/values/colors.xml
+++ b/java/res/values/colors.xml
@@ -62,4 +62,8 @@
<color name="setup_welcome_video_margin_color">#FFCCCCCC</color>
<color name="emoji_category_page_id_view_background">#FF000000</color>
<color name="emoji_category_page_id_view_foreground">#80FFFFFF</color>
+
+ <!-- TODO: Color which should be included in the theme -->
+ <color name="emoji_key_background_color">#00000000</color>
+ <color name="emoji_key_pressed_background_color">#30FFFFFF</color>
</resources>
diff --git a/java/res/values/keypress-vibration-durations.xml b/java/res/values/keypress-vibration-durations.xml
index 53448c3..ee0ac00 100644
--- a/java/res/values/keypress-vibration-durations.xml
+++ b/java/res/values/keypress-vibration-durations.xml
@@ -55,8 +55,8 @@
<item>MODEL=HTL22:MANUFACTURER=HTC,15</item>
<!-- Motorola Razor M -->
<item>MODEL=XT907:MANUFACTURER=motorola,30</item>
- <!-- Sony Xperia Z -->
- <item>MODEL=C6603:MANUFACTURER=Sony,35</item>
+ <!-- Sony Xperia Z, Z Ultra -->
+ <item>MODEL=C6603|C6806:MANUFACTURER=Sony,35</item>
<!-- Default value for unknown device. The negative value means system default. -->
<item>,-1</item>
</string-array>
diff --git a/java/res/xml/key_styles_currency.xml b/java/res/xml/key_styles_currency.xml
index b7677a2..84c2abc 100644
--- a/java/res/xml/key_styles_currency.xml
+++ b/java/res/xml/key_styles_currency.xml
@@ -103,6 +103,8 @@
vi: Vietnamese (Dong) -->
<!-- TODO: The currency sign of Turkish Lira was created in 2012 and assigned U+20BA for
its unicode, although there is no font glyph for it as of November 2012. -->
+ <!-- TODO: The currency sign of Armenian Dram was created in 2012 and assigned U+058F for
+ its unicode, although there is no font glyph for it as of September 2013. -->
<case
latin:languageCode="fa|hi|iw|lo|mn|ne|th|uk|vi"
>
diff --git a/java/res/xml/keys_comma_period.xml b/java/res/xml/keys_comma_period.xml
index 7e7c728..02b46c2 100644
--- a/java/res/xml/keys_comma_period.xml
+++ b/java/res/xml/keys_comma_period.xml
@@ -73,6 +73,20 @@
latin:backgroundType="functional"
latin:keyStyle="hasShiftedLetterHintStyle" />
</case>
+ <case
+ latin:languageCode="hy"
+ >
+ <!-- U+0589: "։" ARMENIAN FULL STOP -->
+ <Key
+ latin:keyLabel="։"
+ latin:keyLabelFlags="hasPopupHint"
+ latin:backgroundType="functional"
+ latin:moreKeys="!text/more_keys_for_punctuation" />
+ <!-- U+055D: "՝" ARMENIAN COMMA -->
+ <Key
+ latin:keyLabel="՝"
+ latin:backgroundType="functional" />
+ </case>
<default>
<Key
latin:keyLabel="."
diff --git a/java/res/xml/row_qwerty4.xml b/java/res/xml/row_qwerty4.xml
index 340beb9..578bc12 100644
--- a/java/res/xml/row_qwerty4.xml
+++ b/java/res/xml/row_qwerty4.xml
@@ -49,6 +49,14 @@
<include
latin:keyboardLayout="@xml/key_nepali_traditional_period" />
</case>
+ <case
+ latin:languageCode="hy"
+ >
+ <!-- U+0589: "։" ARMENIAN FULL STOP -->
+ <Key
+ latin:keyLabel="։"
+ latin:keyStyle="punctuationKeyStyle" />
+ </case>
<default>
<Key
latin:keyStyle="punctuationKeyStyle" />
diff --git a/java/src/com/android/inputmethod/compat/ActivityManagerCompatUtils.java b/java/src/com/android/inputmethod/compat/ActivityManagerCompatUtils.java
new file mode 100644
index 0000000..385e3e0
--- /dev/null
+++ b/java/src/com/android/inputmethod/compat/ActivityManagerCompatUtils.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.compat;
+
+import android.app.ActivityManager;
+import android.content.Context;
+
+import java.lang.reflect.Method;
+
+public class ActivityManagerCompatUtils {
+ private static final Object LOCK = new Object();
+ private static volatile Boolean sBoolean = null;
+ private static final Method METHOD_isLowRamDevice = CompatUtils.getMethod(
+ ActivityManager.class, "isLowRamDevice");
+
+ private ActivityManagerCompatUtils() {
+ // Do not instantiate this class.
+ }
+
+ public static boolean isLowRamDevice(Context context) {
+ if (sBoolean == null) {
+ synchronized(LOCK) {
+ if (sBoolean == null) {
+ final ActivityManager am =
+ (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE);
+ sBoolean = (Boolean)CompatUtils.invoke(am, false, METHOD_isLowRamDevice);
+ }
+ }
+ }
+ return sBoolean;
+ }
+}
diff --git a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
index db65de2..4e61eda 100644
--- a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
+++ b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
@@ -28,11 +28,13 @@
import android.preference.PreferenceManager;
import android.support.v4.view.PagerAdapter;
import android.support.v4.view.ViewPager;
+import android.text.format.DateUtils;
import android.util.AttributeSet;
import android.util.Log;
import android.util.Pair;
import android.util.SparseArray;
import android.view.LayoutInflater;
+import android.view.MotionEvent;
import android.view.View;
import android.view.ViewGroup;
import android.widget.ImageView;
@@ -76,6 +78,7 @@
private final int mEmojiFunctionalKeyBackgroundId;
private final KeyboardLayoutSet mLayoutSet;
private final ColorStateList mTabLabelColor;
+ private final DeleteKeyOnTouchListener mDeleteKeyOnTouchListener;
private EmojiKeyboardAdapter mEmojiKeyboardAdapter;
private TabHost mTabHost;
@@ -203,7 +206,14 @@
}
public int getCategoryPageSize(int categoryId) {
- return mShownCategories.get(categoryId).mPageCount;
+ for (final CategoryProperties prop : mShownCategories) {
+ if (prop.mCategoryId == categoryId) {
+ return prop.mPageCount;
+ }
+ }
+ Log.w(TAG, "Invalid category id: " + categoryId);
+ // Should not reach here.
+ return 0;
}
public void setCurrentCategoryId(int categoryId) {
@@ -395,6 +405,7 @@
mLayoutSet = builder.build();
mEmojiCategory = new EmojiCategory(PreferenceManager.getDefaultSharedPreferences(context),
context.getResources(), builder.build());
+ mDeleteKeyOnTouchListener = new DeleteKeyOnTouchListener(context);
}
@Override
@@ -459,11 +470,9 @@
final LinearLayout actionBar = (LinearLayout)findViewById(R.id.emoji_action_bar);
emojiLp.setActionBarProperties(actionBar);
- // TODO: Implement auto repeat, using View.OnTouchListener?
final ImageView deleteKey = (ImageView)findViewById(R.id.emoji_keyboard_delete);
- deleteKey.setBackgroundResource(mEmojiFunctionalKeyBackgroundId);
deleteKey.setTag(Constants.CODE_DELETE);
- deleteKey.setOnClickListener(this);
+ deleteKey.setOnTouchListener(mDeleteKeyOnTouchListener);
final ImageView alphabetKey = (ImageView)findViewById(R.id.emoji_keyboard_alphabet);
alphabetKey.setBackgroundResource(mEmojiFunctionalKeyBackgroundId);
alphabetKey.setTag(Constants.CODE_SWITCH_ALPHA_SYMBOL);
@@ -556,6 +565,7 @@
public void setKeyboardActionListener(final KeyboardActionListener listener) {
mKeyboardActionListener = listener;
+ mDeleteKeyOnTouchListener.setKeyboardActionListener(mKeyboardActionListener);
}
private void updateEmojiCategoryPageIdView() {
@@ -665,4 +675,92 @@
container.removeView(keyboardView);
}
}
+
+ // TODO: Do the same things done in PointerTracker
+ private static class DeleteKeyOnTouchListener implements OnTouchListener {
+ private static final long MAX_REPEAT_COUNT_TIME = 30 * DateUtils.SECOND_IN_MILLIS;
+ private final int mDeleteKeyPressedBackgroundColor;
+ private final long mKeyRepeatStartTimeout;
+ private final long mKeyRepeatInterval;
+
+ public DeleteKeyOnTouchListener(Context context) {
+ final Resources res = context.getResources();
+ mDeleteKeyPressedBackgroundColor =
+ res.getColor(R.color.emoji_key_pressed_background_color);
+ mKeyRepeatStartTimeout = res.getInteger(R.integer.config_key_repeat_start_timeout);
+ mKeyRepeatInterval = res.getInteger(R.integer.config_key_repeat_interval);
+ }
+
+ private KeyboardActionListener mKeyboardActionListener =
+ KeyboardActionListener.EMPTY_LISTENER;
+ private DummyRepeatKeyRepeatTimer mTimer;
+
+ private synchronized void startRepeat() {
+ if (mTimer != null) {
+ abortRepeat();
+ }
+ mTimer = new DummyRepeatKeyRepeatTimer();
+ mTimer.start();
+ }
+
+ private synchronized void abortRepeat() {
+ mTimer.abort();
+ mTimer = null;
+ }
+
+ // TODO: Remove
+ // This function is mimicking the repeat code in PointerTracker.
+ // Specifically referring to PointerTracker#startRepeatKey and PointerTracker#onKeyRepeat.
+ private class DummyRepeatKeyRepeatTimer extends Thread {
+ public boolean mAborted = false;
+
+ @Override
+ public void run() {
+ int timeCount = 0;
+ while (timeCount < MAX_REPEAT_COUNT_TIME && !mAborted) {
+ if (timeCount > mKeyRepeatStartTimeout) {
+ pressDelete();
+ }
+ timeCount += mKeyRepeatInterval;
+ try {
+ Thread.sleep(mKeyRepeatInterval);
+ } catch (InterruptedException e) {
+ }
+ }
+ }
+
+ public void abort() {
+ mAborted = true;
+ }
+ }
+
+ public void pressDelete() {
+ mKeyboardActionListener.onPressKey(
+ Constants.CODE_DELETE, 0 /* repeatCount */, true /* isSinglePointer */);
+ mKeyboardActionListener.onCodeInput(
+ Constants.CODE_DELETE, NOT_A_COORDINATE, NOT_A_COORDINATE);
+ mKeyboardActionListener.onReleaseKey(
+ Constants.CODE_DELETE, false /* withSliding */);
+ }
+
+ public void setKeyboardActionListener(KeyboardActionListener listener) {
+ mKeyboardActionListener = listener;
+ }
+
+ @Override
+ public boolean onTouch(View v, MotionEvent event) {
+ switch(event.getAction()) {
+ case MotionEvent.ACTION_DOWN:
+ v.setBackgroundColor(mDeleteKeyPressedBackgroundColor);
+ pressDelete();
+ startRepeat();
+ return true;
+ case MotionEvent.ACTION_UP:
+ v.setBackgroundColor(0);
+ abortRepeat();
+ return true;
+ }
+ return false;
+ }
+ }
}
diff --git a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java
index f8ad43e..13db470 100644
--- a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java
+++ b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java
@@ -155,7 +155,6 @@
private final SlidingKeyInputPreview mSlidingKeyInputPreview;
// Key preview
- private static final int PREVIEW_ALPHA = 240;
private final int mKeyPreviewLayoutId;
private final int mKeyPreviewOffset;
private final int mKeyPreviewHeight;
@@ -816,7 +815,6 @@
if (background != null) {
final int hasMoreKeys = (key.getMoreKeys() != null) ? STATE_HAS_MOREKEYS : STATE_NORMAL;
background.setState(KEY_PREVIEW_BACKGROUND_STATE_TABLE[statePosition][hasMoreKeys]);
- background.setAlpha(PREVIEW_ALPHA);
}
ViewLayoutUtils.placeViewAt(
previewText, previewX, previewY, previewWidth, previewHeight);
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java
index 2af2f69..67553fb 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java
@@ -1769,15 +1769,27 @@
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null,
/* ~52 */
+ // U+058A: "֊" ARMENIAN HYPHEN
+ // U+055C: "՜" ARMENIAN EXCLAMATION MARK
+ // U+055D: "՝" ARMENIAN COMMA
// U+055E: "՞" ARMENIAN QUESTION MARK
- /* 53 */ "!fixedColumnOrder!4,\u055E,!,\\,,?,:,;,@",
+ // U+0559: "ՙ" ARMENIAN MODIFIER LETTER LEFT HALF RING
+ // U+055A: "՚" ARMENIAN APOSTROPHE
+ // U+055B: "՛" ARMENIAN EMPHASIS MARK
+ // U+055F: "՟" ARMENIAN ABBREVIATION MARK
+ /* 53 */ "!fixedColumnOrder!8,!,?,\\,,.,\u058A,\u055C,\u055D,\u055E,:,;,@,\u0559,\u055A,\u055B,\u055F",
/* 54~ */
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
- null, null, null, null, null, null, null, null, null,
- /* ~107 */
- /* 108 */ "\u055E,?",
+ null,
+ /* ~99 */
+ // U+055C: "՜" ARMENIAN EXCLAMATION MARK
+ // U+00A1: "¡" INVERTED EXCLAMATION MARK
+ /* 100 */ "\u055C,\u00A1",
+ // U+055E: "՞" ARMENIAN QUESTION MARK
+ // U+00BF: "¿" INVERTED QUESTION MARK
+ /* 101 */ "\u055E,\u00BF",
};
/* Language is: Icelandic */
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index 632ee0d..61ccfcf 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -27,6 +27,7 @@
import com.android.inputmethod.latin.utils.JniUtils;
import com.android.inputmethod.latin.utils.StringUtils;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Locale;
@@ -244,11 +245,18 @@
return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
}
+ private void runGCIfRequired() {
+ if (needsToRunGCNative(mNativeDict)) {
+ flushWithGC();
+ }
+ }
+
// Add a unigram entry to binary dictionary in native code.
public void addUnigramWord(final String word, final int probability) {
if (TextUtils.isEmpty(word)) {
return;
}
+ runGCIfRequired();
final int[] codePoints = StringUtils.toCodePointArray(word);
addUnigramWordNative(mNativeDict, codePoints, probability);
}
@@ -258,6 +266,7 @@
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return;
}
+ runGCIfRequired();
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability);
@@ -268,24 +277,30 @@
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return;
}
+ runGCIfRequired();
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
removeBigramWordsNative(mNativeDict, codePoints0, codePoints1);
}
- @UsedForTesting
public void flush() {
if (!isValidDictionary()) return;
flushNative(mNativeDict, mDictFilePath);
+ closeNative(mNativeDict);
+ final File dictFile = new File(mDictFilePath);
+ mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
+ dictFile.length(), true /* isUpdatable */);
}
- @UsedForTesting
public void flushWithGC() {
if (!isValidDictionary()) return;
flushWithGCNative(mNativeDict, mDictFilePath);
+ closeNative(mNativeDict);
+ final File dictFile = new File(mDictFilePath);
+ mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
+ dictFile.length(), true /* isUpdatable */);
}
- @UsedForTesting
public boolean needsToRunGC() {
if (!isValidDictionary()) return false;
return needsToRunGCNative(mNativeDict);
diff --git a/java/src/com/android/inputmethod/latin/Constants.java b/java/src/com/android/inputmethod/latin/Constants.java
index 28d9e86..c4f9601 100644
--- a/java/src/com/android/inputmethod/latin/Constants.java
+++ b/java/src/com/android/inputmethod/latin/Constants.java
@@ -138,6 +138,9 @@
public static final int SPELL_CHECKER_COORDINATE = -3;
public static final int EXTERNAL_KEYBOARD_COORDINATE = -4;
+ // A hint on how many characters to cache from the TextView. A good value of this is given by
+ // how many characters we need to be able to almost always find the caps mode.
+ public static final int EDITOR_CONTENTS_CACHE_SIZE = 1024;
// Must be equal to MAX_WORD_LENGTH in native/jni/src/defines.h
public static final int DICTIONARY_MAX_WORD_LENGTH = 48;
@@ -162,6 +165,7 @@
public static final int CODE_TAB = '\t';
public static final int CODE_SPACE = ' ';
public static final int CODE_PERIOD = '.';
+ public static final int CODE_ARMENIAN_PERIOD = 0x0589;
public static final int CODE_DASH = '-';
public static final int CODE_SINGLE_QUOTE = '\'';
public static final int CODE_DOUBLE_QUOTE = '"';
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index fcd7ede..0774ce2 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -22,14 +22,22 @@
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
-import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
+import com.android.inputmethod.latin.makedict.DictEncoder;
+import com.android.inputmethod.latin.makedict.FormatSpec;
+import com.android.inputmethod.latin.makedict.FusionDictionary;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
+import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
+import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
+import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
@@ -49,9 +57,9 @@
/** Whether to print debug output to log */
private static boolean DEBUG = false;
- // TODO: Remove and enable dynamic update in native code.
+ // TODO: Remove.
/** Whether to call binary dictionary dynamically updating methods. */
- private static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = false;
+ public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true;
private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100;
@@ -60,6 +68,9 @@
*/
protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
+ private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
+ new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
+
/**
* A static map of time recorders, each of which records the time of accesses to a single binary
* dictionary file. The key for this map is the filename and the value is the shared dictionary
@@ -154,7 +165,11 @@
private static AbstractDictionaryWriter getDictionaryWriter(final Context context,
final String dictType, final boolean isDynamicPersonalizationDictionary) {
if (isDynamicPersonalizationDictionary) {
- return new DynamicPersonalizationDictionaryWriter(context, dictType);
+ if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ return null;
+ } else {
+ return new DynamicPersonalizationDictionaryWriter(context, dictType);
+ }
} else {
return new DictionaryWriter(context, dictType);
}
@@ -198,7 +213,9 @@
mBinaryDictionary.close();
mBinaryDictionary = null;
}
- mDictionaryWriter.close();
+ if (mDictionaryWriter != null) {
+ mDictionaryWriter.close();
+ }
}
});
}
@@ -220,7 +237,23 @@
getExecutor(mFilename).execute(new Runnable() {
@Override
public void run() {
- mDictionaryWriter.clear();
+ if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
+ mBinaryDictionary.close();
+ final File file = new File(mContext.getFilesDir(), mFilename);
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
+ new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
+ false, false));
+ final DictEncoder dictEncoder = new Ver3DictEncoder(file);
+ try {
+ dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
+ } catch (IOException e) {
+ Log.e(TAG, "Exception in creating new dictionary file.", e);
+ } catch (UnsupportedFormatException e) {
+ Log.e(TAG, "Exception in creating new dictionary file.", e);
+ }
+ } else {
+ mDictionaryWriter.clear();
+ }
}
});
}
@@ -257,9 +290,10 @@
public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.addUnigramWord(word, frequency);
+ } else {
+ // TODO: Remove.
+ mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
}
- // TODO: Remove.
- mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
}
});
}
@@ -280,10 +314,11 @@
public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.addBigramWords(word0, word1, frequency);
+ } else {
+ // TODO: Remove.
+ mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
+ 0 /* lastTouchedTime */);
}
- // TODO: Remove.
- mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
- 0 /* lastTouchedTime */);
}
});
}
@@ -303,9 +338,10 @@
public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.removeBigramWords(word0, word1);
+ } else {
+ // TODO: Remove.
+ mDictionaryWriter.removeBigramWords(word0, word1);
}
- // TODO: Remove.
- mDictionaryWriter.removeBigramWords(word0, word1);
}
});
}
@@ -322,26 +358,39 @@
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
public void run() {
- final ArrayList<SuggestedWordInfo> inMemDictSuggestion = composer.isBatchMode() ?
- null : mDictionaryWriter.getSuggestionsWithSessionId(composer, prevWord,
- proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
- sessionId);
- // TODO: Remove checking mIsUpdatable and use native suggestion.
- if (mBinaryDictionary != null && !mIsUpdatable) {
+ if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ if (mBinaryDictionary == null) {
+ holder.set(null);
+ return;
+ }
final ArrayList<SuggestedWordInfo> binarySuggestion =
mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
sessionId);
- if (inMemDictSuggestion == null) {
- holder.set(binarySuggestion);
- } else if (binarySuggestion == null) {
- holder.set(inMemDictSuggestion);
- } else {
- binarySuggestion.addAll(inMemDictSuggestion);
- holder.set(binarySuggestion);
- }
+ holder.set(binarySuggestion);
} else {
- holder.set(inMemDictSuggestion);
+ final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
+ composer.isBatchMode() ? null :
+ mDictionaryWriter.getSuggestionsWithSessionId(composer,
+ prevWord, proximityInfo, blockOffensiveWords,
+ additionalFeaturesOptions, sessionId);
+ // TODO: Remove checking mIsUpdatable and use native suggestion.
+ if (mBinaryDictionary != null && !mIsUpdatable) {
+ final ArrayList<SuggestedWordInfo> binarySuggestion =
+ mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
+ proximityInfo, blockOffensiveWords,
+ additionalFeaturesOptions, sessionId);
+ if (inMemDictSuggestion == null) {
+ holder.set(binarySuggestion);
+ } else if (binarySuggestion == null) {
+ holder.set(inMemDictSuggestion);
+ } else {
+ binarySuggestion.addAll(inMemDictSuggestion);
+ holder.set(binarySuggestion);
+ }
+ } else {
+ holder.set(inMemDictSuggestion);
+ }
}
}
});
@@ -411,8 +460,9 @@
final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length,
true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
- // Ensure all threads accessing the current dictionary have finished before swapping in
- // the new one.
+ // Ensure all threads accessing the current dictionary have finished before
+ // swapping in the new one.
+ // TODO: Ensure multi-thread assignment of mBinaryDictionary.
final BinaryDictionary oldBinaryDictionary = mBinaryDictionary;
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
@@ -443,8 +493,33 @@
if (needsToReloadBeforeWriting()) {
mDictionaryWriter.clear();
loadDictionaryAsync();
+ mDictionaryWriter.write(mFilename);
+ } else {
+ if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
+ final File file = new File(mContext.getFilesDir(), mFilename);
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
+ new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
+ false, false));
+ final DictEncoder dictEncoder = new Ver3DictEncoder(file);
+ try {
+ dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
+ } catch (IOException e) {
+ Log.e(TAG, "Exception in creating new dictionary file.", e);
+ } catch (UnsupportedFormatException e) {
+ Log.e(TAG, "Exception in creating new dictionary file.", e);
+ }
+ } else {
+ if (mBinaryDictionary.needsToRunGC()) {
+ mBinaryDictionary.flushWithGC();
+ } else {
+ mBinaryDictionary.flush();
+ }
+ }
+ } else {
+ mDictionaryWriter.write(mFilename);
+ }
}
- mDictionaryWriter.write(mFilename);
}
/**
@@ -539,7 +614,9 @@
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
public void run() {
- loadDictionaryAsync();
+ if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ loadDictionaryAsync();
+ }
}
});
}
@@ -547,7 +624,7 @@
/**
* Generate binary dictionary using DictionaryWriter.
*/
- protected void asyncWriteBinaryDictionary() {
+ protected void asyncFlashAllBinaryDictionary() {
final Runnable newTask = new Runnable() {
@Override
public void run() {
@@ -620,8 +697,12 @@
@Override
public void run() {
if (mDictType == Dictionary.TYPE_USER_HISTORY) {
- holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
- .isInDictionaryForTests(word));
+ if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ holder.set(mBinaryDictionary.isValidWord(word));
+ } else {
+ holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
+ .isInDictionaryForTests(word));
+ }
}
}
});
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index c9311a6..270dc4c 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -233,6 +233,7 @@
private static final int MSG_RESUME_SUGGESTIONS = 4;
private static final int MSG_REOPEN_DICTIONARIES = 5;
private static final int MSG_ON_END_BATCH_INPUT = 6;
+ private static final int MSG_RESET_CACHES = 7;
private static final int ARG1_NOT_GESTURE_INPUT = 0;
private static final int ARG1_DISMISS_GESTURE_FLOATING_PREVIEW_TEXT = 1;
@@ -297,6 +298,10 @@
case MSG_ON_END_BATCH_INPUT:
latinIme.onEndBatchInputAsyncInternal((SuggestedWords) msg.obj);
break;
+ case MSG_RESET_CACHES:
+ latinIme.retryResetCaches(msg.arg1 == 1 /* tryResumeSuggestions */,
+ msg.arg2 /* remainingTries */);
+ break;
}
}
@@ -313,6 +318,12 @@
sendMessageDelayed(obtainMessage(MSG_RESUME_SUGGESTIONS), mDelayUpdateSuggestions);
}
+ public void postResetCaches(final boolean tryResumeSuggestions, final int remainingTries) {
+ removeMessages(MSG_RESET_CACHES);
+ sendMessage(obtainMessage(MSG_RESET_CACHES, tryResumeSuggestions ? 1 : 0,
+ remainingTries, null));
+ }
+
public void cancelUpdateSuggestionStrip() {
removeMessages(MSG_UPDATE_SUGGESTION_STRIP);
}
@@ -852,7 +863,6 @@
// span, so we should reset our state unconditionally, even if restarting is true.
mEnteredText = null;
resetComposingState(true /* alsoResetLastComposedWord */);
- if (isDifferentTextField) mHandler.postResumeSuggestions();
mDeleteCount = 0;
mSpaceState = SPACE_STATE_NONE;
mRecapitalizeStatus.deactivate();
@@ -871,8 +881,16 @@
}
mSuggestedWords = SuggestedWords.EMPTY;
- mConnection.resetCachesUponCursorMove(editorInfo.initialSelStart,
- false /* shouldFinishComposition */);
+ // Sometimes, while rotating, for some reason the framework tells the app we are not
+ // connected to it and that means we can't refresh the cache. In this case, schedule a
+ // refresh later.
+ if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(editorInfo.initialSelStart,
+ false /* shouldFinishComposition */)) {
+ // We try resetting the caches up to 5 times before giving up.
+ mHandler.postResetCaches(isDifferentTextField, 5 /* remainingTries */);
+ } else {
+ if (isDifferentTextField) mHandler.postResumeSuggestions();
+ }
if (isDifferentTextField) {
mainKeyboardView.closing();
@@ -899,6 +917,9 @@
mLastSelectionStart = editorInfo.initialSelStart;
mLastSelectionEnd = editorInfo.initialSelEnd;
+ // In some cases (namely, after rotation of the device) editorInfo.initialSelStart is lying
+ // so we try using some heuristics to find out about these and fix them.
+ tryFixLyingCursorPosition();
mHandler.cancelUpdateSuggestionStrip();
mHandler.cancelDoubleSpacePeriodTimer();
@@ -918,6 +939,35 @@
if (TRACE) Debug.startMethodTracing("/data/trace/latinime");
}
+ /**
+ * Try to get the text from the editor to expose lies the framework may have been
+ * telling us. Concretely, when the device rotates, the frameworks tells us about where the
+ * cursor used to be initially in the editor at the time it first received the focus; this
+ * may be completely different from the place it is upon rotation. Since we don't have any
+ * means to get the real value, try at least to ask the text view for some characters and
+ * detect the most damaging cases: when the cursor position is declared to be much smaller
+ * than it really is.
+ */
+ private void tryFixLyingCursorPosition() {
+ final CharSequence textBeforeCursor =
+ mConnection.getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, 0);
+ if (null == textBeforeCursor) {
+ mLastSelectionStart = mLastSelectionEnd = NOT_A_CURSOR_POSITION;
+ } else {
+ final int textLength = textBeforeCursor.length();
+ if (textLength > mLastSelectionStart
+ || (textLength < Constants.EDITOR_CONTENTS_CACHE_SIZE
+ && mLastSelectionStart < Constants.EDITOR_CONTENTS_CACHE_SIZE)) {
+ mLastSelectionStart = textLength;
+ // We can't figure out the value of mLastSelectionEnd :(
+ // But at least if it's smaller than mLastSelectionStart something is wrong
+ if (mLastSelectionStart > mLastSelectionEnd) {
+ mLastSelectionEnd = mLastSelectionStart;
+ }
+ }
+ }
+ }
+
// Initialization of personalization debug settings. This must be called inside
// onStartInputView.
private void initPersonalizationDebugSettings(SettingsValues currentSettingsValues) {
@@ -1072,7 +1122,7 @@
// argument as true. But in all cases where we don't reset the entire input state,
// we still want to tell the rich input connection about the new cursor position so
// that it can update its caches.
- mConnection.resetCachesUponCursorMove(newSelStart,
+ mConnection.resetCachesUponCursorMoveAndReturnSuccess(newSelStart,
false /* shouldFinishComposition */);
}
@@ -1308,7 +1358,8 @@
} else {
setSuggestedWords(settingsValues.mSuggestPuncList, false);
}
- mConnection.resetCachesUponCursorMove(newCursorPosition, shouldFinishComposition);
+ mConnection.resetCachesUponCursorMoveAndReturnSuccess(newCursorPosition,
+ shouldFinishComposition);
}
private void resetComposingState(final boolean alsoResetLastComposedWord) {
@@ -2853,6 +2904,27 @@
mHandler.postUpdateSuggestionStrip();
}
+ /**
+ * Retry resetting caches in the rich input connection.
+ *
+ * When the editor can't be accessed we can't reset the caches, so we schedule a retry.
+ * This method handles the retry, and re-schedules a new retry if we still can't access.
+ * We only retry up to 5 times before giving up.
+ *
+ * @param tryResumeSuggestions Whether we should resume suggestions or not.
+ * @param remainingTries How many times we may try again before giving up.
+ */
+ private void retryResetCaches(final boolean tryResumeSuggestions, final int remainingTries) {
+ if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(mLastSelectionStart, false)) {
+ if (0 < remainingTries) {
+ mHandler.postResetCaches(tryResumeSuggestions, remainingTries - 1);
+ }
+ return;
+ }
+ tryFixLyingCursorPosition();
+ if (tryResumeSuggestions) mHandler.postResumeSuggestions();
+ }
+
private void revertCommit() {
final String previousWord = mLastComposedWord.mPrevWord;
final String originallyTypedWord = mLastComposedWord.mTypedWord;
diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java
index 2ecf146..925381b 100644
--- a/java/src/com/android/inputmethod/latin/RichInputConnection.java
+++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java
@@ -73,9 +73,6 @@
* This contains the currently composing text, as LatinIME thinks the TextView is seeing it.
*/
private final StringBuilder mComposingText = new StringBuilder();
- // A hint on how many characters to cache from the TextView. A good value of this is given by
- // how many characters we need to be able to almost always find the caps mode.
- private static final int DEFAULT_TEXT_CACHE_SIZE = 100;
private final InputMethodService mParent;
InputConnection mIC;
@@ -93,7 +90,8 @@
r.token = 1;
r.flags = 0;
final ExtractedText et = mIC.getExtractedText(r, 0);
- final CharSequence beforeCursor = getTextBeforeCursor(DEFAULT_TEXT_CACHE_SIZE, 0);
+ final CharSequence beforeCursor = getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE,
+ 0);
final StringBuilder internal = new StringBuilder().append(mCommittedTextBeforeComposingText)
.append(mComposingText);
if (null == et || null == beforeCursor) return;
@@ -142,19 +140,56 @@
if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug();
}
- public void resetCachesUponCursorMove(final int newCursorPosition,
+ /**
+ * Reset the cached text and retrieve it again from the editor.
+ *
+ * This should be called when the cursor moved. It's possible that we can't connect to
+ * the application when doing this; notably, this happens sometimes during rotation, probably
+ * because of a race condition in the framework. In this case, we just can't retrieve the
+ * data, so we empty the cache and note that we don't know the new cursor position, and we
+ * return false so that the caller knows about this and can retry later.
+ *
+ * @param newCursorPosition The new position of the cursor, as received from the system.
+ * @param shouldFinishComposition Whether we should finish the composition in progress.
+ * @return true if we were able to connect to the editor successfully, false otherwise. When
+ * this method returns false, the caches could not be correctly refreshed so they were only
+ * reset: the caller should try again later to return to normal operation.
+ */
+ public boolean resetCachesUponCursorMoveAndReturnSuccess(final int newCursorPosition,
final boolean shouldFinishComposition) {
mExpectedCursorPosition = newCursorPosition;
mComposingText.setLength(0);
mCommittedTextBeforeComposingText.setLength(0);
- final CharSequence textBeforeCursor = getTextBeforeCursor(DEFAULT_TEXT_CACHE_SIZE, 0);
- if (null != textBeforeCursor) mCommittedTextBeforeComposingText.append(textBeforeCursor);
+ mIC = mParent.getCurrentInputConnection();
+ // Call upon the inputconnection directly since our own method is using the cache, and
+ // we want to refresh it.
+ final CharSequence textBeforeCursor = null == mIC ? null :
+ mIC.getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, 0);
+ if (null == textBeforeCursor) {
+ // For some reason the app thinks we are not connected to it. This looks like a
+ // framework bug... Fall back to ground state and return false.
+ mExpectedCursorPosition = INVALID_CURSOR_POSITION;
+ Log.e(TAG, "Unable to connect to the editor to retrieve text... will retry later");
+ return false;
+ }
+ mCommittedTextBeforeComposingText.append(textBeforeCursor);
+ final int lengthOfTextBeforeCursor = textBeforeCursor.length();
+ if (lengthOfTextBeforeCursor > newCursorPosition
+ || (lengthOfTextBeforeCursor < Constants.EDITOR_CONTENTS_CACHE_SIZE
+ && newCursorPosition < Constants.EDITOR_CONTENTS_CACHE_SIZE)) {
+ // newCursorPosition may be lying -- when rotating the device (probably a framework
+ // bug). If we have less chars than we asked for, then we know how many chars we have,
+ // and if we got more than newCursorPosition says, then we know it was lying. In both
+ // cases the length is more reliable
+ mExpectedCursorPosition = lengthOfTextBeforeCursor;
+ }
if (null != mIC && shouldFinishComposition) {
mIC.finishComposingText();
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
ResearchLogger.richInputConnection_finishComposingText();
}
}
+ return true;
}
private void checkBatchEdit() {
@@ -233,7 +268,8 @@
// getCapsMode should be updated to be able to return a "not enough info" result so that
// we can get more context only when needed.
if (TextUtils.isEmpty(mCommittedTextBeforeComposingText) && 0 != mExpectedCursorPosition) {
- final CharSequence textBeforeCursor = getTextBeforeCursor(DEFAULT_TEXT_CACHE_SIZE, 0);
+ final CharSequence textBeforeCursor = getTextBeforeCursor(
+ Constants.EDITOR_CONTENTS_CACHE_SIZE, 0);
if (!TextUtils.isEmpty(textBeforeCursor)) {
mCommittedTextBeforeComposingText.append(textBeforeCursor);
}
@@ -364,7 +400,7 @@
if (DEBUG_BATCH_NESTING) checkBatchEdit();
if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug();
final CharSequence textBeforeCursor =
- getTextBeforeCursor(DEFAULT_TEXT_CACHE_SIZE + (end - start), 0);
+ getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE + (end - start), 0);
mCommittedTextBeforeComposingText.setLength(0);
if (!TextUtils.isEmpty(textBeforeCursor)) {
final int indexOfStartOfComposingText =
@@ -406,7 +442,8 @@
}
mExpectedCursorPosition = start;
mCommittedTextBeforeComposingText.setLength(0);
- mCommittedTextBeforeComposingText.append(getTextBeforeCursor(DEFAULT_TEXT_CACHE_SIZE, 0));
+ mCommittedTextBeforeComposingText.append(
+ getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, 0));
}
public void commitCorrection(final CorrectionInfo correctionInfo) {
@@ -525,9 +562,9 @@
if (mIC == null || sep == null) {
return null;
}
- final CharSequence before = mIC.getTextBeforeCursor(1000,
+ final CharSequence before = mIC.getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE,
InputConnection.GET_TEXT_WITH_STYLES);
- final CharSequence after = mIC.getTextAfterCursor(1000,
+ final CharSequence after = mIC.getTextAfterCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE,
InputConnection.GET_TEXT_WITH_STYLES);
if (before == null || after == null) {
return null;
@@ -570,8 +607,11 @@
}
}
- return new TextRange(TextUtils.concat(before, after), startIndexInBefore,
- before.length() + endIndexInAfter, before.length());
+ // We don't use TextUtils#concat because it copies all spans without respect to their
+ // nature. If the text includes a PARAGRAPH span and it has been split, then
+ // TextUtils#concat will crash when it tries to concat both sides of it.
+ return new TextRange(StringUtils.concatWithNonParagraphSuggestionSpansOnly(before, after),
+ startIndexInBefore, before.length() + endIndexInAfter, before.length());
}
public boolean isCursorTouchingWord(final SettingsValues settingsValues) {
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
index e43e74d..0af028a 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
@@ -19,6 +19,7 @@
import android.content.Context;
import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.compat.ActivityManagerCompatUtils;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.AbstractDictionaryWriter;
import com.android.inputmethod.latin.ExpandableDictionary;
@@ -41,7 +42,8 @@
public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWriter {
private static final String TAG = DynamicPersonalizationDictionaryWriter.class.getSimpleName();
/** Maximum number of pairs. Pruning will start when databases goes above this number. */
- public static final int MAX_HISTORY_BIGRAMS = 10000;
+ public static final int DEFAULT_MAX_HISTORY_BIGRAMS = 10000;
+ public static final int LOW_MEMORY_MAX_HISTORY_BIGRAMS = 2000;
/** Any pair being typed or picked */
private static final int FREQUENCY_FOR_TYPED = 2;
@@ -53,10 +55,14 @@
private final UserHistoryDictionaryBigramList mBigramList =
new UserHistoryDictionaryBigramList();
private final ExpandableDictionary mExpandableDictionary;
+ private final int mMaxHistoryBigrams;
public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) {
super(context, dictType);
mExpandableDictionary = new ExpandableDictionary(dictType);
+ final boolean isLowRamDevice = ActivityManagerCompatUtils.isLowRamDevice(context);
+ mMaxHistoryBigrams = isLowRamDevice ?
+ LOW_MEMORY_MAX_HISTORY_BIGRAMS : DEFAULT_MAX_HISTORY_BIGRAMS;
}
@Override
@@ -72,6 +78,10 @@
@Override
public void addUnigramWord(final String word, final String shortcutTarget, final int frequency,
final boolean isNotAWord) {
+ if (mBigramList.size() > mMaxHistoryBigrams * 2) {
+ // Too many entries: just stop adding new vocabrary and wait next refresh.
+ return;
+ }
mExpandableDictionary.addWord(word, shortcutTarget, frequency);
mBigramList.addBigram(null, word, (byte)frequency);
}
@@ -79,6 +89,10 @@
@Override
public void addBigramWords(final String word0, final String word1, final int frequency,
final boolean isValid, final long lastModifiedTime) {
+ if (mBigramList.size() > mMaxHistoryBigrams * 2) {
+ // Too many entries: just stop adding new vocabrary and wait next refresh.
+ return;
+ }
if (lastModifiedTime > 0) {
mExpandableDictionary.setBigramAndGetFrequency(word0, word1,
new ForgettingCurveParams(frequency, System.currentTimeMillis(),
@@ -102,19 +116,22 @@
protected void writeDictionary(final DictEncoder dictEncoder)
throws IOException, UnsupportedFormatException {
UserHistoryDictIOUtils.writeDictionary(dictEncoder,
- new FrequencyProvider(mBigramList, mExpandableDictionary), mBigramList,
- FORMAT_OPTIONS);
+ new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams),
+ mBigramList, FORMAT_OPTIONS);
}
private static class FrequencyProvider implements BigramDictionaryInterface {
- final private UserHistoryDictionaryBigramList mBigramList;
- final private ExpandableDictionary mExpandableDictionary;
+ private final UserHistoryDictionaryBigramList mBigramList;
+ private final ExpandableDictionary mExpandableDictionary;
+ private final int mMaxHistoryBigrams;
public FrequencyProvider(final UserHistoryDictionaryBigramList bigramList,
- final ExpandableDictionary expandableDictionary) {
+ final ExpandableDictionary expandableDictionary, final int maxHistoryBigrams) {
mBigramList = bigramList;
mExpandableDictionary = expandableDictionary;
+ mMaxHistoryBigrams = maxHistoryBigrams;
}
+
@Override
public int getFrequency(final String word0, final String word1) {
final int freq;
@@ -130,7 +147,7 @@
if (prevFc > 0 && prevFc == fc) {
freq = fc & 0xFF;
} else if (UserHistoryForgettingCurveUtils.
- needsToSave(fc, isValid, mBigramList.size() <= MAX_HISTORY_BIGRAMS)) {
+ needsToSave(fc, isValid, mBigramList.size() <= mMaxHistoryBigrams)) {
freq = fc & 0xFF;
} else {
// Delete this entry
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
index 9364fb0..075d7e3 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
@@ -74,12 +74,12 @@
@Override
public void close() {
- // Close only binary dictionary to reuse this dictionary.
- // super.close();
- closeBinaryDictionary();
+ if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+ closeBinaryDictionary();
+ }
// Flush pending writes.
// TODO: Remove after this class become to use a dynamic binary dictionary.
- asyncWriteBinaryDictionary();
+ asyncFlashAllBinaryDictionary();
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
}
@@ -212,6 +212,6 @@
// Clear the node structure on memory
clear();
// Then flush the cleared state of the dictionary on disk.
- asyncWriteBinaryDictionary();
+ asyncFlashAllBinaryDictionary();
}
}
diff --git a/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java b/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java
index 2f91c57..60b24d5 100644
--- a/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java
@@ -60,6 +60,11 @@
|| WordComposer.CAPS_MODE_AUTO_SHIFT_LOCKED == mode;
}
+ private static boolean isPeriod(final int codePoint) {
+ // TODO: make this a resource.
+ return codePoint == Constants.CODE_PERIOD || codePoint == Constants.CODE_ARMENIAN_PERIOD;
+ }
+
/**
* Determine what caps mode should be in effect at the current offset in
* the text. Only the mode bits set in <var>reqModes</var> will be
@@ -190,7 +195,7 @@
if (c == Constants.CODE_QUESTION_MARK || c == Constants.CODE_EXCLAMATION_MARK) {
return (TextUtils.CAP_MODE_CHARACTERS | TextUtils.CAP_MODE_SENTENCES) & reqModes;
}
- if (c != Constants.CODE_PERIOD || j <= 0) {
+ if (!isPeriod(c) || j <= 0) {
return (TextUtils.CAP_MODE_CHARACTERS | TextUtils.CAP_MODE_WORDS) & reqModes;
}
@@ -240,7 +245,7 @@
case WORD:
if (Character.isLetter(c)) {
state = WORD;
- } else if (c == Constants.CODE_PERIOD) {
+ } else if (isPeriod(c)) {
state = PERIOD;
} else {
return caps;
@@ -256,7 +261,7 @@
case LETTER:
if (Character.isLetter(c)) {
state = LETTER;
- } else if (c == Constants.CODE_PERIOD) {
+ } else if (isPeriod(c)) {
state = PERIOD;
} else {
return noCaps;
diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
index 121aecf..327780a 100644
--- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
@@ -20,7 +20,12 @@
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.settings.SettingsValues;
+import android.text.Spannable;
+import android.text.SpannableString;
+import android.text.Spanned;
+import android.text.SpannedString;
import android.text.TextUtils;
+import android.text.style.SuggestionSpan;
import android.util.JsonReader;
import android.util.JsonWriter;
import android.util.Log;
@@ -462,4 +467,88 @@
}
return "";
}
+
+ /**
+ * Copies the spans from the region <code>start...end</code> in
+ * <code>source</code> to the region
+ * <code>destoff...destoff+end-start</code> in <code>dest</code>.
+ * Spans in <code>source</code> that begin before <code>start</code>
+ * or end after <code>end</code> but overlap this range are trimmed
+ * as if they began at <code>start</code> or ended at <code>end</code>.
+ * Only SuggestionSpans that don't have the SPAN_PARAGRAPH span are copied.
+ *
+ * This code is almost entirely taken from {@link TextUtils#copySpansFrom}, except for the
+ * kind of span that is copied.
+ *
+ * @throws IndexOutOfBoundsException if any of the copied spans
+ * are out of range in <code>dest</code>.
+ */
+ public static void copyNonParagraphSuggestionSpansFrom(Spanned source, int start, int end,
+ Spannable dest, int destoff) {
+ Object[] spans = source.getSpans(start, end, SuggestionSpan.class);
+
+ for (int i = 0; i < spans.length; i++) {
+ int fl = source.getSpanFlags(spans[i]);
+ if (0 != (fl & Spannable.SPAN_PARAGRAPH)) continue;
+
+ int st = source.getSpanStart(spans[i]);
+ int en = source.getSpanEnd(spans[i]);
+
+ if (st < start)
+ st = start;
+ if (en > end)
+ en = end;
+
+ dest.setSpan(spans[i], st - start + destoff, en - start + destoff,
+ fl);
+ }
+ }
+
+ /**
+ * Returns a CharSequence concatenating the specified CharSequences, retaining their
+ * SuggestionSpans that don't have the PARAGRAPH flag, but not other spans.
+ *
+ * This code is almost entirely taken from {@link TextUtils#concat(CharSequence...)}, except
+ * it calls copyNonParagraphSuggestionSpansFrom instead of {@link TextUtils#copySpansFrom}.
+ */
+ public static CharSequence concatWithNonParagraphSuggestionSpansOnly(CharSequence... text) {
+ if (text.length == 0) {
+ return "";
+ }
+
+ if (text.length == 1) {
+ return text[0];
+ }
+
+ boolean spanned = false;
+ for (int i = 0; i < text.length; i++) {
+ if (text[i] instanceof Spanned) {
+ spanned = true;
+ break;
+ }
+ }
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < text.length; i++) {
+ sb.append(text[i]);
+ }
+
+ if (!spanned) {
+ return sb.toString();
+ }
+
+ SpannableString ss = new SpannableString(sb);
+ int off = 0;
+ for (int i = 0; i < text.length; i++) {
+ int len = text[i].length();
+
+ if (text[i] instanceof Spanned) {
+ copyNonParagraphSuggestionSpansFrom((Spanned) text[i], 0, len, ss, off);
+ }
+
+ off += len;
+ }
+
+ return new SpannedString(ss);
+ }
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp
index f8ed2b9..1926b98 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp
@@ -16,6 +16,7 @@
#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"
+#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
@@ -78,8 +79,10 @@
offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos);
break;
}
- if (offset == 0) {
+ if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID) {
return NOT_A_DICT_POS;
+ } else if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET) {
+ return origin;
}
if (isOffsetNegative(flags)) {
return origin - offset;
@@ -88,6 +91,24 @@
}
}
+/* static */ bool BigramListReadWriteUtils::setHasNextFlag(
+ BufferWithExtendableBuffer *const buffer, const bool hasNext, const int entryPos) {
+ const bool usesAdditionalBuffer = buffer->isInAdditionalBuffer(entryPos);
+ int readingPos = entryPos;
+ if (usesAdditionalBuffer) {
+ readingPos -= buffer->getOriginalBufferSize();
+ }
+ BigramFlags bigramFlags = ByteArrayUtils::readUint8AndAdvancePosition(
+ buffer->getBuffer(usesAdditionalBuffer), &readingPos);
+ if (hasNext) {
+ bigramFlags = bigramFlags | FLAG_ATTRIBUTE_HAS_NEXT;
+ } else {
+ bigramFlags = bigramFlags & (~FLAG_ATTRIBUTE_HAS_NEXT);
+ }
+ int writingPos = entryPos;
+ return buffer->writeUintAndAdvancePosition(bigramFlags, 1 /* size */, &writingPos);
+}
+
/* static */ bool BigramListReadWriteUtils::createAndWriteBigramEntry(
BufferWithExtendableBuffer *const buffer, const int targetPos, const int probability,
const bool hasNext, int *const writingPos) {
@@ -101,10 +122,12 @@
/* static */ bool BigramListReadWriteUtils::writeBigramEntry(
BufferWithExtendableBuffer *const bufferToWrite, const BigramFlags flags,
const int targetPtNodePos, int *const writingPos) {
- if (!bufferToWrite->writeUintAndAdvancePosition(flags, 1 /* size */, writingPos)) {
+ const int offset = getBigramTargetOffset(targetPtNodePos, *writingPos);
+ const BigramFlags flagsToWrite = (offset < 0) ?
+ (flags | FLAG_ATTRIBUTE_OFFSET_NEGATIVE) : (flags & ~FLAG_ATTRIBUTE_OFFSET_NEGATIVE);
+ if (!bufferToWrite->writeUintAndAdvancePosition(flagsToWrite, 1 /* size */, writingPos)) {
return false;
}
- const int offset = (targetPtNodePos != NOT_A_DICT_POS) ? targetPtNodePos - *writingPos : 0;
const uint32_t absOffest = abs(offset);
const int bigramTargetFieldSize = attributeAddressSize(flags);
return bufferToWrite->writeUintAndAdvancePosition(absOffest, bigramTargetFieldSize,
@@ -113,14 +136,13 @@
// Returns true if the bigram entry is valid and put entry flags into out*.
/* static */ bool BigramListReadWriteUtils::createAndGetBigramFlags(const int entryPos,
- const int targetPos, const int probability, const bool hasNext,
+ const int targetPtNodePos, const int probability, const bool hasNext,
BigramFlags *const outBigramFlags) {
BigramFlags flags = probability & MASK_ATTRIBUTE_PROBABILITY;
if (hasNext) {
flags |= FLAG_ATTRIBUTE_HAS_NEXT;
}
- const int targetFieldPos = entryPos + 1;
- const int offset = (targetPos != NOT_A_DICT_POS) ? targetPos - targetFieldPos : 0;
+ const int offset = getBigramTargetOffset(targetPtNodePos, entryPos);
if (offset < 0) {
flags |= FLAG_ATTRIBUTE_OFFSET_NEGATIVE;
}
@@ -143,4 +165,18 @@
return true;
}
+/* static */ int BigramListReadWriteUtils::getBigramTargetOffset(const int targetPtNodePos,
+ const int entryPos) {
+ if (targetPtNodePos == NOT_A_DICT_POS) {
+ return DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID;
+ } else {
+ const int offset = targetPtNodePos - (entryPos + 1 /* bigramFlagsField */);
+ if (offset == 0) {
+ return DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET;
+ } else {
+ return offset;
+ }
+ }
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h
index 234a0ea..eabe4e0 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h
@@ -59,9 +59,8 @@
*/
}
- static AK_FORCE_INLINE BigramFlags setHasNextFlag(const BigramFlags flags) {
- return flags | FLAG_ATTRIBUTE_HAS_NEXT;
- }
+ static bool setHasNextFlag(BufferWithExtendableBuffer *const buffer,
+ const bool hasNext, const int entryPos);
static AK_FORCE_INLINE BigramFlags setProbabilityInFlags(const BigramFlags flags,
const int probability) {
@@ -96,6 +95,8 @@
static int getBigramAddressAndAdvancePosition(const uint8_t *const bigramsBuf,
const BigramFlags flags, int *const pos);
+
+ static int getBigramTargetOffset(const int targetPtNodePos, const int entryPos);
};
} // namespace latinime
#endif // LATINIME_BIGRAM_LIST_READ_WRITE_UTILS_H
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
index ba43bdb..bc2f5ee 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
@@ -19,6 +19,7 @@
#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"
+#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
@@ -69,9 +70,11 @@
*outBigramsCount = 0;
BigramListReadWriteUtils::BigramFlags bigramFlags;
int bigramEntryCount = 0;
+ int lastWrittenEntryPos = NOT_A_DICT_POS;
do {
if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {
- AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);
+ AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",
+ bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);
ASSERT(false);
return false;
}
@@ -88,6 +91,11 @@
originalBigramPos += mBuffer->getOriginalBufferSize();
}
const int bigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos);
+ if (bigramPos == NOT_A_DICT_POS) {
+ // Target PtNode has been invalidated.
+ continue;
+ }
+ lastWrittenEntryPos = *toPos;
if (!BigramListReadWriteUtils::createAndWriteBigramEntry(bufferToWrite, bigramPos,
BigramListReadWriteUtils::getProbabilityFromFlags(bigramFlags),
BigramListReadWriteUtils::hasNext(bigramFlags), toPos)) {
@@ -95,6 +103,13 @@
}
(*outBigramsCount)++;
} while(BigramListReadWriteUtils::hasNext(bigramFlags));
+ // Makes the last entry the terminal of the list. Updates the flags.
+ if (lastWrittenEntryPos != NOT_A_DICT_POS) {
+ if (!BigramListReadWriteUtils::setHasNextFlag(bufferToWrite, false /* hasNext */,
+ lastWrittenEntryPos)) {
+ return false;
+ }
+ }
if (usesAdditionalBuffer) {
*fromPos += mBuffer->getOriginalBufferSize();
}
@@ -114,7 +129,8 @@
int bigramEntryCount = 0;
do {
if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {
- AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);
+ AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",
+ bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);
ASSERT(false);
return false;
}
@@ -150,6 +166,54 @@
return true;
}
+// Updates bigram target PtNode positions in the list after the placing step in GC.
+bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos,
+ const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const
+ ptNodePositionRelocationMap) {
+ const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
+ if (usesAdditionalBuffer) {
+ *bigramListPos -= mBuffer->getOriginalBufferSize();
+ }
+ BigramListReadWriteUtils::BigramFlags bigramFlags;
+ int bigramEntryCount = 0;
+ do {
+ if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {
+ AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",
+ bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);
+ ASSERT(false);
+ return false;
+ }
+ int bigramEntryPos = *bigramListPos;
+ if (usesAdditionalBuffer) {
+ bigramEntryPos += mBuffer->getOriginalBufferSize();
+ }
+ int bigramTargetPtNodePos;
+ // The buffer address can be changed after calling buffer writing methods.
+ BigramListReadWriteUtils::getBigramEntryPropertiesAndAdvancePosition(
+ mBuffer->getBuffer(usesAdditionalBuffer), &bigramFlags, &bigramTargetPtNodePos,
+ bigramListPos);
+ if (bigramTargetPtNodePos == NOT_A_DICT_POS) {
+ continue;
+ }
+ if (usesAdditionalBuffer) {
+ bigramTargetPtNodePos += mBuffer->getOriginalBufferSize();
+ }
+
+ DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it =
+ ptNodePositionRelocationMap->find(bigramTargetPtNodePos);
+ if (it != ptNodePositionRelocationMap->end()) {
+ bigramTargetPtNodePos = it->second;
+ } else {
+ bigramTargetPtNodePos = NOT_A_DICT_POS;
+ }
+ if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, bigramFlags,
+ bigramTargetPtNodePos, &bigramEntryPos)) {
+ return false;
+ }
+ } while(BigramListReadWriteUtils::hasNext(bigramFlags));
+ return true;
+}
+
bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTargetPos,
const int probability, int *const bigramListPos) {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
@@ -160,7 +224,8 @@
int bigramEntryCount = 0;
do {
if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {
- AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);
+ AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",
+ bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);
ASSERT(false);
return false;
}
@@ -188,10 +253,7 @@
}
// The current last entry is found.
// First, update the flags of the last entry.
- const BigramListReadWriteUtils::BigramFlags updatedFlags =
- BigramListReadWriteUtils::setHasNextFlag(bigramFlags);
- if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, updatedFlags, originalBigramPos,
- &entryPos)) {
+ if (!BigramListReadWriteUtils::setHasNextFlag(mBuffer, true /* hasNext */, entryPos)) {
return false;
}
if (usesAdditionalBuffer) {
@@ -222,7 +284,8 @@
int bigramEntryCount = 0;
do {
if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {
- AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);
+ AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",
+ bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);
ASSERT(false);
return false;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
index 16b080a..8ea318a 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
@@ -21,6 +21,7 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
+#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"
namespace latinime {
@@ -51,6 +52,10 @@
bool updateAllBigramEntriesAndDeleteUselessEntries(int *const bigramListPos);
+ bool updateAllBigramTargetPtNodePositions(int *const bigramListPos,
+ const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const
+ ptNodePositionRelocationMap);
+
bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability,
int *const bigramListPos);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
index ffa02e3..c60e458 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
@@ -19,7 +19,7 @@
namespace latinime {
bool DynamicPatriciaTrieGcEventListeners
- ::ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted
+ ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
const int *const nodeCodePoints) {
// PtNode is useless when the PtNode is not a terminal and doesn't have any not useless
@@ -47,11 +47,13 @@
}
// Writes dummy PtNode array size when the head of PtNode array is read.
-bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer
+bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
::onDescend(const int ptNodeArrayPos) {
mValidPtNodeCount = 0;
int writingPos = mBufferToWrite->getTailPosition();
- mPositionMap->insert(hash_map_compat<int, int>::value_type(ptNodeArrayPos, writingPos));
+ mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.insert(
+ DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::value_type(
+ ptNodeArrayPos, writingPos));
// Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes.
// This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count.
mPtNodeArraySizeFieldPos = writingPos;
@@ -60,7 +62,7 @@
}
// Write PtNode array terminal and actual PtNode array size.
-bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer
+bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
::onReadingPtNodeArrayTail() {
int writingPos = mBufferToWrite->getTailPosition();
// Write PtNode array terminal.
@@ -77,22 +79,71 @@
}
// Write valid PtNode to buffer and memorize mapping from the old position to the new position.
-bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer
+bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
const int *const nodeCodePoints) {
if (node->isDeleted()) {
// Current PtNode is not written in new buffer because it has been deleted.
- mPositionMap->insert(hash_map_compat<int, int>::value_type(node->getHeadPos(),
- NOT_A_DICT_POS));
+ mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert(
+ DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type(
+ node->getHeadPos(), NOT_A_DICT_POS));
return true;
}
int writingPos = mBufferToWrite->getTailPosition();
- mPositionMap->insert(hash_map_compat<int, int>::value_type(node->getHeadPos(), writingPos));
+ mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert(
+ DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type(
+ node->getHeadPos(), writingPos));
mValidPtNodeCount++;
// Writes current PtNode.
return mWritingHelper->writePtNodeToBufferByCopyingPtNodeInfo(mBufferToWrite, node,
- node->getParentPos(), nodeCodePoints, node->getCodePointCount(),
+ node->getParentPos(), nodeCodePoints, node->getCodePointCount(),
node->getProbability(), &writingPos);
}
+bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields
+ ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
+ const int *const nodeCodePoints) {
+ // Updates parent position.
+ int parentPos = node->getParentPos();
+ if (parentPos != NOT_A_DICT_POS) {
+ DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it =
+ mDictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos);
+ if (it != mDictPositionRelocationMap->mPtNodePositionRelocationMap.end()) {
+ parentPos = it->second;
+ }
+ }
+ int writingPos = node->getHeadPos() + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE;
+ // Write updated parent offset.
+ if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mBufferToWrite,
+ parentPos, node->getHeadPos(), &writingPos)) {
+ return false;
+ }
+
+ // Updates children position.
+ int childrenPos = node->getChildrenPos();
+ if (childrenPos != NOT_A_DICT_POS) {
+ DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::const_iterator it =
+ mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos);
+ if (it != mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) {
+ childrenPos = it->second;
+ }
+ }
+ writingPos = node->getChildrenPosFieldPos();
+ if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBufferToWrite,
+ childrenPos, &writingPos)) {
+ return false;
+ }
+
+ // Updates bigram target PtNode positions in the bigram list.
+ int bigramsPos = node->getBigramsPos();
+ if (bigramsPos != NOT_A_DICT_POS) {
+ if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos,
+ &mDictPositionRelocationMap->mPtNodePositionRelocationMap)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
index 7285593..4256f22 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
@@ -34,16 +34,16 @@
// Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or
// not and marks useless PtNodes as deleted. Such deleted PtNodes will be discarded in the GC.
// TODO: Concatenate non-terminal PtNodes.
- class ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted
+ class TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
public:
- ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted(
+ TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
DynamicPatriciaTrieWritingHelper *const writingHelper,
BufferWithExtendableBuffer *const buffer)
: mWritingHelper(writingHelper), mBuffer(buffer), valueStack(),
mChildrenValue(0) {}
- ~ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted() {};
+ ~TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted() {};
bool onAscend() {
if (valueStack.empty()) {
@@ -66,7 +66,7 @@
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(
- ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted);
+ TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted);
DynamicPatriciaTrieWritingHelper *const mWritingHelper;
BufferWithExtendableBuffer *const mBuffer;
@@ -76,10 +76,10 @@
// Updates all bigram entries that are held by valid PtNodes. This removes useless bigram
// entries.
- class ListenerForUpdatingBigramProbability
+ class TraversePolicyToUpdateBigramProbability
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
public:
- ListenerForUpdatingBigramProbability(DynamicBigramListPolicy *const bigramPolicy)
+ TraversePolicyToUpdateBigramProbability(DynamicBigramListPolicy *const bigramPolicy)
: mBigramPolicy(bigramPolicy) {}
bool onAscend() { return true; }
@@ -102,20 +102,21 @@
}
private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(ListenerForUpdatingBigramProbability);
+ DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability);
DynamicBigramListPolicy *const mBigramPolicy;
};
- class ListenerForPlacingAndWritingValidPtNodesToBuffer
+ class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
public:
- ListenerForPlacingAndWritingValidPtNodesToBuffer(
+ TraversePolicyToPlaceAndWriteValidPtNodesToBuffer(
DynamicPatriciaTrieWritingHelper *const writingHelper,
BufferWithExtendableBuffer *const bufferToWrite,
- hash_map_compat<int, int> *const positionMap)
+ DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
+ dictPositionRelocationMap)
: mWritingHelper(writingHelper), mBufferToWrite(bufferToWrite),
- mPositionMap(positionMap), mValidPtNodeCount(0),
+ mDictPositionRelocationMap(dictPositionRelocationMap), mValidPtNodeCount(0),
mPtNodeArraySizeFieldPos(NOT_A_DICT_POS) {};
bool onAscend() { return true; }
@@ -127,20 +128,49 @@
bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
const int *const nodeCodePoints);
- hash_map_compat<int, int> *getPositionMap() const {
- return mPositionMap;
- }
-
private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(ListenerForPlacingAndWritingValidPtNodesToBuffer);
+ DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToPlaceAndWriteValidPtNodesToBuffer);
DynamicPatriciaTrieWritingHelper *const mWritingHelper;
BufferWithExtendableBuffer *const mBufferToWrite;
- hash_map_compat<int, int> *const mPositionMap;
+ DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
+ mDictPositionRelocationMap;
int mValidPtNodeCount;
int mPtNodeArraySizeFieldPos;
};
+ class TraversePolicyToUpdateAllPositionFields
+ : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
+ public:
+ TraversePolicyToUpdateAllPositionFields(
+ DynamicPatriciaTrieWritingHelper *const writingHelper,
+ DynamicBigramListPolicy *const bigramPolicy,
+ BufferWithExtendableBuffer *const bufferToWrite,
+ const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
+ dictPositionRelocationMap)
+ : mWritingHelper(writingHelper), mBigramPolicy(bigramPolicy),
+ mBufferToWrite(bufferToWrite),
+ mDictPositionRelocationMap(dictPositionRelocationMap) {};
+
+ bool onAscend() { return true; }
+
+ bool onDescend(const int ptNodeArrayPos) { return true; }
+
+ bool onReadingPtNodeArrayTail() { return true; }
+
+ bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
+ const int *const nodeCodePoints);
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields);
+
+ DynamicPatriciaTrieWritingHelper *const mWritingHelper;
+ DynamicBigramListPolicy *const mBigramPolicy;
+ BufferWithExtendableBuffer *const mBufferToWrite;
+ const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
+ mDictPositionRelocationMap;
+ };
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieGcEventListeners);
};
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
index d52de7a..456352c 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp
@@ -40,9 +40,10 @@
pos -= mBuffer->getOriginalBufferSize();
}
mFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
- const int parentPos =
- DynamicPatriciaTrieReadingUtils::getParentPosAndAdvancePosition(dictBuf, &pos);
- mParentPos = (parentPos != 0) ? ptNodePos + parentPos : NOT_A_DICT_POS;
+ const int parentPosOffset =
+ DynamicPatriciaTrieReadingUtils::getParentPtNodePosOffsetAndAdvancePosition(dictBuf,
+ &pos);
+ mParentPos = DynamicPatriciaTrieReadingUtils::getParentPtNodePos(parentPosOffset, mHeadPos);
if (outCodePoints != 0) {
mCodePointCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition(
dictBuf, mFlags, maxCodePointCount, outCodePoints, &pos);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
index 70a09c2..42397c1 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
@@ -258,7 +258,9 @@
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
return;
}
- // TODO: Implement.
+ DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
+ &mBigramListPolicy, &mShortcutListPolicy);
+ writingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy);
}
bool DynamicPatriciaTriePolicy::needsToRunGC() const {
@@ -266,8 +268,8 @@
AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
return false;
}
- // TODO: Implement.
- return false;
+ // TODO: Implement more properly.
+ return mBufferWithExtendableBuffer.isNearSizeLimit();
}
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp
index 754b679..f4a2ef3 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp
@@ -59,6 +59,9 @@
if (!listener->onReadingPtNodeArrayTail()) {
return false;
}
+ if (mReadingStateStack.size() <= 0) {
+ break;
+ }
if (!listener->onAscend()) {
return false;
}
@@ -101,6 +104,9 @@
if (!listener->onAscend()) {
return false;
}
+ if (mReadingStateStack.size() <= 0) {
+ break;
+ }
popReadingStateFromStack();
alreadyVisitedChildren = true;
alreadyVisitedAllPtNodesInArray = true;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp
index 8428c0b..d68446d 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp
@@ -28,24 +28,42 @@
const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40;
const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80;
+// TODO: Make DICT_OFFSET_ZERO_OFFSET = 0.
+// Currently, DICT_OFFSET_INVALID is 0 in Java side but offset can be 0 during GC. So, the maximum
+// value of offsets, which is 0x7FFFFF is used to represent 0 offset.
+const int DptReadingUtils::DICT_OFFSET_INVALID = 0;
+const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF;
+
/* static */ int DptReadingUtils::getForwardLinkPosition(const uint8_t *const buffer,
const int pos) {
int linkAddressPos = pos;
return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos);
}
-/* static */ int DptReadingUtils::getParentPosAndAdvancePosition(const uint8_t *const buffer,
- int *const pos) {
+/* static */ int DptReadingUtils::getParentPtNodePosOffsetAndAdvancePosition(
+ const uint8_t *const buffer, int *const pos) {
return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos);
}
+/* static */ int DptReadingUtils::getParentPtNodePos(const int parentOffset, const int ptNodePos) {
+ if (parentOffset == DICT_OFFSET_INVALID) {
+ return NOT_A_DICT_POS;
+ } else if (parentOffset == DICT_OFFSET_ZERO_OFFSET) {
+ return ptNodePos;
+ } else {
+ return parentOffset + ptNodePos;
+ }
+}
+
/* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition(
const uint8_t *const buffer, int *const pos) {
const int base = *pos;
const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos);
- if (offset == 0) {
- // 0 offset means that the node does not have children.
+ if (offset == DICT_OFFSET_INVALID) {
+ // The PtNode does not have children.
return NOT_A_DICT_POS;
+ } else if (offset == DICT_OFFSET_ZERO_OFFSET) {
+ return base;
} else {
return base + offset;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h
index db5f9b1..67c3cc5 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h
@@ -27,13 +27,19 @@
public:
typedef uint8_t NodeFlags;
+ static const int DICT_OFFSET_INVALID;
+ static const int DICT_OFFSET_ZERO_OFFSET;
+
static int getForwardLinkPosition(const uint8_t *const buffer, const int pos);
static AK_FORCE_INLINE bool isValidForwardLinkPosition(const int forwardLinkAddress) {
return forwardLinkAddress != 0;
}
- static int getParentPosAndAdvancePosition(const uint8_t *const buffer, int *const pos);
+ static int getParentPtNodePosOffsetAndAdvancePosition(const uint8_t *const buffer,
+ int *const pos);
+
+ static int getParentPtNodePos(const int parentOffset, const int ptNodePos);
static int readChildrenPositionAndAdvancePosition(const uint8_t *const buffer, int *const pos);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
index 3fc9c4c..a51ae5e 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
@@ -28,12 +28,15 @@
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h"
+#include "utils/hash_map_compat.h"
namespace latinime {
const int DynamicPatriciaTrieWritingHelper::CHILDREN_POSITION_FIELD_SIZE = 3;
const char *const DynamicPatriciaTrieWritingHelper::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
".tmp";
+// TODO: Make MAX_DICTIONARY_SIZE 8MB.
+const size_t DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;
bool DynamicPatriciaTrieWritingHelper::addUnigramWord(
DynamicPatriciaTrieReadingHelper *const readingHelper,
@@ -153,7 +156,8 @@
if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */)) {
return;
}
- BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
+ BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */,
+ MAX_DICTIONARY_SIZE);
if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) {
return;
}
@@ -202,9 +206,8 @@
return false;
}
// Update moved position, which is stored in the parent offset field.
- const int movedPosOffset = movedPos - originalNode->getHeadPos();
- if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(
- mBuffer, movedPosOffset, &writingPos)) {
+ if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(
+ mBuffer, movedPos, originalNode->getHeadPos(), &writingPos)) {
return false;
}
// Update bigram linked node position, which is stored in the children position field.
@@ -219,11 +222,10 @@
const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader();
readingHelper.initWithPtNodeArrayPos(originalNode->getChildrenPos());
while (!readingHelper.isEnd()) {
- const int childPtNodeWrittenPos = nodeReader->getHeadPos();
- const int parentOffset = movedPos - childPtNodeWrittenPos;
- int parentOffsetFieldPos = childPtNodeWrittenPos + 1 /* Flags */;
- if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(
- mBuffer, parentOffset, &parentOffsetFieldPos)) {
+ int parentOffsetFieldPos = nodeReader->getHeadPos()
+ + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE;
+ if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(
+ mBuffer, movedPos, nodeReader->getHeadPos(), &parentOffsetFieldPos)) {
// Parent offset cannot be written because of a bug or a broken dictionary; thus,
// we give up to update dictionary.
return false;
@@ -249,9 +251,8 @@
return false;
}
// Calculate a parent offset and write the offset.
- const int parentOffset = (parentPos != NOT_A_DICT_POS) ? parentPos - nodePos : NOT_A_DICT_POS;
- if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(bufferToWrite,
- parentOffset, writingPos)) {
+ if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(bufferToWrite,
+ parentPos, nodePos, writingPos)) {
return false;
}
// Write code points
@@ -521,30 +522,48 @@
DynamicPatriciaTrieReadingHelper readingHelper(mBuffer, mBigramPolicy, mShortcutPolicy);
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners
- ::ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted
- listenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted(
+ ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
+ traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
this, mBuffer);
if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner(
- &listenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted)) {
+ &traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted)) {
return false;
}
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
- DynamicPatriciaTrieGcEventListeners::ListenerForUpdatingBigramProbability
- listenerForupdatingBigramProbability(mBigramPolicy);
+ DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability
+ traversePolicyToUpdateBigramProbability(mBigramPolicy);
if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner(
- &listenerForupdatingBigramProbability)) {
+ &traversePolicyToUpdateBigramProbability)) {
return false;
}
// Mapping from positions in mBuffer to positions in bufferToWrite.
- hash_map_compat<int, int> positionMap;
+ DictPositionRelocationMap dictPositionRelocationMap;
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
- DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer
- listenerForPlacingAndWritingLivingPtNodesToBuffer(this, mBuffer, &positionMap);
+ DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
+ traversePolicyToPlaceAndWriteValidPtNodesToBuffer(this, bufferToWrite,
+ &dictPositionRelocationMap);
+ if (!readingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
+ &traversePolicyToPlaceAndWriteValidPtNodesToBuffer)) {
+ return false;
+ }
- // TODO: Implement.
- return false;
+ // Create policy instance for the GCed dictionary.
+ DynamicShortcutListPolicy newDictShortcutPolicy(bufferToWrite);
+ DynamicBigramListPolicy newDictBigramPolicy(bufferToWrite, &newDictShortcutPolicy);
+ // Create reading helper for the GCed dictionary.
+ DynamicPatriciaTrieReadingHelper newDictReadingHelper(bufferToWrite, &newDictBigramPolicy,
+ &newDictShortcutPolicy);
+ newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
+ DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields
+ traversePolicyToUpdateAllPositionFields(this, &newDictBigramPolicy, bufferToWrite,
+ &dictPositionRelocationMap);
+ if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
+ &traversePolicyToUpdateAllPositionFields)) {
+ return false;
+ }
+ return true;
}
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
index e82b80a..028fa60 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
@@ -21,6 +21,7 @@
#include <stdint.h>
#include "defines.h"
+#include "utils/hash_map_compat.h"
namespace latinime {
@@ -33,6 +34,20 @@
class DynamicPatriciaTrieWritingHelper {
public:
+ typedef hash_map_compat<int, int> PtNodeArrayPositionRelocationMap;
+ typedef hash_map_compat<int, int> PtNodePositionRelocationMap;
+ struct DictPositionRelocationMap {
+ public:
+ DictPositionRelocationMap()
+ : mPtNodeArrayPositionRelocationMap(), mPtNodePositionRelocationMap() {}
+
+ PtNodeArrayPositionRelocationMap mPtNodeArrayPositionRelocationMap;
+ PtNodePositionRelocationMap mPtNodePositionRelocationMap;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(DictPositionRelocationMap);
+ };
+
DynamicPatriciaTrieWritingHelper(BufferWithExtendableBuffer *const buffer,
DynamicBigramListPolicy *const bigramPolicy,
DynamicShortcutListPolicy *const shortcutPolicy)
@@ -71,6 +86,7 @@
static const int CHILDREN_POSITION_FIELD_SIZE;
static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE;
+ static const size_t MAX_DICTIONARY_SIZE;
BufferWithExtendableBuffer *const mBuffer;
DynamicBigramListPolicy *const mBigramPolicy;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp
index b261e59..5a39837 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp
@@ -39,18 +39,20 @@
/* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int forwardLinkPos,
int *const forwardLinkFieldPos) {
- const int offset = (forwardLinkPos != NOT_A_DICT_POS) ?
- forwardLinkPos - (*forwardLinkFieldPos) : 0;
- return writeDictOffset(buffer, offset, forwardLinkFieldPos);
+ return writeDictOffset(buffer, forwardLinkPos, (*forwardLinkFieldPos), forwardLinkFieldPos);
}
/* static */ bool DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const size_t arraySize,
int *const arraySizeFieldPos) {
- if (arraySize <= MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD) {
+ // Currently, all array size field to be created has LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE to
+ // simplify updating process.
+ // TODO: Use SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE for small arrays.
+ /*if (arraySize <= MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD) {
return buffer->writeUintAndAdvancePosition(arraySize, SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE,
arraySizeFieldPos);
- } else if (arraySize <= MAX_PTNODE_ARRAY_SIZE) {
+ } else */
+ if (arraySize <= MAX_PTNODE_ARRAY_SIZE) {
uint32_t data = arraySize | LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG;
return buffer->writeUintAndAdvancePosition(data, LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE,
arraySizeFieldPos);
@@ -69,11 +71,10 @@
}
// Note that parentOffset is offset from node's head position.
-/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(
- BufferWithExtendableBuffer *const buffer, const int parentOffset,
+/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(
+ BufferWithExtendableBuffer *const buffer, const int parentPos, const int basePos,
int *const parentPosFieldPos) {
- int offset = (parentOffset != NOT_A_DICT_POS) ? parentOffset : 0;
- return writeDictOffset(buffer, offset, parentPosFieldPos);
+ return writeDictOffset(buffer, parentPos, basePos, parentPosFieldPos);
}
/* static */ bool DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(
@@ -106,13 +107,19 @@
/* static */ bool DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int childrenPosition,
int *const childrenPositionFieldPos) {
- int offset = (childrenPosition != NOT_A_DICT_POS) ?
- childrenPosition - (*childrenPositionFieldPos) : 0;
- return writeDictOffset(buffer, offset, childrenPositionFieldPos);
+ return writeDictOffset(buffer, childrenPosition, (*childrenPositionFieldPos),
+ childrenPositionFieldPos);
}
/* static */ bool DynamicPatriciaTrieWritingUtils::writeDictOffset(
- BufferWithExtendableBuffer *const buffer, const int offset, int *const offsetFieldPos) {
+ BufferWithExtendableBuffer *const buffer, const int targetPos, const int basePos,
+ int *const offsetFieldPos) {
+ int offset = targetPos - basePos;
+ if (targetPos == NOT_A_DICT_POS) {
+ offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID;
+ } else if (offset == 0) {
+ offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET;
+ }
if (offset > MAX_DICT_OFFSET_VALUE || offset < MIN_DICT_OFFSET_VALUE) {
AKLOGI("offset cannot be written because the offset is too large or too small: %d",
offset);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h
index 183ede4..a37e9fb 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h
@@ -28,6 +28,8 @@
class DynamicPatriciaTrieWritingUtils {
public:
+ static const int NODE_FLAG_FIELD_SIZE;
+
static bool writeForwardLinkPositionAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int forwardLinkPos,
int *const forwardLinkFieldPos);
@@ -39,8 +41,8 @@
const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags,
int *const nodeFlagsFieldPos);
- static bool writeParentOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer,
- const int parentPosition, int *const parentPosFieldPos);
+ static bool writeParentPosOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer,
+ const int parentPosition, const int basePos, int *const parentPosFieldPos);
static bool writeCodePointsAndAdvancePosition(BufferWithExtendableBuffer *const buffer,
const int *const codePoints, const int codePointCount, int *const codePointFieldPos);
@@ -63,11 +65,10 @@
static const int MAX_DICT_OFFSET_VALUE;
static const int MIN_DICT_OFFSET_VALUE;
static const int DICT_OFFSET_NEGATIVE_FLAG;
- static const int NODE_FLAG_FIELD_SIZE;
static const int PROBABILITY_FIELD_SIZE;
- static bool writeDictOffset(BufferWithExtendableBuffer *const buffer, const int offset,
- int *const offsetFieldPos);
+ static bool writeDictOffset(BufferWithExtendableBuffer *const buffer, const int targetPos,
+ const int basePos, int *const offsetFieldPos);
};
} // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp
index 0fed275..f692882 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp
@@ -18,9 +18,10 @@
namespace latinime {
-const size_t BufferWithExtendableBuffer::INITIAL_ADDITIONAL_BUFFER_SIZE = 16 * 1024;
const size_t BufferWithExtendableBuffer::MAX_ADDITIONAL_BUFFER_SIZE = 1024 * 1024;
-const size_t BufferWithExtendableBuffer::EXTEND_ADDITIONAL_BUFFER_SIZE_STEP = 16 * 1024;
+const int BufferWithExtendableBuffer::NEAR_BUFFER_LIMIT_THRESHOLD_PERCENTILE = 90;
+// TODO: Needs to allocate larger memory corresponding to the current vector size.
+const size_t BufferWithExtendableBuffer::EXTEND_ADDITIONAL_BUFFER_SIZE_STEP = 128 * 1024;
bool BufferWithExtendableBuffer::writeUintAndAdvancePosition(const uint32_t data, const int size,
int *const pos) {
@@ -64,6 +65,16 @@
return true;
}
+bool BufferWithExtendableBuffer::extendBuffer() {
+ const size_t sizeAfterExtending =
+ mAdditionalBuffer.size() + EXTEND_ADDITIONAL_BUFFER_SIZE_STEP;
+ if (sizeAfterExtending > mMaxAdditionalBufferSize) {
+ return false;
+ }
+ mAdditionalBuffer.resize(mAdditionalBuffer.size() + EXTEND_ADDITIONAL_BUFFER_SIZE_STEP);
+ return true;
+}
+
bool BufferWithExtendableBuffer::checkAndPrepareWriting(const int pos, const int size) {
if (isInAdditionalBuffer(pos)) {
const int tailPosition = getTailPosition();
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
index c6a4841..17d2e39 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
@@ -32,9 +32,11 @@
// raw pointer but provides several methods that handle boundary checking for writing data.
class BufferWithExtendableBuffer {
public:
- BufferWithExtendableBuffer(uint8_t *const originalBuffer, const int originalBufferSize)
+ BufferWithExtendableBuffer(uint8_t *const originalBuffer, const int originalBufferSize,
+ const int maxAdditionalBufferSize = MAX_ADDITIONAL_BUFFER_SIZE)
: mOriginalBuffer(originalBuffer), mOriginalBufferSize(originalBufferSize),
- mAdditionalBuffer(INITIAL_ADDITIONAL_BUFFER_SIZE), mUsedAdditionalBufferSize(0) {}
+ mAdditionalBuffer(EXTEND_ADDITIONAL_BUFFER_SIZE_STEP), mUsedAdditionalBufferSize(0),
+ mMaxAdditionalBufferSize(maxAdditionalBufferSize) {}
AK_FORCE_INLINE int getTailPosition() const {
return mOriginalBufferSize + mUsedAdditionalBufferSize;
@@ -61,6 +63,11 @@
return mOriginalBufferSize;
}
+ AK_FORCE_INLINE bool isNearSizeLimit() const {
+ return mAdditionalBuffer.size() >= ((mMaxAdditionalBufferSize
+ * NEAR_BUFFER_LIMIT_THRESHOLD_PERCENTILE) / 100);
+ }
+
/**
* For writing.
*
@@ -75,28 +82,22 @@
private:
DISALLOW_COPY_AND_ASSIGN(BufferWithExtendableBuffer);
- static const size_t INITIAL_ADDITIONAL_BUFFER_SIZE;
static const size_t MAX_ADDITIONAL_BUFFER_SIZE;
+ static const int NEAR_BUFFER_LIMIT_THRESHOLD_PERCENTILE;
static const size_t EXTEND_ADDITIONAL_BUFFER_SIZE_STEP;
uint8_t *const mOriginalBuffer;
const int mOriginalBufferSize;
std::vector<uint8_t> mAdditionalBuffer;
int mUsedAdditionalBufferSize;
+ const size_t mMaxAdditionalBufferSize;
// Return if the buffer is successfully extended or not.
- AK_FORCE_INLINE bool extendBuffer() {
- if (mAdditionalBuffer.size() + EXTEND_ADDITIONAL_BUFFER_SIZE_STEP
- > MAX_ADDITIONAL_BUFFER_SIZE) {
- return false;
- }
- mAdditionalBuffer.resize(mAdditionalBuffer.size() + EXTEND_ADDITIONAL_BUFFER_SIZE_STEP);
- return true;
- }
+ bool extendBuffer();
// Returns if it is possible to write size-bytes from pos. When pos is at the tail position of
// the additional buffer, try extending the buffer.
- AK_FORCE_INLINE bool checkAndPrepareWriting(const int pos, const int size);
+ bool checkAndPrepareWriting(const int pos, const int size);
};
}
#endif /* LATINIME_BUFFER_WITH_EXTENDABLE_BUFFER_H */
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
index 00d76c9..96a2217 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
@@ -18,6 +18,7 @@
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
+import android.util.Pair;
import com.android.inputmethod.latin.makedict.CodePointUtils;
import com.android.inputmethod.latin.makedict.DictEncoder;
@@ -384,4 +385,269 @@
dictFile.delete();
}
+
+ public void testFlushWithGCDictionary() {
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ } catch (UnsupportedFormatException e) {
+ fail("UnsupportedFormatException while writing an initial dictionary : " + e);
+ }
+ BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+
+ final int unigramProbability = 100;
+ final int bigramProbability = 10;
+ binaryDictionary.addUnigramWord("aaa", unigramProbability);
+ binaryDictionary.addUnigramWord("abb", unigramProbability);
+ binaryDictionary.addUnigramWord("bcc", unigramProbability);
+ binaryDictionary.addBigramWords("aaa", "abb", bigramProbability);
+ binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability);
+ binaryDictionary.addBigramWords("abb", "aaa", bigramProbability);
+ binaryDictionary.addBigramWords("abb", "bcc", bigramProbability);
+ binaryDictionary.flushWithGC();
+ binaryDictionary.close();
+
+ binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ final int probability = binaryDictionary.calculateProbability(unigramProbability,
+ bigramProbability);
+ assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
+ assertEquals(unigramProbability, binaryDictionary.getFrequency("abb"));
+ assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc"));
+ assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb"));
+ assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc"));
+ assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa"));
+ assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc"));
+ assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa"));
+ assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc"));
+ assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa"));
+ binaryDictionary.flushWithGC();
+ binaryDictionary.close();
+
+ dictFile.delete();
+ }
+
+ // TODO: Evaluate performance of GC
+ public void testAddBigramWordsAndFlashWithGC() {
+ final int wordCount = 100;
+ final int bigramCount = 1000;
+ final int codePointSetSize = 30;
+ // TODO: Use various seeds such as a current timestamp to make this test more random.
+ final int seed = 314159265;
+
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ } catch (UnsupportedFormatException e) {
+ fail("UnsupportedFormatException while writing an initial dictionary : " + e);
+ }
+
+ BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ final ArrayList<String> words = new ArrayList<String>();
+ // Test a word that isn't contained within the dictionary.
+ final Random random = new Random(seed);
+ final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
+ final int[] unigramProbabilities = new int[wordCount];
+ for (int i = 0; i < wordCount; ++i) {
+ final String word = CodePointUtils.generateWord(random, codePointSet);
+ words.add(word);
+ final int unigramProbability = random.nextInt(0xFF);
+ unigramProbabilities[i] = unigramProbability;
+ binaryDictionary.addUnigramWord(word, unigramProbability);
+ }
+
+ final int[][] probabilities = new int[wordCount][wordCount];
+
+ for (int i = 0; i < wordCount; ++i) {
+ for (int j = 0; j < wordCount; ++j) {
+ probabilities[i][j] = Dictionary.NOT_A_PROBABILITY;
+ }
+ }
+
+ for (int i = 0; i < bigramCount; i++) {
+ final int word0Index = random.nextInt(wordCount);
+ final int word1Index = random.nextInt(wordCount);
+ final String word0 = words.get(word0Index);
+ final String word1 = words.get(word1Index);
+ final int bigramProbability = random.nextInt(0xF);
+ probabilities[word0Index][word1Index] = binaryDictionary.calculateProbability(
+ unigramProbabilities[word1Index], bigramProbability);
+ binaryDictionary.addBigramWords(word0, word1, bigramProbability);
+ }
+
+ binaryDictionary.flushWithGC();
+ binaryDictionary.close();
+ binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+
+ for (int i = 0; i < words.size(); i++) {
+ for (int j = 0; j < words.size(); j++) {
+ assertEquals(probabilities[i][j],
+ binaryDictionary.getBigramProbability(words.get(i), words.get(j)));
+ }
+ }
+ dictFile.delete();
+ }
+
+ public void testRandomOperetionsAndFlashWithGC() {
+ final int flashWithGCIterationCount = 50;
+ final int operationCountInEachIteration = 200;
+ final int initialUnigramCount = 100;
+ final float addUnigramProb = 0.5f;
+ final float addBigramProb = 0.8f;
+ final float removeBigramProb = 0.2f;
+ final int codePointSetSize = 30;
+ final int seed = 141421356;
+
+ final Random random = new Random(seed);
+
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ } catch (UnsupportedFormatException e) {
+ fail("UnsupportedFormatException while writing an initial dictionary : " + e);
+ }
+
+ BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ final ArrayList<String> words = new ArrayList<String>();
+ final ArrayList<Pair<String, String>> bigramWords = new ArrayList<Pair<String,String>>();
+ final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
+ final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>();
+ final HashMap<Pair<String, String>, Integer> bigramProbabilities =
+ new HashMap<Pair<String, String>, Integer>();
+ for (int i = 0; i < initialUnigramCount; ++i) {
+ final String word = CodePointUtils.generateWord(random, codePointSet);
+ words.add(word);
+ final int unigramProbability = random.nextInt(0xFF);
+ unigramProbabilities.put(word, unigramProbability);
+ binaryDictionary.addUnigramWord(word, unigramProbability);
+ }
+ binaryDictionary.flushWithGC();
+ binaryDictionary.close();
+
+ for (int gcCount = 0; gcCount < flashWithGCIterationCount; gcCount++) {
+ binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ for (int opCount = 0; opCount < operationCountInEachIteration; opCount++) {
+ // Add unigram.
+ if (random.nextFloat() < addUnigramProb) {
+ final String word = CodePointUtils.generateWord(random, codePointSet);
+ words.add(word);
+ final int unigramProbability = random.nextInt(0xFF);
+ unigramProbabilities.put(word, unigramProbability);
+ binaryDictionary.addUnigramWord(word, unigramProbability);
+ }
+ // Add bigram.
+ if (random.nextFloat() < addBigramProb && words.size() > 2) {
+ final int word0Index = random.nextInt(words.size());
+ int word1Index = random.nextInt(words.size() - 1);
+ if (word0Index <= word1Index) {
+ word1Index++;
+ }
+ final String word0 = words.get(word0Index);
+ final String word1 = words.get(word1Index);
+ final int bigramProbability = random.nextInt(0xF);
+ final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
+ bigramWords.add(bigram);
+ bigramProbabilities.put(bigram, bigramProbability);
+ binaryDictionary.addBigramWords(word0, word1, bigramProbability);
+ }
+ // Remove bigram.
+ if (random.nextFloat() < removeBigramProb && !bigramWords.isEmpty()) {
+ final int bigramIndex = random.nextInt(bigramWords.size());
+ final Pair<String, String> bigram = bigramWords.get(bigramIndex);
+ bigramWords.remove(bigramIndex);
+ bigramProbabilities.remove(bigram);
+ binaryDictionary.removeBigramWords(bigram.first, bigram.second);
+ }
+ }
+
+ // Test whether the all unigram operations are collectlly handled.
+ for (int i = 0; i < words.size(); i++) {
+ final String word = words.get(i);
+ final int unigramProbability = unigramProbabilities.get(word);
+ assertEquals(word, unigramProbability, binaryDictionary.getFrequency(word));
+ }
+ // Test whether the all bigram operations are collectlly handled.
+ for (int i = 0; i < bigramWords.size(); i++) {
+ final Pair<String, String> bigram = bigramWords.get(i);
+ final int unigramProbability = unigramProbabilities.get(bigram.second);
+ final int probability;
+ if (bigramProbabilities.containsKey(bigram)) {
+ final int bigramProbability = bigramProbabilities.get(bigram);
+ probability = binaryDictionary.calculateProbability(unigramProbability,
+ bigramProbability);
+ } else {
+ probability = Dictionary.NOT_A_PROBABILITY;
+ }
+ assertEquals(probability,
+ binaryDictionary.getBigramProbability(bigram.first, bigram.second));
+ }
+ binaryDictionary.flushWithGC();
+ binaryDictionary.close();
+ }
+
+ dictFile.delete();
+ }
+
+ public void testAddManyUnigramsAndFlushWithGC() {
+ final int flashWithGCIterationCount = 3;
+ final int codePointSetSize = 50;
+ final int seed = 22360679;
+
+ final Random random = new Random(seed);
+
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ } catch (UnsupportedFormatException e) {
+ fail("UnsupportedFormatException while writing an initial dictionary : " + e);
+ }
+
+ final ArrayList<String> words = new ArrayList<String>();
+ final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>();
+ final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
+
+ BinaryDictionary binaryDictionary;
+ for (int i = 0; i < flashWithGCIterationCount; i++) {
+ binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ while(!binaryDictionary.needsToRunGC()) {
+ final String word = CodePointUtils.generateWord(random, codePointSet);
+ words.add(word);
+ final int unigramProbability = random.nextInt(0xFF);
+ unigramProbabilities.put(word, unigramProbability);
+ binaryDictionary.addUnigramWord(word, unigramProbability);
+ }
+
+ for (int j = 0; j < words.size(); j++) {
+ final String word = words.get(j);
+ final int unigramProbability = unigramProbabilities.get(word);
+ assertEquals(word, unigramProbability, binaryDictionary.getFrequency(word));
+ }
+
+ binaryDictionary.flushWithGC();
+ binaryDictionary.close();
+ }
+
+ dictFile.delete();
+ }
}
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
index cedd0df..a4d9426 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
@@ -494,7 +494,7 @@
formatOptions, "unigram"));
results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, bufferType,
formatOptions, "chain"));
- results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, bufferType,
+ results.add(runReadUnigramsAndBigramsBinary(sWords, sStarBigrams, bufferType,
formatOptions, "star"));
}
diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
index bf44a14..d605cdb 100644
--- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
@@ -100,7 +100,11 @@
Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS));
} catch (InterruptedException e) {
}
- for (int i = 0; i < 10 && i < numberOfWords; ++i) {
+ // Limit word count to check when using a Java on memory dictionary.
+ final int wordCountToCheck =
+ ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
+ numberOfWords : 10;
+ for (int i = 0; i < wordCountToCheck; ++i) {
final String word = words.get(i);
// This may fail as long as we use tryLock on inserting the bigram words
assertTrue(dict.isInDictionaryForTests(word));
@@ -202,4 +206,41 @@
}
}
}
+
+ public void testAddManyWords() {
+ File dictFile = null;
+ final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis();
+ final int numberOfWords =
+ ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
+ 10000 : 1000;
+ final Random random = new Random(123456);
+
+ UserHistoryPredictionDictionary dict =
+ PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+ testFilenameSuffix, mPrefs);
+ try {
+ addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random,
+ true /* checksContents */);
+ dict.close();
+ } finally {
+ try {
+ Log.d(TAG, "waiting for writing ...");
+ dict.shutdownExecutorForTests();
+ while (!dict.isTerminatedForTests()) {
+ Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS);
+ }
+ } catch (InterruptedException e) {
+ Log.d(TAG, "InterruptedException: ", e);
+ }
+ final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
+ + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
+ dictFile = new File(getContext().getFilesDir(), fileName);
+ if (dictFile != null) {
+ assertTrue(dictFile.exists());
+ assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
+ dictFile.delete();
+ }
+ }
+ }
+
}
diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
index 4e396a1..eb9fb98 100644
--- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
+++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
@@ -20,6 +20,11 @@
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.SmallTest;
+import android.text.style.SuggestionSpan;
+import android.text.style.URLSpan;
+import android.text.SpannableStringBuilder;
+import android.text.Spannable;
+import android.text.Spanned;
import java.util.Arrays;
import java.util.List;
@@ -280,4 +285,34 @@
assertEquals(objs[i], newObjArray.get(i));
}
}
+
+ public void testConcatWithSuggestionSpansOnly() {
+ SpannableStringBuilder s = new SpannableStringBuilder("test string\ntest string\n"
+ + "test string\ntest string\ntest string\ntest string\ntest string\ntest string\n"
+ + "test string\ntest string\n");
+ final int N = 10;
+ for (int i = 0; i < N; ++i) {
+ // Put a PARAGRAPH-flagged span that should not be found in the result.
+ s.setSpan(new SuggestionSpan(getContext(),
+ new String[] {"" + i}, Spannable.SPAN_PARAGRAPH),
+ i * 12, i * 12 + 12, Spannable.SPAN_PARAGRAPH);
+ // Put a normal suggestion span that should be found in the result.
+ s.setSpan(new SuggestionSpan(getContext(), new String[] {"" + i}, 0), i, i * 2, 0);
+ // Put a URL span than should not be found in the result.
+ s.setSpan(new URLSpan("http://a"), i, i * 2, 0);
+ }
+
+ final CharSequence a = s.subSequence(0, 15);
+ final CharSequence b = s.subSequence(15, s.length());
+ final Spanned result =
+ (Spanned)StringUtils.concatWithNonParagraphSuggestionSpansOnly(a, b);
+
+ Object[] spans = result.getSpans(0, result.length(), SuggestionSpan.class);
+ for (int i = 0; i < spans.length; i++) {
+ final int flags = result.getSpanFlags(spans[i]);
+ assertEquals("Should not find a span with PARAGRAPH flag",
+ flags & Spannable.SPAN_PARAGRAPH, 0);
+ assertTrue("Should be a SuggestionSpan", spans[i] instanceof SuggestionSpan);
+ }
+ }
}
diff --git a/tools/make-keyboard-text/res/values-hy/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-hy/donottranslate-more-keys.xml
index f6c6428..2f34128 100644
--- a/tools/make-keyboard-text/res/values-hy/donottranslate-more-keys.xml
+++ b/tools/make-keyboard-text/res/values-hy/donottranslate-more-keys.xml
@@ -18,7 +18,23 @@
*/
-->
<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <!-- U+058A: "֊" ARMENIAN HYPHEN -->
+ <!-- U+055C: "՜" ARMENIAN EXCLAMATION MARK -->
+ <!-- U+055D: "՝" ARMENIAN COMMA -->
<!-- U+055E: "՞" ARMENIAN QUESTION MARK -->
- <string name="more_keys_for_punctuation">"!fixedColumnOrder!4,՞,!,\\,,\?,:,;,\@"</string>
- <string name="more_keys_for_tablet_period">՞,\?</string>
+ <!-- U+0559: "ՙ" ARMENIAN MODIFIER LETTER LEFT HALF RING -->
+ <!-- U+055A: "՚" ARMENIAN APOSTROPHE -->
+ <!-- U+055B: "՛" ARMENIAN EMPHASIS MARK -->
+ <!-- U+055F: "՟" ARMENIAN ABBREVIATION MARK -->
+ <string name="more_keys_for_punctuation">"!fixedColumnOrder!8,!,?,\\,,.,֊,՜,՝,՞,:,;,\@,ՙ,՚,՛,՟"</string>
+ <!-- U+055E: "՞" ARMENIAN QUESTION MARK -->
+ <!-- U+00BF: "¿" INVERTED QUESTION MARK -->
+ <string name="more_keys_for_symbols_question">՞,¿</string>
+ <!-- U+055C: "՜" ARMENIAN EXCLAMATION MARK -->
+ <!-- U+00A1: "¡" INVERTED EXCLAMATION MARK -->
+ <string name="more_keys_for_symbols_exclamation">՜,¡</string>
+ <!-- U+058F: "֏" ARMENIAN DRAM SIGN -->
+ <!-- TODO: Enable this when we have glyph for the following letter
+ <string name="keylabel_for_currency">֏</string>
+ -->
</resources>
