Merge "Try decaying user history at hourly intervals."
diff --git a/java/res/drawable-hdpi/sym_keyboard_smiley_holo_dark.png b/java/res/drawable-hdpi/sym_keyboard_smiley_holo_dark.png
new file mode 100644
index 0000000..cfacbc2
--- /dev/null
+++ b/java/res/drawable-hdpi/sym_keyboard_smiley_holo_dark.png
Binary files differ
diff --git a/java/res/drawable-mdpi/sym_keyboard_smiley_holo_dark.png b/java/res/drawable-mdpi/sym_keyboard_smiley_holo_dark.png
index 71272bb..067ad54 100644
--- a/java/res/drawable-mdpi/sym_keyboard_smiley_holo_dark.png
+++ b/java/res/drawable-mdpi/sym_keyboard_smiley_holo_dark.png
Binary files differ
diff --git a/java/res/drawable-xhdpi/sym_keyboard_smiley_holo_dark.png b/java/res/drawable-xhdpi/sym_keyboard_smiley_holo_dark.png
index 686831f..e6baa2e 100644
--- a/java/res/drawable-xhdpi/sym_keyboard_smiley_holo_dark.png
+++ b/java/res/drawable-xhdpi/sym_keyboard_smiley_holo_dark.png
Binary files differ
diff --git a/java/res/drawable-xxhdpi/sym_keyboard_smiley_holo_dark.png b/java/res/drawable-xxhdpi/sym_keyboard_smiley_holo_dark.png
index 04b7216..5973ac3 100644
--- a/java/res/drawable-xxhdpi/sym_keyboard_smiley_holo_dark.png
+++ b/java/res/drawable-xxhdpi/sym_keyboard_smiley_holo_dark.png
Binary files differ
diff --git a/java/res/layout/emoji_keyboard_tab_icon.xml b/java/res/layout/emoji_keyboard_tab_icon.xml
index d79276e..1609f6a 100644
--- a/java/res/layout/emoji_keyboard_tab_icon.xml
+++ b/java/res/layout/emoji_keyboard_tab_icon.xml
@@ -23,4 +23,5 @@
android:layout_weight="1.0"
android:layout_height="wrap_content"
android:gravity="center"
+ android:scaleType="center"
/>
diff --git a/java/res/values-v19/emoji-categories.xml b/java/res/values-v19/emoji-categories.xml
index 658bbfa..0d5aa1a 100644
--- a/java/res/values-v19/emoji-categories.xml
+++ b/java/res/values-v19/emoji-categories.xml
@@ -204,18 +204,18 @@
name="emoji_symbols"
format="string"
>
- <item>fe82e|0031,20e3</item>
- <item>fe82f|0032,20e3</item>
- <item>fe830|0033,20e3</item>
- <item>fe831|0034,20e3</item>
- <item>fe832|0035,20e3</item>
- <item>fe833|0036,20e3</item>
- <item>fe834|0037,20e3</item>
- <item>fe835|0038,20e3</item>
- <item>fe836|0039,20e3</item>
- <item>fe837|0030,20e3</item>
+ <item>fe82e|0031,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe82f|0032,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe830|0033,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe831|0034,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe832|0035,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe833|0036,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe834|0037,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe835|0038,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe836|0039,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe837|0030,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
<item>1f51f</item>
- <item>fe82c|0023,20e3</item>
+ <item>fe82c|0023,20e3|99</item> <!-- TODO: fix support min sdk version (99) -->
<item>1f51d</item>
<item>1f519</item>
<item>1f51b</item>
@@ -875,16 +875,16 @@
<item>1f48e</item>
<item>1f490</item>
<item>1f492</item>
- <item>fe4e5|1f1ef,1f1f5</item>
- <item>fe4e6|1f1fa,1f1f8</item>
- <item>fe4e7|1f1eb,1f1f7</item>
- <item>fe4e8|1f1e9,1f1ea</item>
- <item>fe4e9|1f1ee,1f1f9</item>
- <item>fe4ea|1f1ec,1f1e7</item>
- <item>fe4eb|1f1ea,1f1f8</item>
- <item>fe4ec|1f1f7,1f1fa</item>
- <item>fe4ed|1f1e8,1f1f3</item>
- <item>fe4ee|1f1f0,1f1f7</item>
+ <item>fe4e5|1f1ef,1f1f5|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe4e6|1f1fa,1f1f8|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe4e7|1f1eb,1f1f7|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe4e8|1f1e9,1f1ea|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe4e9|1f1ee,1f1f9|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe4ea|1f1ec,1f1e7|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe4eb|1f1ea,1f1f8|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe4ec|1f1f7,1f1fa|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe4ed|1f1e8,1f1f3|99</item> <!-- TODO: fix support min sdk version (99) -->
+ <item>fe4ee|1f1f0,1f1f7|99</item> <!-- TODO: fix support min sdk version (99) -->
</array>
<array
name="emoji_emoticons"
diff --git a/java/res/xml-sw600dp/keys_comma_period.xml b/java/res/xml-sw600dp/keys_comma_period.xml
new file mode 100644
index 0000000..7604e03
--- /dev/null
+++ b/java/res/xml-sw600dp/keys_comma_period.xml
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+/*
+**
+** Copyright 2013, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+-->
+
+<merge
+ xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin"
+>
+ <switch>
+ <case
+ latin:mode="email|url"
+ >
+ <Key
+ latin:keyLabel=","
+ latin:keyHintLabel="-"
+ latin:moreKeys="-"
+ latin:backgroundType="functional"
+ latin:keyStyle="hasShiftedLetterHintStyle" />
+ <Key
+ latin:keyLabel="."
+ latin:keyHintLabel="_"
+ latin:moreKeys="_"
+ latin:backgroundType="functional"
+ latin:keyStyle="hasShiftedLetterHintStyle" />
+ </case>
+ <case
+ latin:languageCode="ar"
+ >
+ <Key
+ latin:keyLabel="!text/keylabel_for_apostrophe"
+ latin:keyHintLabel="!text/keyhintlabel_for_apostrophe"
+ latin:moreKeys="!text/more_keys_for_apostrophe"
+ latin:backgroundType="functional"
+ latin:keyStyle="hasShiftedLetterHintStyle" />
+ <Key
+ latin:keyLabel="."
+ latin:keyHintLabel="!text/keyhintlabel_for_arabic_diacritics"
+ latin:keyLabelFlags="hasPopupHint"
+ latin:moreKeys="!text/more_keys_for_arabic_diacritics"
+ latin:backgroundType="functional"
+ latin:keyStyle="hasShiftedLetterHintStyle" />
+ </case>
+ <case
+ latin:languageCode="fa"
+ >
+ <Key
+ latin:keyLabel="!text/keylabel_for_apostrophe"
+ latin:keyHintLabel="!text/keyhintlabel_for_apostrophe"
+ latin:keyLabelFlags="hasPopupHint"
+ latin:moreKeys="!text/more_keys_for_apostrophe"
+ latin:backgroundType="functional"
+ latin:keyStyle="hasShiftedLetterHintStyle" />
+ <Key
+ latin:keyLabel="."
+ latin:keyHintLabel="!text/keyhintlabel_for_arabic_diacritics"
+ latin:keyLabelFlags="hasPopupHint"
+ latin:moreKeys="!text/more_keys_for_arabic_diacritics"
+ latin:backgroundType="functional"
+ latin:keyStyle="hasShiftedLetterHintStyle" />
+ </case>
+ <case
+ latin:languageCode="hy"
+ >
+ <!-- U+055D: "՝" ARMENIAN COMMA -->
+ <Key
+ latin:keyLabel="՝"
+ latin:backgroundType="functional" />
+ <!-- U+0589: "։" ARMENIAN FULL STOP -->
+ <Key
+ latin:keyLabel="։"
+ latin:keyLabelFlags="hasPopupHint"
+ latin:backgroundType="functional"
+ latin:moreKeys="!text/more_keys_for_punctuation" />
+ </case>
+ <default>
+ <Key
+ latin:keyLabel="!text/keylabel_for_tablet_comma"
+ latin:keyHintLabel="!text/keyhintlabel_for_tablet_comma"
+ latin:backgroundType="functional"
+ latin:moreKeys="!text/more_keys_for_tablet_comma" />
+ <Key
+ latin:keyLabel="."
+ latin:keyHintLabel="!text/keyhintlabel_for_period"
+ latin:backgroundType="functional"
+ latin:moreKeys="!text/more_keys_for_period" />
+ </default>
+ </switch>
+</merge>
diff --git a/java/res/xml-sw600dp/keys_exclamation_question.xml b/java/res/xml-sw600dp/keys_exclamation_question.xml
index 983ef38..cd38282 100644
--- a/java/res/xml-sw600dp/keys_exclamation_question.xml
+++ b/java/res/xml-sw600dp/keys_exclamation_question.xml
@@ -22,7 +22,7 @@
xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin"
>
<Key
- latin:keyLabel="\?" />
- <Key
latin:keyLabel="!" />
+ <Key
+ latin:keyLabel="\?" />
</merge>
diff --git a/java/res/xml/keys_comma_period.xml b/java/res/xml/keys_comma_period.xml
index 02b46c2..1b51e45 100644
--- a/java/res/xml/keys_comma_period.xml
+++ b/java/res/xml/keys_comma_period.xml
@@ -23,22 +23,6 @@
>
<switch>
<case
- latin:mode="email|url"
- >
- <Key
- latin:keyLabel="."
- latin:keyHintLabel="_"
- latin:moreKeys="_"
- latin:backgroundType="functional"
- latin:keyStyle="hasShiftedLetterHintStyle" />
- <Key
- latin:keyLabel=","
- latin:keyHintLabel="-"
- latin:moreKeys="-"
- latin:backgroundType="functional"
- latin:keyStyle="hasShiftedLetterHintStyle" />
- </case>
- <case
latin:languageCode="ar"
>
<Key
@@ -76,28 +60,28 @@
<case
latin:languageCode="hy"
>
+ <!-- U+055D: "՝" ARMENIAN COMMA -->
+ <Key
+ latin:keyLabel="՝"
+ latin:backgroundType="functional" />
<!-- U+0589: "։" ARMENIAN FULL STOP -->
<Key
latin:keyLabel="։"
latin:keyLabelFlags="hasPopupHint"
latin:backgroundType="functional"
latin:moreKeys="!text/more_keys_for_punctuation" />
- <!-- U+055D: "՝" ARMENIAN COMMA -->
- <Key
- latin:keyLabel="՝"
- latin:backgroundType="functional" />
</case>
<default>
<Key
- latin:keyLabel="."
- latin:keyHintLabel="!text/keyhintlabel_for_tablet_period"
- latin:backgroundType="functional"
- latin:moreKeys="!text/more_keys_for_tablet_period" />
- <Key
latin:keyLabel="!text/keylabel_for_tablet_comma"
latin:keyHintLabel="!text/keyhintlabel_for_tablet_comma"
latin:backgroundType="functional"
latin:moreKeys="!text/more_keys_for_tablet_comma" />
+ <Key
+ latin:keyLabel="."
+ latin:keyHintLabel="!text/keyhintlabel_for_period"
+ latin:backgroundType="functional"
+ latin:moreKeys="!text/more_keys_for_period" />
</default>
</switch>
</merge>
diff --git a/java/res/xml/rowkeys_symbols3.xml b/java/res/xml/rowkeys_symbols3.xml
index 9f5e620..074078c 100644
--- a/java/res/xml/rowkeys_symbols3.xml
+++ b/java/res/xml/rowkeys_symbols3.xml
@@ -54,9 +54,9 @@
latin:keyLabel="!text/keylabel_for_symbols_semicolon"
latin:moreKeys="!text/more_keys_for_symbols_semicolon" />
<Key
- latin:keyLabel="!text/keylabel_for_symbols_question"
- latin:moreKeys="!text/more_keys_for_symbols_question" />
- <Key
latin:keyLabel="!"
latin:moreKeys="!text/more_keys_for_symbols_exclamation" />
+ <Key
+ latin:keyLabel="!text/keylabel_for_symbols_question"
+ latin:moreKeys="!text/more_keys_for_symbols_question" />
</merge>
diff --git a/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsActivity.java b/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsActivity.java
index 6841652..c28d729 100644
--- a/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsActivity.java
+++ b/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsActivity.java
@@ -24,6 +24,8 @@
* Preference screen.
*/
public final class DictionarySettingsActivity extends PreferenceActivity {
+ private static final String DEFAULT_FRAGMENT = DictionarySettingsFragment.class.getName();
+
@Override
protected void onCreate(final Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
@@ -32,11 +34,17 @@
@Override
public Intent getIntent() {
final Intent modIntent = new Intent(super.getIntent());
- modIntent.putExtra(EXTRA_SHOW_FRAGMENT, DictionarySettingsFragment.class.getName());
+ modIntent.putExtra(EXTRA_SHOW_FRAGMENT, DEFAULT_FRAGMENT);
modIntent.putExtra(EXTRA_NO_HEADERS, true);
// Important note : the original intent should contain a String extra with the key
// DictionarySettingsFragment.DICT_SETTINGS_FRAGMENT_CLIENT_ID_ARGUMENT so that the
// fragment can know who the client is.
return modIntent;
}
+
+ // TODO: Uncomment the override annotation once we start using SDK version 19.
+ // @Override
+ public boolean isValidFragment(String fragmentName) {
+ return fragmentName.equals(DEFAULT_FRAGMENT);
+ }
}
diff --git a/java/src/com/android/inputmethod/keyboard/EmojiCategoryPageIndicatorView.java b/java/src/com/android/inputmethod/keyboard/EmojiCategoryPageIndicatorView.java
index fed134e..e23131a 100644
--- a/java/src/com/android/inputmethod/keyboard/EmojiCategoryPageIndicatorView.java
+++ b/java/src/com/android/inputmethod/keyboard/EmojiCategoryPageIndicatorView.java
@@ -50,8 +50,9 @@
@Override
protected void onDraw(Canvas canvas) {
- if (mCategoryPageSize == 0) {
- // If the category is not set yet, just clear and return.
+ if (mCategoryPageSize <= 1) {
+ // If the category is not set yet or contains only one category,
+ // just clear and return.
canvas.drawColor(0);
return;
}
diff --git a/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java b/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java
index 267fad5..71790b7 100644
--- a/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java
+++ b/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java
@@ -75,9 +75,7 @@
public void setActionBarProperties(LinearLayout ll) {
final LinearLayout.LayoutParams lp = (LinearLayout.LayoutParams) ll.getLayoutParams();
- lp.height = mEmojiActionBarHeight;
- lp.topMargin = 0;
- lp.bottomMargin = mBottomPadding;
+ lp.height = mEmojiActionBarHeight - mBottomPadding;
ll.setLayoutParams(lp);
}
diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
index 74edd87..ad6e2c0 100644
--- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
+++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
@@ -155,7 +155,7 @@
}
public void saveKeyboardState() {
- if (getKeyboard() != null) {
+ if (getKeyboard() != null || isShowingEmojiKeyboard()) {
mState.onSaveKeyboardState();
}
}
diff --git a/java/src/com/android/inputmethod/keyboard/PointerTracker.java b/java/src/com/android/inputmethod/keyboard/PointerTracker.java
index ee4ac95..52f190e 100644
--- a/java/src/com/android/inputmethod/keyboard/PointerTracker.java
+++ b/java/src/com/android/inputmethod/keyboard/PointerTracker.java
@@ -823,14 +823,16 @@
final int size = sAggregratedPointers.getPointerSize();
if (size > sLastRecognitionPointSize
&& stroke.hasRecognitionTimePast(eventTime, sLastRecognitionTime)) {
- sLastRecognitionPointSize = size;
- sLastRecognitionTime = eventTime;
if (DEBUG_LISTENER) {
Log.d(TAG, String.format("[%d] onUpdateBatchInput: batchPoints=%d", mPointerId,
size));
}
mTimerProxy.startUpdateBatchInputTimer(this);
mListener.onUpdateBatchInput(sAggregratedPointers);
+ // The listener may change the size of the pointers (when auto-committing
+ // for example), so we need to get the size from the pointers again.
+ sLastRecognitionPointSize = sAggregratedPointers.getPointerSize();
+ sLastRecognitionTime = eventTime;
}
}
}
diff --git a/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java b/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java
index c10fdba..4ccecb2 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java
@@ -18,6 +18,8 @@
import com.android.inputmethod.latin.Constants;
+import android.text.TextUtils;
+
/**
* The string parser of codesArray specification for <GridRows />. The attribute codesArray is an
* array of string.
@@ -34,7 +36,7 @@
public final class CodesArrayParser {
// Constants for parsing.
private static final char COMMA = ',';
- private static final char VERTICAL_BAR = '|';
+ private static final String VERTICAL_BAR_STRING = "\\|";
private static final String COMMA_STRING = ",";
private static final int BASE_HEX = 16;
@@ -43,8 +45,11 @@
}
private static String getLabelSpec(final String codesArraySpec) {
- final int pos = codesArraySpec.indexOf(VERTICAL_BAR);
- return (pos < 0) ? codesArraySpec : codesArraySpec.substring(0, pos);
+ final String[] strs = codesArraySpec.split(VERTICAL_BAR_STRING, -1);
+ if (strs.length <= 1) {
+ return codesArraySpec;
+ }
+ return strs[0];
}
public static String parseLabel(final String codesArraySpec) {
@@ -58,8 +63,25 @@
}
private static String getCodeSpec(final String codesArraySpec) {
- final int pos = codesArraySpec.indexOf(VERTICAL_BAR);
- return (pos < 0) ? codesArraySpec : codesArraySpec.substring(pos + 1);
+ final String[] strs = codesArraySpec.split(VERTICAL_BAR_STRING, -1);
+ if (strs.length <= 1) {
+ return codesArraySpec;
+ }
+ return TextUtils.isEmpty(strs[1]) ? strs[0] : strs[1];
+ }
+
+ // codesArraySpec consists of:
+ // <label>|<code0>,<code1>,...|<minSupportSdkVersion>
+ public static int getMinSupportSdkVersion(final String codesArraySpec) {
+ final String[] strs = codesArraySpec.split(VERTICAL_BAR_STRING, -1);
+ if (strs.length <= 2) {
+ return 0;
+ }
+ try {
+ return Integer.parseInt(strs[2]);
+ } catch (NumberFormatException e) {
+ return 0;
+ }
}
public static int parseCode(final String codesArraySpec) {
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java
index 22f7a83..c1ae656 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java
@@ -20,6 +20,7 @@
import android.content.res.Resources;
import android.content.res.TypedArray;
import android.content.res.XmlResourceParser;
+import android.os.Build;
import android.util.AttributeSet;
import android.util.Log;
import android.util.TypedValue;
@@ -436,17 +437,24 @@
final String label;
final int code;
final String outputText;
+ final int supportedMinSdkVersion;
if (codesArrayId != 0) {
final String codeArraySpec = array[i];
label = CodesArrayParser.parseLabel(codeArraySpec);
code = CodesArrayParser.parseCode(codeArraySpec);
outputText = CodesArrayParser.parseOutputText(codeArraySpec);
+ supportedMinSdkVersion =
+ CodesArrayParser.getMinSupportSdkVersion(codeArraySpec);
} else {
final String textArraySpec = array[i];
// TODO: Utilize KeySpecParser or write more generic TextsArrayParser.
label = textArraySpec;
code = Constants.CODE_OUTPUT_TEXT;
outputText = textArraySpec + (char)Constants.CODE_SPACE;
+ supportedMinSdkVersion = 0;
+ }
+ if (Build.VERSION.SDK_INT < supportedMinSdkVersion) {
+ continue;
}
final int x = (int)row.getKeyX(null);
final int y = row.getKeyY();
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
index 9f9fdaa..506dfa7 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
@@ -178,6 +178,8 @@
if (!state.mIsAlphabetShiftLocked) {
setShifted(state.mShiftMode);
}
+ // TODO: is this the right place to do this? Should we do this in setShift* instead?
+ mSwitchActions.requestUpdatingShiftState();
} else {
mPrevMainKeyboardWasShiftLocked = state.mIsAlphabetShiftLocked;
}
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java
index 67553fb..684cf63 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java
@@ -209,8 +209,8 @@
/* 104 */ "keylabel_for_tablet_comma",
/* 105 */ "keyhintlabel_for_tablet_comma",
/* 106 */ "more_keys_for_tablet_comma",
- /* 107 */ "keyhintlabel_for_tablet_period",
- /* 108 */ "more_keys_for_tablet_period",
+ /* 107 */ "keyhintlabel_for_period",
+ /* 108 */ "more_keys_for_period",
/* 109 */ "keylabel_for_apostrophe",
/* 110 */ "keyhintlabel_for_apostrophe",
/* 111 */ "more_keys_for_apostrophe",
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index d9bad7e..541e697 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -44,9 +44,9 @@
private static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
// Must be equal to MAX_RESULTS in native/jni/src/defines.h
private static final int MAX_RESULTS = 18;
- // Required space count for auto commit.
- // TODO: Remove this heuristic.
- private static final int SPACE_COUNT_FOR_AUTO_COMMIT = 3;
+ // The cutoff returned by native for auto-commit confidence.
+ // Must be equal to CONFIDENCE_TO_AUTO_COMMIT in native/jni/src/defines.h
+ private static final int CONFIDENCE_TO_AUTO_COMMIT = 1000000;
@UsedForTesting
public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
@@ -343,18 +343,7 @@
@Override
public boolean shouldAutoCommit(final SuggestedWordInfo candidate) {
- // TODO: actually use the confidence rather than use this completely broken heuristic
- final String word = candidate.mWord;
- final int length = word.length();
- int remainingSpaces = SPACE_COUNT_FOR_AUTO_COMMIT;
- for (int i = 0; i < length; ++i) {
- // This is okay because no low-surrogate and no high-surrogate can ever match the
- // space character, so we don't need to take care of iterating on code points.
- if (Constants.CODE_SPACE == word.charAt(i)) {
- if (0 >= --remainingSpaces) return true;
- }
- }
- return false;
+ return candidate.mAutoCommitFirstWordConfidence > CONFIDENCE_TO_AUTO_COMMIT;
}
@Override
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index 5b7e935..0f3d289 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -2932,6 +2932,7 @@
return;
}
tryFixLyingCursorPosition();
+ mKeyboardSwitcher.updateShiftState();
if (tryResumeSuggestions) mHandler.postResumeSuggestions();
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
index 665c7a2..2c3d134 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
@@ -295,7 +295,6 @@
return address;
}
}
- int address;
switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) {
case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE:
return dictBuffer.readUnsignedByte();
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
index af61f29..b602424 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
@@ -278,7 +278,6 @@
// For future reference, the code to remove duplicate is a simple : list.remove(node);
list.add(ptNodeArray);
final ArrayList<PtNode> branches = ptNodeArray.mData;
- final int nodeSize = branches.size();
for (PtNode ptNode : branches) {
if (null != ptNode.mChildren) flattenTreeInner(list, ptNode.mChildren);
}
@@ -427,9 +426,6 @@
nodeCountSize + nodeArrayOffset + nodeffset;
nodeffset += ptNode.mCachedSize;
}
- final int nodeSize = nodeCountSize + nodeffset
- + (formatOptions.mSupportsDynamicUpdate
- ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0);
nodeArrayOffset += nodeArray.mCachedSize;
}
return nodeArrayOffset;
@@ -653,8 +649,8 @@
return flags;
}
- /* package */ static byte makePtNodeFlags(final PtNode node, final int ptNodeAddress,
- final int childrenOffset, final FormatOptions formatOptions) {
+ /* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset,
+ final FormatOptions formatOptions) {
return (byte) makePtNodeFlags(node.mChars.length > 1, node.mFrequency >= 0,
getByteSize(childrenOffset),
node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(),
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
index a282f59..e901376 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
@@ -288,6 +288,8 @@
return BinaryDictEncoderUtils.getByteSize(value);
}
+ // TODO: Remove this method.
+ @Deprecated
static void skipPtNode(final DictBuffer dictBuffer, final FormatOptions formatOptions) {
final int flags = dictBuffer.readUnsignedByte();
BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions);
diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
index 3796a46..e251f7d 100644
--- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
@@ -391,4 +391,6 @@
return readLength;
}
}
+
+ public abstract void skipPtNode(final FormatOptions formatOptions);
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
index 411e265..5c69941 100644
--- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
@@ -59,7 +59,7 @@
throws IOException, UnsupportedFormatException {
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
dictBuffer.position(0);
- final FileHeader header = dictDecoder.readHeader();
+ dictDecoder.readHeader();
final int wordPosition = dictDecoder.getTerminalPosition(word);
if (wordPosition == FormatSpec.NOT_VALID_WORD) return;
@@ -142,8 +142,7 @@
final int originalPosition = dictBuffer.position();
dictBuffer.position(ptNodeOriginAddress);
final int flags = dictBuffer.readUnsignedByte();
- final int parentAddress = BinaryDictDecoderUtils.readParentAddress(dictBuffer,
- formatOptions);
+ BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions);
BinaryDictIOUtils.skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) dictBuffer.readUnsignedByte();
final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS
diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java
index 96d057a..7592a0c 100644
--- a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java
+++ b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java
@@ -17,6 +17,7 @@
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
import java.io.FileInputStream;
@@ -37,35 +38,39 @@
/**
* mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize
* terminals.
- * It contains at index i = j / mBlockSize the index in mContentsTable where the values for
- * terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized integer array.
+ * It contains at index i = j / mBlockSize the index in each ArrayList in mContentsTables where
+ * the values for terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized
+ * integer array.
*/
private final ArrayList<Integer> mLookupTable;
- private final ArrayList<Integer> mContentTable;
+ private final ArrayList<ArrayList<Integer>> mContentTables;
private final int mBlockSize;
+ private final int mContentTableCount;
public static final int NOT_EXIST = -1;
+ public static final int SIZE_OF_INT_IN_BYTES = 4;
@UsedForTesting
- public SparseTable(final int initialCapacity, final int blockSize) {
+ public SparseTable(final int initialCapacity, final int blockSize,
+ final int contentTableCount) {
mBlockSize = blockSize;
final int lookupTableSize = initialCapacity / mBlockSize
+ (initialCapacity % mBlockSize > 0 ? 1 : 0);
mLookupTable = new ArrayList<Integer>(Collections.nCopies(lookupTableSize, NOT_EXIST));
- mContentTable = new ArrayList<Integer>();
+ mContentTableCount = contentTableCount;
+ mContentTables = CollectionUtils.newArrayList();
+ for (int i = 0; i < mContentTableCount; ++i) {
+ mContentTables.add(new ArrayList<Integer>());
+ }
}
@UsedForTesting
- public SparseTable(final int[] lookupTable, final int[] contentTable, final int blockSize) {
+ public SparseTable(final ArrayList<Integer> lookupTable,
+ final ArrayList<ArrayList<Integer>> contentTables, final int blockSize) {
mBlockSize = blockSize;
- mLookupTable = new ArrayList<Integer>(lookupTable.length);
- for (int i = 0; i < lookupTable.length; ++i) {
- mLookupTable.add(lookupTable[i]);
- }
- mContentTable = new ArrayList<Integer>(contentTable.length);
- for (int i = 0; i < contentTable.length; ++i) {
- mContentTable.add(contentTable[i]);
- }
+ mContentTableCount = contentTables.size();
+ mLookupTable = lookupTable;
+ mContentTables = contentTables;
}
/**
@@ -75,8 +80,8 @@
* Otherwise, IndexOutOfBoundsException will be raised.
*/
@UsedForTesting
- private static void convertByteArrayToIntegerArray(final byte[] byteArray,
- final ArrayList<Integer> integerArray) {
+ private static ArrayList<Integer> convertByteArrayToIntegerArray(final byte[] byteArray) {
+ final ArrayList<Integer> integerArray = new ArrayList<Integer>(byteArray.length / 4);
for (int i = 0; i < byteArray.length; i += 4) {
int value = 0;
for (int j = i; j < i + 4; ++j) {
@@ -85,39 +90,43 @@
}
integerArray.add(value);
}
+ return integerArray;
}
@UsedForTesting
- public SparseTable(final byte[] lookupTable, final byte[] contentTable, final int blockSize) {
- mBlockSize = blockSize;
- mLookupTable = new ArrayList<Integer>(lookupTable.length / 4);
- mContentTable = new ArrayList<Integer>(contentTable.length / 4);
- convertByteArrayToIntegerArray(lookupTable, mLookupTable);
- convertByteArrayToIntegerArray(contentTable, mContentTable);
- }
-
- @UsedForTesting
- public int get(final int index) {
- if (index < 0 || index / mBlockSize >= mLookupTable.size()
- || mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
+ public int get(final int contentTableIndex, final int index) {
+ if (!contains(index)) {
return NOT_EXIST;
}
- return mContentTable.get(mLookupTable.get(index / mBlockSize) + (index % mBlockSize));
+ return mContentTables.get(contentTableIndex).get(
+ mLookupTable.get(index / mBlockSize) + (index % mBlockSize));
}
@UsedForTesting
- public void set(final int index, final int value) {
- if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
- mLookupTable.set(index / mBlockSize, mContentTable.size());
- for (int i = 0; i < mBlockSize; ++i) {
- mContentTable.add(NOT_EXIST);
- }
+ public ArrayList<Integer> getAll(final int index) {
+ final ArrayList<Integer> ret = CollectionUtils.newArrayList();
+ for (int i = 0; i < mContentTableCount; ++i) {
+ ret.add(get(i, index));
}
- mContentTable.set(mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value);
+ return ret;
}
- public void remove(final int index) {
- set(index, NOT_EXIST);
+ @UsedForTesting
+ public void set(final int contentTableIndex, final int index, final int value) {
+ if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
+ mLookupTable.set(index / mBlockSize, mContentTables.get(contentTableIndex).size());
+ for (int i = 0; i < mContentTableCount; ++i) {
+ for (int j = 0; j < mBlockSize; ++j) {
+ mContentTables.get(i).add(NOT_EXIST);
+ }
+ }
+ }
+ mContentTables.get(contentTableIndex).set(
+ mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value);
+ }
+
+ public void remove(final int indexOfContent, final int index) {
+ set(indexOfContent, index, NOT_EXIST);
}
@UsedForTesting
@@ -127,7 +136,8 @@
@UsedForTesting
/* package */ int getContentTableSize() {
- return mContentTable.size();
+ // This class always has at least one content table.
+ return mContentTables.get(0).size();
}
@UsedForTesting
@@ -136,36 +146,51 @@
}
public boolean contains(final int index) {
- return get(index) != NOT_EXIST;
+ if (index < 0 || index / mBlockSize >= mLookupTable.size()
+ || mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
+ return false;
+ }
+ return true;
}
@UsedForTesting
- public void write(final OutputStream lookupOutStream, final OutputStream contentOutStream)
+ public void write(final OutputStream lookupOutStream, final OutputStream[] contentOutStreams)
throws IOException {
+ if (contentOutStreams.length != mContentTableCount) {
+ throw new RuntimeException(contentOutStreams.length + " streams are given, but the"
+ + " table has " + mContentTableCount + " content tables.");
+ }
for (final int index : mLookupTable) {
- BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, 4);
+ BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, SIZE_OF_INT_IN_BYTES);
}
- for (final int index : mContentTable) {
- BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4);
+ for (int i = 0; i < contentOutStreams.length; ++i) {
+ for (final int data : mContentTables.get(i)) {
+ BinaryDictEncoderUtils.writeUIntToStream(contentOutStreams[i], data,
+ SIZE_OF_INT_IN_BYTES);
+ }
}
}
@UsedForTesting
- public void writeToFiles(final File lookupTableFile, final File contentFile)
+ public void writeToFiles(final File lookupTableFile, final File[] contentFiles)
throws IOException {
- FileOutputStream lookupTableOutStream = null;
- FileOutputStream contentOutStream = null;
+ FileOutputStream lookupTableOutStream = null;
+ final FileOutputStream[] contentTableOutStreams = new FileOutputStream[mContentTableCount];
try {
lookupTableOutStream = new FileOutputStream(lookupTableFile);
- contentOutStream = new FileOutputStream(contentFile);
- write(lookupTableOutStream, contentOutStream);
+ for (int i = 0; i < contentFiles.length; ++i) {
+ contentTableOutStreams[i] = new FileOutputStream(contentFiles[i]);
+ }
+ write(lookupTableOutStream, contentTableOutStreams);
} finally {
if (lookupTableOutStream != null) {
lookupTableOutStream.close();
}
- if (contentOutStream != null) {
- contentOutStream.close();
+ for (int i = 0; i < contentTableOutStreams.length; ++i) {
+ if (contentTableOutStreams[i] != null) {
+ contentTableOutStreams[i].close();
+ }
}
}
}
@@ -185,10 +210,14 @@
}
@UsedForTesting
- public static SparseTable readFromFiles(final File lookupTableFile, final File contentFile,
+ public static SparseTable readFromFiles(final File lookupTableFile, final File[] contentFiles,
final int blockSize) throws IOException {
- final byte[] lookupTable = readFileToByteArray(lookupTableFile);
- final byte[] content = readFileToByteArray(contentFile);
- return new SparseTable(lookupTable, content, blockSize);
+ final ArrayList<ArrayList<Integer>> contentTables =
+ new ArrayList<ArrayList<Integer>>(contentFiles.length);
+ for (int i = 0; i < contentFiles.length; ++i) {
+ contentTables.add(convertByteArrayToIntegerArray(readFileToByteArray(contentFiles[i])));
+ }
+ return new SparseTable(convertByteArrayToIntegerArray(readFileToByteArray(lookupTableFile)),
+ contentTables, blockSize);
}
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java
index 848277c..bf5a28d 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java
@@ -169,7 +169,8 @@
addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams,
addressPointer);
if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
- MakedictLog.d("too many bigrams in a PtNode.");
+ throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size()
+ + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")");
}
} else {
bigrams = null;
@@ -231,4 +232,40 @@
public boolean hasNextPtNodeArray() {
return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS;
}
+
+ @Override
+ public void skipPtNode(final FormatOptions formatOptions) {
+ final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer);
+ PtNodeReader.readParentAddress(mDictBuffer, formatOptions);
+ BinaryDictIOUtils.skipString(mDictBuffer,
+ (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
+ PtNodeReader.readChildrenAddress(mDictBuffer, flags, formatOptions);
+ if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) PtNodeReader.readFrequency(mDictBuffer);
+ if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) {
+ final int shortcutsSize = mDictBuffer.readUnsignedShort();
+ mDictBuffer.position(mDictBuffer.position() + shortcutsSize
+ - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE);
+ }
+ if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) {
+ int bigramCount = 0;
+ while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
+ final int bigramFlags = mDictBuffer.readUnsignedByte();
+ switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) {
+ case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE:
+ mDictBuffer.readUnsignedByte();
+ break;
+ case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES:
+ mDictBuffer.readUnsignedShort();
+ break;
+ case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES:
+ mDictBuffer.readUnsignedInt24();
+ break;
+ }
+ if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break;
+ }
+ if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
+ throw new RuntimeException("Too many bigrams in a PtNode.");
+ }
+ }
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java
index 76f0f40..d9e1989 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java
@@ -133,12 +133,10 @@
countSize);
}
- private void writePtNodeFlags(final PtNode ptNode, final int parentAddress,
- final FormatOptions formatOptions) {
+ private void writePtNodeFlags(final PtNode ptNode, final FormatOptions formatOptions) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition,
- BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mPosition, childrenPos,
- formatOptions),
+ BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos, formatOptions),
FormatSpec.PTNODE_FLAGS_SIZE);
}
@@ -244,7 +242,7 @@
@Override
public void writePtNode(final PtNode ptNode, final int parentPosition,
final FormatOptions formatOptions, final FusionDictionary dict) {
- writePtNodeFlags(ptNode, parentPosition, formatOptions);
+ writePtNodeFlags(ptNode, formatOptions);
writeParentPosition(parentPosition, ptNode, formatOptions);
writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
writeFrequency(ptNode.mFrequency);
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
index 0aa4319..624b278 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
@@ -95,7 +95,6 @@
@Override
public void openDictBuffer() throws FileNotFoundException, IOException {
- final String filename = mDictDirectory.getName();
mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE));
mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY));
mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer(
@@ -131,7 +130,7 @@
mDictDirectory.getName() + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION);
final File contentFile = new File(mDictDirectory,
mDictDirectory.getName() + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION);
- mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, contentFile,
+ mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, new File[] { contentFile },
FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE);
}
@@ -208,7 +207,7 @@
final ArrayList<PendingAttribute> bigrams;
if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
bigrams = new ArrayList<PendingAttribute>();
- final int posOfBigrams = mBigramAddressTable.get(terminalId);
+ final int posOfBigrams = mBigramAddressTable.get(0 /* contentTableIndex */, terminalId);
mBigramBuffer.position(posOfBigrams);
while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
// If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE,
@@ -224,7 +223,8 @@
if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break;
}
if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
- MakedictLog.d("too many bigrams in a node.");
+ throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size()
+ + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")");
}
} else {
bigrams = null;
@@ -293,4 +293,14 @@
public boolean hasNextPtNodeArray() {
return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS;
}
+
+ @Override
+ public void skipPtNode(final FormatOptions formatOptions) {
+ final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer);
+ PtNodeReader.readParentAddress(mDictBuffer, formatOptions);
+ BinaryDictIOUtils.skipString(mDictBuffer,
+ (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
+ if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) PtNodeReader.readTerminalId(mDictBuffer);
+ PtNodeReader.readChildrenAddress(mDictBuffer, flags, formatOptions);
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
index 4c25faf..a403e25 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
@@ -136,7 +136,7 @@
writeTerminalData(flatNodes, terminalCount);
mBigramAddressTable = new SparseTable(terminalCount,
- FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE);
+ FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, 1 /* contentTableCount */);
writeBigrams(flatNodes, dict);
writeBigramAddressSparseTable();
@@ -181,12 +181,10 @@
countSize);
}
- private void writePtNodeFlags(final PtNode ptNode, final int parentAddress,
- final FormatOptions formatOptions) {
+ private void writePtNodeFlags(final PtNode ptNode, final FormatOptions formatOptions) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos,
- BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mTriePos, childrenPos,
- formatOptions),
+ BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos, formatOptions),
FormatSpec.PTNODE_FLAGS_SIZE);
}
@@ -231,8 +229,7 @@
while (shortcutIterator.hasNext()) {
final WeightedString target = shortcutIterator.next();
final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
- shortcutIterator.hasNext(),
- target.mFrequency);
+ shortcutIterator.hasNext(), target.mFrequency);
mTrieBuf[mTriePos++] = (byte)shortcutFlags;
final int shortcutShift = CharEncoding.writeString(mTrieBuf, mTriePos,
target.mWord);
@@ -254,7 +251,8 @@
for (final PtNode ptNode : nodeArray.mData) {
if (ptNode.mBigrams != null) {
final int startPos = bigramBuffer.size();
- mBigramAddressTable.set(ptNode.mTerminalId, startPos);
+ mBigramAddressTable.set(0 /* contentTableIndex */, ptNode.mTerminalId,
+ startPos);
final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator();
while (bigramIterator.hasNext()) {
final WeightedString bigram = bigramIterator.next();
@@ -280,7 +278,7 @@
new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION);
final File contentFile =
new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION);
- mBigramAddressTable.writeToFiles(lookupIndexFile, contentFile);
+ mBigramAddressTable.writeToFiles(lookupIndexFile, new File[] { contentFile });
}
@Override
@@ -292,7 +290,7 @@
@Override
public void writePtNode(final PtNode ptNode, final int parentPosition,
final FormatOptions formatOptions, final FusionDictionary dict) {
- writePtNodeFlags(ptNode, parentPosition, formatOptions);
+ writePtNodeFlags(ptNode, formatOptions);
writeParentPosition(parentPosition, ptNode, formatOptions);
writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
if (ptNode.isTerminal()) {
diff --git a/java/src/com/android/inputmethod/latin/settings/DebugSettingsActivity.java b/java/src/com/android/inputmethod/latin/settings/DebugSettingsActivity.java
index b499c26..ef6ab2a 100644
--- a/java/src/com/android/inputmethod/latin/settings/DebugSettingsActivity.java
+++ b/java/src/com/android/inputmethod/latin/settings/DebugSettingsActivity.java
@@ -38,4 +38,10 @@
super.onCreate(savedInstanceState);
setTitle(R.string.english_ime_debug_settings);
}
+
+ // TODO: Uncomment the override annotation once we start using SDK version 19.
+ // @Override
+ public boolean isValidFragment(String fragmentName) {
+ return fragmentName.equals(DEFAULT_FRAGMENT);
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsActivity.java b/java/src/com/android/inputmethod/latin/settings/SettingsActivity.java
index 6c38186..ad68f8c 100644
--- a/java/src/com/android/inputmethod/latin/settings/SettingsActivity.java
+++ b/java/src/com/android/inputmethod/latin/settings/SettingsActivity.java
@@ -32,4 +32,10 @@
intent.putExtra(EXTRA_NO_HEADERS, true);
return intent;
}
+
+ // TODO: Uncomment the override annotation once we start using SDK version 19.
+ // @Override
+ public boolean isValidFragment(String fragmentName) {
+ return fragmentName.equals(DEFAULT_FRAGMENT);
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/spellcheck/SpellCheckerSettingsActivity.java b/java/src/com/android/inputmethod/latin/spellcheck/SpellCheckerSettingsActivity.java
index 119ca47..aba5637 100644
--- a/java/src/com/android/inputmethod/latin/spellcheck/SpellCheckerSettingsActivity.java
+++ b/java/src/com/android/inputmethod/latin/spellcheck/SpellCheckerSettingsActivity.java
@@ -24,6 +24,8 @@
* Spell checker preference screen.
*/
public final class SpellCheckerSettingsActivity extends PreferenceActivity {
+ private static final String DEFAULT_FRAGMENT = SpellCheckerSettingsFragment.class.getName();
+
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
@@ -32,8 +34,14 @@
@Override
public Intent getIntent() {
final Intent modIntent = new Intent(super.getIntent());
- modIntent.putExtra(EXTRA_SHOW_FRAGMENT, SpellCheckerSettingsFragment.class.getName());
+ modIntent.putExtra(EXTRA_SHOW_FRAGMENT, DEFAULT_FRAGMENT);
modIntent.putExtra(EXTRA_NO_HEADERS, true);
return modIntent;
}
+
+ // TODO: Uncomment the override annotation once we start using SDK version 19.
+ // @Override
+ public boolean isValidFragment(String fragmentName) {
+ return fragmentName.equals(DEFAULT_FRAGMENT);
+ }
}
diff --git a/native/jni/Android.mk b/native/jni/Android.mk
index 36afea5..ca6a779 100644
--- a/native/jni/Android.mk
+++ b/native/jni/Android.mk
@@ -85,8 +85,8 @@
$(addprefix suggest/policyimpl/dictionary/utils/, \
buffer_with_extendable_buffer.cpp \
byte_array_utils.cpp \
- decaying_utils.cpp \
dict_file_writing_utils.cpp \
+ forgetting_curve_utils.cpp \
format_utils.cpp) \
suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
$(addprefix suggest/policyimpl/typing/, \
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 21c15ea..38159b0 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -344,8 +344,7 @@
static const int GET_PROPERTY_RESULT_LENGTH = 100;
char resultChars[GET_PROPERTY_RESULT_LENGTH];
resultChars[0] = '\0';
- dictionary->getDictionaryStructurePolicy()->getProperty(queryChars, resultChars,
- GET_PROPERTY_RESULT_LENGTH);
+ dictionary->getProperty(queryChars, resultChars, GET_PROPERTY_RESULT_LENGTH);
return env->NewStringUTF(resultChars);
}
diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h
index c920f64..742e388 100644
--- a/native/jni/src/defines.h
+++ b/native/jni/src/defines.h
@@ -298,9 +298,19 @@
#define NOT_AN_INDEX (-1)
#define NOT_A_PROBABILITY (-1)
#define NOT_A_DICT_POS (S_INT_MIN)
+
// A special value to mean the first word confidence makes no sense in this case,
// e.g. this is not a multi-word suggestion.
-#define NOT_A_FIRST_WORD_CONFIDENCE (S_INT_MIN)
+#define NOT_A_FIRST_WORD_CONFIDENCE (S_INT_MAX)
+// How high the confidence needs to be for us to auto-commit. Arbitrary.
+// This needs to be the same as CONFIDENCE_FOR_AUTO_COMMIT in BinaryDictionary.java
+#define CONFIDENCE_FOR_AUTO_COMMIT (1000000)
+// 80% of the full confidence
+#define DISTANCE_WEIGHT_FOR_AUTO_COMMIT (80 * CONFIDENCE_FOR_AUTO_COMMIT / 100)
+// 100% of the full confidence
+#define LENGTH_WEIGHT_FOR_AUTO_COMMIT (CONFIDENCE_FOR_AUTO_COMMIT)
+// 80% of the full confidence
+#define SPACE_COUNT_WEIGHT_FOR_AUTO_COMMIT (80 * CONFIDENCE_FOR_AUTO_COMMIT / 100)
#define KEYCODE_SPACE ' '
#define KEYCODE_SINGLE_QUOTE '\''
diff --git a/native/jni/src/suggest/core/dicnode/dic_node.h b/native/jni/src/suggest/core/dicnode/dic_node.h
index 9099e82..49cfdec 100644
--- a/native/jni/src/suggest/core/dicnode/dic_node.h
+++ b/native/jni/src/suggest/core/dicnode/dic_node.h
@@ -271,7 +271,7 @@
return isTerminalNodes && currentNodeDepth > 0 && currentNodeDepth == terminalNodeDepth;
}
- bool shouldBeFilterdBySafetyNetForBigram() const {
+ bool shouldBeFilteredBySafetyNetForBigram() const {
const uint16_t currentDepth = getNodeCodePointCount();
const int prevWordLen = mDicNodeState.mDicNodeStatePrevWord.getPrevWordLength()
- mDicNodeState.mDicNodeStatePrevWord.getPrevWordStart() - 1;
@@ -321,6 +321,16 @@
DUMP_WORD_AND_SCORE("OUTPUT");
}
+ // "Total" in this context (and other methods in this class) means the whole suggestion. When
+ // this represents a multi-word suggestion, the referenced PtNode (in mDicNodeState) is only
+ // the one that corresponds to the last word of the suggestion, and all the previous words
+ // are concatenated together in mPrevWord - which contains a space at the end.
+ int getTotalNodeSpaceCount() const {
+ if (isFirstWord()) return 0;
+ return CharUtils::getSpaceCount(mDicNodeState.mDicNodeStatePrevWord.mPrevWord,
+ mDicNodeState.mDicNodeStatePrevWord.getPrevWordLength());
+ }
+
int getSecondWordFirstInputIndex(const ProximityInfoState *const pInfoState) const {
const int inputIndex = mDicNodeState.mDicNodeStatePrevWord.getSecondWordFirstInputIndex();
if (inputIndex == NOT_AN_INDEX) {
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp
index 5969b31..59ead18 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp
@@ -129,7 +129,7 @@
}
void Dictionary::getProperty(const char *const query, char *const outResult,
- const int maxResultLength) const {
+ const int maxResultLength) {
return mDictionaryStructureWithBufferPolicy->getProperty(query, outResult, maxResultLength);
}
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h
index 43d3b96..0195d5b 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.h
+++ b/native/jni/src/suggest/core/dictionary/dictionary.h
@@ -84,7 +84,7 @@
bool needsToRunGC(const bool mindsBlockByGC);
void getProperty(const char *const query, char *const outResult,
- const int maxResultLength) const;
+ const int maxResultLength);
const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
return mDictionaryStructureWithBufferPolicy;
diff --git a/native/jni/src/suggest/core/layout/proximity_info_params.cpp b/native/jni/src/suggest/core/layout/proximity_info_params.cpp
index 0e887f7..49df103 100644
--- a/native/jni/src/suggest/core/layout/proximity_info_params.cpp
+++ b/native/jni/src/suggest/core/layout/proximity_info_params.cpp
@@ -69,13 +69,13 @@
const float ProximityInfoParams::SKIP_CORNER_PROBABILITY = 0.4f;
const float ProximityInfoParams::SPEED_MARGIN = 0.1f;
const float ProximityInfoParams::CENTER_VALUE_OF_NORMALIZED_DISTRIBUTION = 0.0f;
-// TODO: The variance is critical for accuracy; thus, adjusting these parameter by machine
+// TODO: The variance is critical for accuracy; thus, adjusting these parameters by machine
// learning or something would be efficient.
-const float ProximityInfoParams::SPEEDxANGLE_WEIGHT_FOR_STANDARD_DIVIATION = 0.3f;
-const float ProximityInfoParams::MAX_SPEEDxANGLE_RATE_FOR_STANDERD_DIVIATION = 0.25f;
-const float ProximityInfoParams::SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DIVIATION = 0.5f;
-const float ProximityInfoParams::MAX_SPEEDxNEAREST_RATE_FOR_STANDERD_DIVIATION = 0.15f;
-const float ProximityInfoParams::MIN_STANDERD_DIVIATION = 0.37f;
+const float ProximityInfoParams::SPEEDxANGLE_WEIGHT_FOR_STANDARD_DEVIATION = 0.3f;
+const float ProximityInfoParams::MAX_SPEEDxANGLE_RATE_FOR_STANDARD_DEVIATION = 0.25f;
+const float ProximityInfoParams::SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DEVIATION = 0.5f;
+const float ProximityInfoParams::MAX_SPEEDxNEAREST_RATE_FOR_STANDARD_DEVIATION = 0.15f;
+const float ProximityInfoParams::MIN_STANDARD_DEVIATION = 0.37f;
const float ProximityInfoParams::PREV_DISTANCE_WEIGHT = 0.5f;
const float ProximityInfoParams::NEXT_DISTANCE_WEIGHT = 0.6f;
diff --git a/native/jni/src/suggest/core/layout/proximity_info_params.h b/native/jni/src/suggest/core/layout/proximity_info_params.h
index 4e47f73..ae1f82c 100644
--- a/native/jni/src/suggest/core/layout/proximity_info_params.h
+++ b/native/jni/src/suggest/core/layout/proximity_info_params.h
@@ -73,11 +73,11 @@
static const float SKIP_CORNER_PROBABILITY;
static const float SPEED_MARGIN;
static const float CENTER_VALUE_OF_NORMALIZED_DISTRIBUTION;
- static const float SPEEDxANGLE_WEIGHT_FOR_STANDARD_DIVIATION;
- static const float MAX_SPEEDxANGLE_RATE_FOR_STANDERD_DIVIATION;
- static const float SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DIVIATION;
- static const float MAX_SPEEDxNEAREST_RATE_FOR_STANDERD_DIVIATION;
- static const float MIN_STANDERD_DIVIATION;
+ static const float SPEEDxANGLE_WEIGHT_FOR_STANDARD_DEVIATION;
+ static const float MAX_SPEEDxANGLE_RATE_FOR_STANDARD_DEVIATION;
+ static const float SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DEVIATION;
+ static const float MAX_SPEEDxNEAREST_RATE_FOR_STANDARD_DEVIATION;
+ static const float MIN_STANDARD_DEVIATION;
static const float PREV_DISTANCE_WEIGHT;
static const float NEXT_DISTANCE_WEIGHT;
diff --git a/native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp b/native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp
index 904671f..e1b3534 100644
--- a/native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp
+++ b/native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp
@@ -708,13 +708,13 @@
const float inputCharProbability = 1.0f - skipProbability;
const float speedxAngleRate = min(speedRate * currentAngle / M_PI_F
- * ProximityInfoParams::SPEEDxANGLE_WEIGHT_FOR_STANDARD_DIVIATION,
- ProximityInfoParams::MAX_SPEEDxANGLE_RATE_FOR_STANDERD_DIVIATION);
+ * ProximityInfoParams::SPEEDxANGLE_WEIGHT_FOR_STANDARD_DEVIATION,
+ ProximityInfoParams::MAX_SPEEDxANGLE_RATE_FOR_STANDARD_DEVIATION);
const float speedxNearestKeyDistanceRate = min(speedRate * nearestKeyDistance
- * ProximityInfoParams::SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DIVIATION,
- ProximityInfoParams::MAX_SPEEDxNEAREST_RATE_FOR_STANDERD_DIVIATION);
+ * ProximityInfoParams::SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DEVIATION,
+ ProximityInfoParams::MAX_SPEEDxNEAREST_RATE_FOR_STANDARD_DEVIATION);
const float sigma = speedxAngleRate + speedxNearestKeyDistanceRate
- + ProximityInfoParams::MIN_STANDERD_DIVIATION;
+ + ProximityInfoParams::MIN_STANDARD_DEVIATION;
ProximityInfoUtils::NormalDistribution
distribution(ProximityInfoParams::CENTER_VALUE_OF_NORMALIZED_DISTRIBUTION, sigma);
diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
index c7ffef0..41f8204 100644
--- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
+++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
@@ -80,8 +80,10 @@
virtual bool needsToRunGC(const bool mindsBlockByGC) const = 0;
+ // Currently, this method is used only for testing. You may want to consider creating new
+ // dedicated method instead of this if you want to use this in the production.
virtual void getProperty(const char *const query, char *const outResult,
- const int maxResultLength) const = 0;
+ const int maxResultLength) = 0;
protected:
DictionaryStructureWithBufferPolicy() {}
diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp
index 51cfba1..73ccebc 100644
--- a/native/jni/src/suggest/core/suggest.cpp
+++ b/native/jni/src/suggest/core/suggest.cpp
@@ -166,7 +166,11 @@
// TODO: have partial commit work even with multiple pointers.
const bool outputSecondWordFirstLetterInputIndex =
traverseSession->isOnlyOnePointerUsed(0 /* pointerId */);
- outputAutoCommitFirstWordConfidence[0] = computeFirstWordConfidence();
+ if (terminalSize > 0) {
+ // If we have no suggestions, don't write this
+ outputAutoCommitFirstWordConfidence[0] =
+ computeFirstWordConfidence(&terminals[0]);
+ }
// Output suggestion results here
for (int terminalIndex = 0; terminalIndex < terminalSize && outputWordIndex < MAX_RESULTS;
@@ -255,9 +259,55 @@
return outputWordIndex;
}
-int Suggest::computeFirstWordConfidence() const {
- // TODO: implement this.
- return NOT_A_FIRST_WORD_CONFIDENCE;
+int Suggest::computeFirstWordConfidence(const DicNode *const terminalDicNode) const {
+ // Get the number of spaces in the first suggestion
+ const int spaceCount = terminalDicNode->getTotalNodeSpaceCount();
+ // Get the number of characters in the first suggestion
+ const int length = terminalDicNode->getTotalNodeCodePointCount();
+ // Get the distance for the first word of the suggestion
+ const float distance = terminalDicNode->getNormalizedCompoundDistanceAfterFirstWord();
+
+ // Arbitrarily, we give a score whose useful values range from 0 to 1,000,000.
+ // 1,000,000 will be the cutoff to auto-commit. It's fine if the number is under 0 or
+ // above 1,000,000 : under 0 just means it's very bad to commit, and above 1,000,000 means
+ // we are very confident.
+ // Expected space count is 1 ~ 5
+ static const int MIN_EXPECTED_SPACE_COUNT = 1;
+ static const int MAX_EXPECTED_SPACE_COUNT = 5;
+ // Expected length is about 4 ~ 30
+ static const int MIN_EXPECTED_LENGTH = 4;
+ static const int MAX_EXPECTED_LENGTH = 30;
+ // Expected distance is about 0.2 ~ 2.0, but consider 0.0 ~ 2.0
+ static const float MIN_EXPECTED_DISTANCE = 0.0;
+ static const float MAX_EXPECTED_DISTANCE = 2.0;
+ // This is not strict: it's where most stuff will be falling, but it's still fine if it's
+ // outside these values. We want to output a value that reflects all of these. Each factor
+ // contributes a bit.
+
+ // We need at least a space.
+ if (spaceCount < 1) return NOT_A_FIRST_WORD_CONFIDENCE;
+
+ // The smaller the edit distance, the higher the contribution. MIN_EXPECTED_DISTANCE means 0
+ // contribution, while MAX_EXPECTED_DISTANCE means full contribution according to the
+ // weight of the distance. Clamp to avoid overflows.
+ const float clampedDistance = distance < MIN_EXPECTED_DISTANCE ? MIN_EXPECTED_DISTANCE
+ : distance > MAX_EXPECTED_DISTANCE ? MAX_EXPECTED_DISTANCE : distance;
+ const int distanceContribution = DISTANCE_WEIGHT_FOR_AUTO_COMMIT
+ * (MAX_EXPECTED_DISTANCE - clampedDistance)
+ / (MAX_EXPECTED_DISTANCE - MIN_EXPECTED_DISTANCE);
+ // The larger the suggestion length, the larger the contribution. MIN_EXPECTED_LENGTH is no
+ // contribution, MAX_EXPECTED_LENGTH is full contribution according to the weight of the
+ // length. Length is guaranteed to be between 1 and 48, so we don't need to clamp.
+ const int lengthContribution = LENGTH_WEIGHT_FOR_AUTO_COMMIT
+ * (length - MIN_EXPECTED_LENGTH) / (MAX_EXPECTED_LENGTH - MIN_EXPECTED_LENGTH);
+ // The more spaces, the larger the contribution. MIN_EXPECTED_SPACE_COUNT space is no
+ // contribution, MAX_EXPECTED_SPACE_COUNT spaces is full contribution according to the
+ // weight of the space count.
+ const int spaceContribution = SPACE_COUNT_WEIGHT_FOR_AUTO_COMMIT
+ * (spaceCount - MIN_EXPECTED_SPACE_COUNT)
+ / (MAX_EXPECTED_SPACE_COUNT - MIN_EXPECTED_SPACE_COUNT);
+
+ return distanceContribution + lengthContribution + spaceContribution;
}
/**
@@ -395,7 +445,7 @@
if (!dicNode->isTerminalWordNode()) {
return;
}
- if (dicNode->shouldBeFilterdBySafetyNetForBigram()) {
+ if (dicNode->shouldBeFilteredBySafetyNetForBigram()) {
return;
}
// Create a non-cached node here.
diff --git a/native/jni/src/suggest/core/suggest.h b/native/jni/src/suggest/core/suggest.h
index 0e8bd11..b20343d 100644
--- a/native/jni/src/suggest/core/suggest.h
+++ b/native/jni/src/suggest/core/suggest.h
@@ -58,7 +58,7 @@
int outputSuggestions(DicTraverseSession *traverseSession, int *frequencies,
int *outputCodePoints, int *outputIndicesToPartialCommit, int *outputTypes,
int *outputAutoCommitFirstWordConfidence) const;
- int computeFirstWordConfidence() const;
+ int computeFirstWordConfidence(const DicNode *const terminalDicNode) const;
void initializeSearch(DicTraverseSession *traverseSession, int commitPoint) const;
void expandCurrentDicNodes(DicTraverseSession *traverseSession) const;
void processTerminalDicNode(DicTraverseSession *traverseSession, DicNode *dicNode) const;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
index 67a085d..8753c6e 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
@@ -20,7 +20,7 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
-#include "suggest/policyimpl/dictionary/utils/decaying_utils.h"
+#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
@@ -43,7 +43,7 @@
}
*outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(bigramFlags);
*outHasNext = BigramListReadWriteUtils::hasNext(bigramFlags);
- if (mIsDecayingDict && !DecayingUtils::isValidBigram(*outProbability)) {
+ if (mIsDecayingDict && !ForgettingCurveUtils::isValidEncodedProbability(*outProbability)) {
// This bigram is too weak to output.
*outBigramPos = NOT_A_DICT_POS;
} else {
@@ -261,8 +261,8 @@
const int originalProbability = BigramListReadWriteUtils::getProbabilityFromFlags(
bigramFlags);
const int probabilityToWrite = mIsDecayingDict ?
- DecayingUtils::getUpdatedBigramProbabilityDelta(
- originalProbability, probability) : probability;
+ ForgettingCurveUtils::getUpdatedEncodedProbability(originalProbability,
+ probability) : probability;
const BigramListReadWriteUtils::BigramFlags updatedFlags =
BigramListReadWriteUtils::setProbabilityInFlags(bigramFlags,
probabilityToWrite);
@@ -294,7 +294,7 @@
int *const writingPos) {
// hasNext is false because we are adding a new bigram entry at the end of the bigram list.
const int probabilityToWrite = mIsDecayingDict ?
- DecayingUtils::getUpdatedBigramProbabilityDelta(NOT_A_PROBABILITY, probability) :
+ ForgettingCurveUtils::getUpdatedEncodedProbability(NOT_A_PROBABILITY, probability) :
probability;
return BigramListReadWriteUtils::createAndWriteBigramEntry(mBuffer, bigramTargetPos,
probabilityToWrite, false /* hasNext */, writingPos);
@@ -365,9 +365,9 @@
*outRemoved = false;
if (mIsDecayingDict) {
// Update bigram probability for decaying.
- const int newProbability = DecayingUtils::getBigramProbabilityDeltaToSave(
+ const int newProbability = ForgettingCurveUtils::getEncodedProbabilityToSave(
BigramListReadWriteUtils::getProbabilityFromFlags(bigramFlags));
- if (DecayingUtils::isValidBigram(newProbability)) {
+ if (ForgettingCurveUtils::isValidEncodedProbability(newProbability)) {
// Write new probability.
const BigramListReadWriteUtils::BigramFlags updatedBigramFlags =
BigramListReadWriteUtils::setProbabilityInFlags(
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
index 081163a..324b530 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
@@ -16,7 +16,7 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h"
-#include "suggest/policyimpl/dictionary/utils/decaying_utils.h"
+#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
namespace latinime {
@@ -29,14 +29,14 @@
bool isUselessPtNode = !node->isTerminal();
if (node->isTerminal() && mIsDecayingDict) {
const int newProbability =
- DecayingUtils::getUnigramProbabilityToSave(node->getProbability());
+ ForgettingCurveUtils::getEncodedProbabilityToSave(node->getProbability());
int writingPos = node->getProbabilityFieldPos();
// Update probability.
if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(
mBuffer, newProbability, &writingPos)) {
return false;
}
- if (!DecayingUtils::isValidUnigram(newProbability)) {
+ if (!ForgettingCurveUtils::isValidEncodedProbability(newProbability)) {
isUselessPtNode = false;
}
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
index 0d8c927..60d0db0 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
@@ -28,17 +28,21 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
-#include "suggest/policyimpl/dictionary/utils/decaying_utils.h"
+#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
#include "suggest/policyimpl/dictionary/utils/probability_utils.h"
namespace latinime {
+// Note that these are corresponding definitions in Java side in BinaryDictionaryTests and
+// BinaryDictionaryDecayingTests.
const char *const DynamicPatriciaTriePolicy::UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
const char *const DynamicPatriciaTriePolicy::BIGRAM_COUNT_QUERY = "BIGRAM_COUNT";
+const char *const DynamicPatriciaTriePolicy::SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY =
+ "SET_NEEDS_TO_DECAY_FOR_TESTING";
const int DynamicPatriciaTriePolicy::MAX_DICT_EXTENDED_REGION_SIZE = 1024 * 1024;
const int DynamicPatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS =
DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE - 1024;
-const int DynamicPatriciaTriePolicy::MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING = 2 * 60 * 60;
+const int DynamicPatriciaTriePolicy::DECAY_INTERVAL_FOR_DECAYING_DICTS = 2 * 60 * 60;
void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode,
DicNodeVector *const childDicNodes) const {
@@ -150,7 +154,7 @@
int DynamicPatriciaTriePolicy::getProbability(const int unigramProbability,
const int bigramProbability) const {
if (mHeaderPolicy.isDecayingDict()) {
- return DecayingUtils::getProbability(unigramProbability, bigramProbability);
+ return ForgettingCurveUtils::getProbability(unigramProbability, bigramProbability);
} else {
if (unigramProbability == NOT_A_PROBABILITY) {
return NOT_A_PROBABILITY;
@@ -301,7 +305,7 @@
return;
}
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
- &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict());
+ &mBigramListPolicy, &mShortcutListPolicy, false /* needsToDecay */);
writingHelper.writeToDictFile(filePath, &mHeaderPolicy, mUnigramCount, mBigramCount);
}
@@ -310,9 +314,15 @@
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
return;
}
+ const bool runGCwithDecay = needsToDecay();
+ DynamicBigramListPolicy bigramListPolicyForGC(&mBufferWithExtendableBuffer,
+ &mShortcutListPolicy, runGCwithDecay);
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
- &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict());
+ &bigramListPolicyForGC, &mShortcutListPolicy, runGCwithDecay);
writingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy);
+ if (runGCwithDecay) {
+ mNeedsToDecayForTesting = false;
+ }
}
bool DynamicPatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
@@ -334,14 +344,13 @@
// Needs to reduce dictionary size.
return true;
} else if (mHeaderPolicy.isDecayingDict()) {
- if (mUnigramCount >= DecayingUtils::MAX_UNIGRAM_COUNT) {
+ if (mUnigramCount >= ForgettingCurveUtils::MAX_UNIGRAM_COUNT) {
// Unigram count exceeds the limit.
return true;
- } else if (mBigramCount >= DecayingUtils::MAX_BIGRAM_COUNT) {
+ } else if (mBigramCount >= ForgettingCurveUtils::MAX_BIGRAM_COUNT) {
// Bigram count exceeds the limit.
return true;
- } else if (mindsBlockByGC && mHeaderPolicy.getLastUpdatedTime()
- + MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING < time(0)) {
+ } else if (mindsBlockByGC && needsToDecay()) {
// Time to update probabilities for decaying.
return true;
}
@@ -350,12 +359,19 @@
}
void DynamicPatriciaTriePolicy::getProperty(const char *const query, char *const outResult,
- const int maxResultLength) const {
+ const int maxResultLength) {
if (strncmp(query, UNIGRAM_COUNT_QUERY, maxResultLength) == 0) {
snprintf(outResult, maxResultLength, "%d", mUnigramCount);
} else if (strncmp(query, BIGRAM_COUNT_QUERY, maxResultLength) == 0) {
snprintf(outResult, maxResultLength, "%d", mBigramCount);
+ } else if (strncmp(query, SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY, maxResultLength) == 0) {
+ mNeedsToDecayForTesting = true;
}
}
+bool DynamicPatriciaTriePolicy::needsToDecay() const {
+ return mHeaderPolicy.isDecayingDict() && (mNeedsToDecayForTesting
+ || mHeaderPolicy.getLastDecayedTime() + DECAY_INTERVAL_FOR_DECAYING_DICTS < time(0));
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
index d3150c6..c3bbe99 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
@@ -40,7 +40,7 @@
mBigramListPolicy(&mBufferWithExtendableBuffer, &mShortcutListPolicy,
mHeaderPolicy.isDecayingDict()),
mUnigramCount(mHeaderPolicy.getUnigramCount()),
- mBigramCount(mHeaderPolicy.getBigramCount()) {}
+ mBigramCount(mHeaderPolicy.getBigramCount()), mNeedsToDecayForTesting(false) {}
~DynamicPatriciaTriePolicy() {
delete mBuffer;
@@ -95,16 +95,17 @@
bool needsToRunGC(const bool mindsBlockByGC) const;
void getProperty(const char *const query, char *const outResult,
- const int maxResultLength) const;
+ const int maxResultLength);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);
- static const char*const UNIGRAM_COUNT_QUERY;
- static const char*const BIGRAM_COUNT_QUERY;
+ static const char *const UNIGRAM_COUNT_QUERY;
+ static const char *const BIGRAM_COUNT_QUERY;
+ static const char *const SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY;
static const int MAX_DICT_EXTENDED_REGION_SIZE;
static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS;
- static const int MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING;
+ static const int DECAY_INTERVAL_FOR_DECAYING_DICTS;
const MmappedBuffer *const mBuffer;
const HeaderPolicy mHeaderPolicy;
@@ -113,6 +114,9 @@
DynamicBigramListPolicy mBigramListPolicy;
int mUnigramCount;
int mBigramCount;
+ int mNeedsToDecayForTesting;
+
+ bool needsToDecay() const;
};
} // namespace latinime
#endif // LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
index 28124d2..70a9ee5 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
@@ -25,8 +25,8 @@
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h"
-#include "suggest/policyimpl/dictionary/utils/decaying_utils.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
+#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
#include "utils/hash_map_compat.h"
namespace latinime {
@@ -153,7 +153,7 @@
const int extendedRegionSize = headerPolicy->getExtendedRegionSize() +
mBuffer->getUsedAdditionalBufferSize();
if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */,
- unigramCount, bigramCount, extendedRegionSize)) {
+ false /* updatesLastDecayedTime */, unigramCount, bigramCount, extendedRegionSize)) {
return;
}
DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, mBuffer);
@@ -170,7 +170,7 @@
}
BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */,
- unigramCount, bigramCount, 0 /* extendedRegionSize */)) {
+ mNeedsToDecay, unigramCount, bigramCount, 0 /* extendedRegionSize */)) {
return;
}
DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, &newDictBuffer);
@@ -488,13 +488,13 @@
DynamicPatriciaTrieGcEventListeners
::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
- this, mBuffer, mIsDecayingDict);
+ this, mBuffer, mNeedsToDecay);
if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner(
&traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted)) {
return false;
}
- if (mIsDecayingDict && traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
- .getValidUnigramCount() > DecayingUtils::MAX_UNIGRAM_COUNT_AFTER_GC) {
+ if (mNeedsToDecay && traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
+ .getValidUnigramCount() > ForgettingCurveUtils::MAX_UNIGRAM_COUNT_AFTER_GC) {
// TODO: Remove more unigrams.
}
@@ -506,8 +506,8 @@
return false;
}
- if (mIsDecayingDict && traversePolicyToUpdateBigramProbability.getValidBigramEntryCount()
- > DecayingUtils::MAX_BIGRAM_COUNT_AFTER_GC) {
+ if (mNeedsToDecay && traversePolicyToUpdateBigramProbability.getValidBigramEntryCount()
+ > ForgettingCurveUtils::MAX_BIGRAM_COUNT_AFTER_GC) {
// TODO: Remove more bigrams.
}
@@ -525,7 +525,7 @@
// Create policy instance for the GCed dictionary.
DynamicShortcutListPolicy newDictShortcutPolicy(bufferToWrite);
DynamicBigramListPolicy newDictBigramPolicy(bufferToWrite, &newDictShortcutPolicy,
- mIsDecayingDict);
+ mNeedsToDecay);
// Create reading helper for the GCed dictionary.
DynamicPatriciaTrieReadingHelper newDictReadingHelper(bufferToWrite, &newDictBigramPolicy,
&newDictShortcutPolicy);
@@ -544,8 +544,9 @@
int DynamicPatriciaTrieWritingHelper::getUpdatedProbability(const int originalProbability,
const int newProbability) {
- if (mIsDecayingDict) {
- return DecayingUtils::getUpdatedUnigramProbability(originalProbability, newProbability);
+ if (mNeedsToDecay) {
+ return ForgettingCurveUtils::getUpdatedEncodedProbability(originalProbability,
+ newProbability);
} else {
return newProbability;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
index ecee2cd..0caf291 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
@@ -51,9 +51,9 @@
DynamicPatriciaTrieWritingHelper(BufferWithExtendableBuffer *const buffer,
DynamicBigramListPolicy *const bigramPolicy,
- DynamicShortcutListPolicy *const shortcutPolicy, const bool isDecayingDict)
+ DynamicShortcutListPolicy *const shortcutPolicy, const bool needsToDecay)
: mBuffer(buffer), mBigramPolicy(bigramPolicy), mShortcutPolicy(shortcutPolicy),
- mIsDecayingDict(isDecayingDict) {}
+ mNeedsToDecay(needsToDecay) {}
~DynamicPatriciaTrieWritingHelper() {}
@@ -94,7 +94,7 @@
BufferWithExtendableBuffer *const mBuffer;
DynamicBigramListPolicy *const mBigramPolicy;
DynamicShortcutListPolicy *const mShortcutPolicy;
- const bool mIsDecayingDict;
+ const bool mNeedsToDecay;
bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate,
const int movedPos, const int bigramLinkedNodePos);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
index 9ce9994..eb072fb 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
@@ -23,6 +23,7 @@
// TODO: Change attribute string to "IS_DECAYING_DICT".
const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE";
const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date";
+const char *const HeaderPolicy::LAST_DECAYED_TIME_KEY = "LAST_DECAYED_TIME";
const char *const HeaderPolicy::UNIGRAM_COUNT_KEY = "UNIGRAM_COUNT";
const char *const HeaderPolicy::BIGRAM_COUNT_KEY = "BIGRAM_COUNT";
const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE";
@@ -63,8 +64,8 @@
}
bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
- const bool updatesLastUpdatedTime, const int unigramCount, const int bigramCount,
- const int extendedRegionSize) const {
+ const bool updatesLastUpdatedTime, const bool updatesLastDecayedTime,
+ const int unigramCount, const int bigramCount, const int extendedRegionSize) const {
int writingPos = 0;
if (!HeaderReadWriteUtils::writeDictionaryVersion(bufferToWrite, mDictFormatVersion,
&writingPos)) {
@@ -90,6 +91,11 @@
HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, LAST_UPDATED_TIME_KEY,
time(0));
}
+ if (updatesLastDecayedTime) {
+ // Set current time as a last updated time.
+ HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, LAST_DECAYED_TIME_KEY,
+ time(0));
+ }
if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &attributeMapTowrite,
&writingPos)) {
return false;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
index 4261667..a9c7805 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
@@ -40,6 +40,8 @@
IS_DECAYING_DICT_KEY, false /* defaultValue */)),
mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)),
+ mLastDecayedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ LAST_DECAYED_TIME_KEY, time(0) /* defaultValue */)),
mUnigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
UNIGRAM_COUNT_KEY, 0 /* defaultValue */)),
mBigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
@@ -58,6 +60,8 @@
IS_DECAYING_DICT_KEY, false /* defaultValue */)),
mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)),
+ mLastDecayedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)),
mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0) {}
~HeaderPolicy() {}
@@ -90,6 +94,10 @@
return mLastUpdatedTime;
}
+ AK_FORCE_INLINE int getLastDecayedTime() const {
+ return mLastDecayedTime;
+ }
+
AK_FORCE_INLINE int getUnigramCount() const {
return mUnigramCount;
}
@@ -106,8 +114,8 @@
int *outValue, int outValueSize) const;
bool writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
- const bool updatesLastUpdatedTime, const int unigramCount,
- const int bigramCount, const int extendedRegionSize) const;
+ const bool updatesLastUpdatedTime, const bool updatesLastDecayedTime,
+ const int unigramCount, const int bigramCount, const int extendedRegionSize) const;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy);
@@ -115,6 +123,7 @@
static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
static const char *const IS_DECAYING_DICT_KEY;
static const char *const LAST_UPDATED_TIME_KEY;
+ static const char *const LAST_DECAYED_TIME_KEY;
static const char *const UNIGRAM_COUNT_KEY;
static const char *const BIGRAM_COUNT_KEY;
static const char *const EXTENDED_REGION_SIZE_KEY;
@@ -128,6 +137,7 @@
const float mMultiWordCostMultiplier;
const bool mIsDecayingDict;
const int mLastUpdatedTime;
+ const int mLastDecayedTime;
const int mUnigramCount;
const int mBigramCount;
const int mExtendedRegionSize;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
index 8d88c68..0f8662a 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
@@ -114,7 +114,7 @@
}
void getProperty(const char *const query, char *const outResult,
- const int maxResultLength) const {
+ const int maxResultLength) {
// getProperty is not supported for this class.
if (maxResultLength > 0) {
outResult[0] = '\0';
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.cpp
deleted file mode 100644
index 942a742..0000000
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.cpp
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright (C) 2013, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "suggest/policyimpl/dictionary/utils/decaying_utils.h"
-
-#include "suggest/policyimpl/dictionary/utils/probability_utils.h"
-
-namespace latinime {
-
-const int DecayingUtils::MAX_UNIGRAM_COUNT = 12000;
-const int DecayingUtils::MAX_UNIGRAM_COUNT_AFTER_GC = 10000;
-const int DecayingUtils::MAX_BIGRAM_COUNT = 12000;
-const int DecayingUtils::MAX_BIGRAM_COUNT_AFTER_GC = 10000;
-
-const int DecayingUtils::MAX_COMPUTED_PROBABILITY = 127;
-const int DecayingUtils::MAX_UNIGRAM_PROBABILITY = 120;
-const int DecayingUtils::MIN_VALID_UNIGRAM_PROBABILITY = 24;
-const int DecayingUtils::UNIGRAM_PROBABILITY_STEP = 8;
-const int DecayingUtils::MAX_BIGRAM_PROBABILITY_DELTA = 15;
-const int DecayingUtils::MIN_VALID_BIGRAM_PROBABILITY_DELTA = 3;
-const int DecayingUtils::BIGRAM_PROBABILITY_DELTA_STEP = 1;
-
-/* static */ int DecayingUtils::getProbability(const int encodedUnigramProbability,
- const int encodedBigramProbabilityDelta) {
- if (encodedUnigramProbability == NOT_A_PROBABILITY) {
- return NOT_A_PROBABILITY;
- } else if (encodedBigramProbabilityDelta == NOT_A_PROBABILITY) {
- const int rawProbability = ProbabilityUtils::backoff(decodeUnigramProbability(
- encodedUnigramProbability));
- return min(getDecayedProbability(rawProbability), MAX_COMPUTED_PROBABILITY);
- } else {
- const int rawProbability = ProbabilityUtils::computeProbabilityForBigram(
- decodeUnigramProbability(encodedUnigramProbability),
- decodeBigramProbabilityDelta(encodedBigramProbabilityDelta));
- return min(getDecayedProbability(rawProbability), MAX_COMPUTED_PROBABILITY);
- }
-}
-
-/* static */ int DecayingUtils::getUpdatedUnigramProbability(const int originalEncodedProbability,
- const int newProbability) {
- if (originalEncodedProbability == NOT_A_PROBABILITY) {
- // The unigram is not in this dictionary.
- if (newProbability == NOT_A_PROBABILITY) {
- // The unigram is not in other dictionaries.
- return 0;
- } else {
- return MIN_VALID_UNIGRAM_PROBABILITY;
- }
- } else {
- if (newProbability != NOT_A_PROBABILITY
- && originalEncodedProbability < MIN_VALID_UNIGRAM_PROBABILITY) {
- return MIN_VALID_UNIGRAM_PROBABILITY;
- }
- return min(originalEncodedProbability + UNIGRAM_PROBABILITY_STEP, MAX_UNIGRAM_PROBABILITY);
- }
-}
-
-/* static */ int DecayingUtils::getUnigramProbabilityToSave(const int encodedProbability) {
- return max(encodedProbability - UNIGRAM_PROBABILITY_STEP, 0);
-}
-
-/* static */ int DecayingUtils::getBigramProbabilityDeltaToSave(const int encodedProbabilityDelta) {
- return max(encodedProbabilityDelta - BIGRAM_PROBABILITY_DELTA_STEP, 0);
-}
-
-/* static */ int DecayingUtils::getUpdatedBigramProbabilityDelta(
- const int originalEncodedProbabilityDelta, const int newProbability) {
- if (originalEncodedProbabilityDelta == NOT_A_PROBABILITY) {
- // The bigram relation is not in this dictionary.
- if (newProbability == NOT_A_PROBABILITY) {
- // The bigram target is not in other dictionaries.
- return 0;
- } else {
- return MIN_VALID_BIGRAM_PROBABILITY_DELTA;
- }
- } else {
- if (newProbability != NOT_A_PROBABILITY
- && originalEncodedProbabilityDelta < MIN_VALID_BIGRAM_PROBABILITY_DELTA) {
- return MIN_VALID_BIGRAM_PROBABILITY_DELTA;
- }
- return min(originalEncodedProbabilityDelta + BIGRAM_PROBABILITY_DELTA_STEP,
- MAX_BIGRAM_PROBABILITY_DELTA);
- }
-}
-
-/* static */ int DecayingUtils::isValidUnigram(const int encodedUnigramProbability) {
- return encodedUnigramProbability >= MIN_VALID_UNIGRAM_PROBABILITY;
-}
-
-/* static */ int DecayingUtils::isValidBigram(const int encodedBigramProbabilityDelta) {
- return encodedBigramProbabilityDelta >= MIN_VALID_BIGRAM_PROBABILITY_DELTA;
-}
-
-/* static */ int DecayingUtils::decodeUnigramProbability(const int encodedProbability) {
- const int probability = encodedProbability - MIN_VALID_UNIGRAM_PROBABILITY;
- if (probability < 0) {
- return NOT_A_PROBABILITY;
- } else {
- return min(probability, MAX_UNIGRAM_PROBABILITY);
- }
-}
-
-/* static */ int DecayingUtils::decodeBigramProbabilityDelta(const int encodedProbabilityDelta) {
- const int probabilityDelta = encodedProbabilityDelta - MIN_VALID_BIGRAM_PROBABILITY_DELTA;
- if (probabilityDelta < 0) {
- return NOT_A_PROBABILITY;
- } else {
- return min(probabilityDelta, MAX_BIGRAM_PROBABILITY_DELTA);
- }
-}
-
-/* static */ int DecayingUtils::getDecayedProbability(const int rawProbability) {
- return rawProbability;
-}
-
-} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.h
deleted file mode 100644
index 1ca0391..0000000
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright (C) 2013, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_DECAYING_UTILS_H
-#define LATINIME_DECAYING_UTILS_H
-
-#include "defines.h"
-
-namespace latinime {
-
-// TODO: Check the elapsed time and decrease the probability depending on the time. Time field is
-// required to introduced to each terminal PtNode and bigram entry.
-// TODO: Quit using bigram probability to indicate the delta.
-// TODO: Quit using bigram probability delta.
-class DecayingUtils {
- public:
- static const int MAX_UNIGRAM_COUNT;
- static const int MAX_UNIGRAM_COUNT_AFTER_GC;
- static const int MAX_BIGRAM_COUNT;
- static const int MAX_BIGRAM_COUNT_AFTER_GC;
-
- static int getProbability(const int encodedUnigramProbability,
- const int encodedBigramProbabilityDelta);
-
- static int getUpdatedUnigramProbability(const int originalEncodedProbability,
- const int newProbability);
-
- static int getUpdatedBigramProbabilityDelta(const int originalEncodedProbabilityDelta,
- const int newProbability);
-
- static int isValidUnigram(const int encodedUnigramProbability);
-
- static int isValidBigram(const int encodedProbabilityDelta);
-
- static int getUnigramProbabilityToSave(const int encodedProbability);
-
- static int getBigramProbabilityDeltaToSave(const int encodedProbabilityDelta);
-
- private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(DecayingUtils);
-
- static const int MAX_COMPUTED_PROBABILITY;
- static const int MAX_UNIGRAM_PROBABILITY;
- static const int MIN_VALID_UNIGRAM_PROBABILITY;
- static const int UNIGRAM_PROBABILITY_STEP;
- static const int MAX_BIGRAM_PROBABILITY_DELTA;
- static const int MIN_VALID_BIGRAM_PROBABILITY_DELTA;
- static const int BIGRAM_PROBABILITY_DELTA_STEP;
-
- static int decodeUnigramProbability(const int encodedProbability);
-
- static int decodeBigramProbabilityDelta(const int encodedProbability);
-
- static int getDecayedProbability(const int rawProbability);
-};
-} // namespace latinime
-#endif /* LATINIME_DECAYING_UTILS_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
index f22e94c..994826f 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
@@ -44,7 +44,8 @@
BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
HeaderPolicy headerPolicy(FormatUtils::VERSION_3, attributeMap);
headerPolicy.writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */,
- 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);
+ true /* updatesLastDecayedTime */, 0 /* unigramCount */, 0 /* bigramCount */,
+ 0 /* extendedRegionSize */);
BufferWithExtendableBuffer bodyBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(&bodyBuffer, 0 /* rootPos */)) {
return false;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
new file mode 100644
index 0000000..4ff31ba
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp
@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cmath>
+#include <stdlib.h>
+
+#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
+
+#include "suggest/policyimpl/dictionary/utils/probability_utils.h"
+
+namespace latinime {
+
+const int ForgettingCurveUtils::MAX_UNIGRAM_COUNT = 12000;
+const int ForgettingCurveUtils::MAX_UNIGRAM_COUNT_AFTER_GC = 10000;
+const int ForgettingCurveUtils::MAX_BIGRAM_COUNT = 12000;
+const int ForgettingCurveUtils::MAX_BIGRAM_COUNT_AFTER_GC = 10000;
+
+const int ForgettingCurveUtils::MAX_COMPUTED_PROBABILITY = 127;
+const int ForgettingCurveUtils::MAX_ENCODED_PROBABILITY = 15;
+const int ForgettingCurveUtils::MIN_VALID_ENCODED_PROBABILITY = 3;
+const int ForgettingCurveUtils::ENCODED_PROBABILITY_STEP = 1;
+// Currently, we try to decay each uni/bigram once every 2 hours. Accordingly, the expected
+// duration of the decay is approximately 66hours.
+const float ForgettingCurveUtils::MIN_PROBABILITY_TO_DECAY = 0.03f;
+
+const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityTable;
+
+/* static */ int ForgettingCurveUtils::getProbability(const int encodedUnigramProbability,
+ const int encodedBigramProbability) {
+ if (encodedUnigramProbability == NOT_A_PROBABILITY) {
+ return NOT_A_PROBABILITY;
+ } else if (encodedBigramProbability == NOT_A_PROBABILITY) {
+ return backoff(decodeProbability(encodedUnigramProbability));
+ } else {
+ const int unigramProbability = decodeProbability(encodedUnigramProbability);
+ const int bigramProbability = decodeProbability(encodedBigramProbability);
+ return min(max(unigramProbability, bigramProbability), MAX_COMPUTED_PROBABILITY);
+ }
+}
+
+// Caveat: Unlike getProbability(), this method doesn't assume special bigram probability encoding
+// (i.e. unigram probability + bigram probability delta).
+/* static */ int ForgettingCurveUtils::getUpdatedEncodedProbability(
+ const int originalEncodedProbability, const int newProbability) {
+ if (originalEncodedProbability == NOT_A_PROBABILITY) {
+ // The bigram relation is not in this dictionary.
+ if (newProbability == NOT_A_PROBABILITY) {
+ // The bigram target is not in other dictionaries.
+ return 0;
+ } else {
+ return MIN_VALID_ENCODED_PROBABILITY;
+ }
+ } else {
+ if (newProbability != NOT_A_PROBABILITY
+ && originalEncodedProbability < MIN_VALID_ENCODED_PROBABILITY) {
+ return MIN_VALID_ENCODED_PROBABILITY;
+ }
+ return min(originalEncodedProbability + ENCODED_PROBABILITY_STEP, MAX_ENCODED_PROBABILITY);
+ }
+}
+
+/* static */ int ForgettingCurveUtils::isValidEncodedProbability(const int encodedProbability) {
+ return encodedProbability >= MIN_VALID_ENCODED_PROBABILITY;
+}
+
+/* static */ int ForgettingCurveUtils::getEncodedProbabilityToSave(const int encodedProbability) {
+ const int currentEncodedProbability = max(min(encodedProbability, MAX_ENCODED_PROBABILITY), 0);
+ // TODO: Implement the decay in more proper way.
+ const float currentRate = static_cast<float>(currentEncodedProbability)
+ / static_cast<float>(MAX_ENCODED_PROBABILITY);
+ const float thresholdToDecay = MIN_PROBABILITY_TO_DECAY
+ + (1.0f - MIN_PROBABILITY_TO_DECAY) * (1.0f - currentRate);
+ const float randValue = static_cast<float>(rand()) / static_cast<float>(RAND_MAX);
+ if (thresholdToDecay < randValue) {
+ return max(currentEncodedProbability - ENCODED_PROBABILITY_STEP, 0);
+ } else {
+ return currentEncodedProbability;
+ }
+}
+
+/* static */ int ForgettingCurveUtils::decodeProbability(const int encodedProbability) {
+ const int probability = encodedProbability - MIN_VALID_ENCODED_PROBABILITY;
+ if (encodedProbability < MIN_VALID_ENCODED_PROBABILITY) {
+ return NOT_A_PROBABILITY;
+ } else {
+ return min(sProbabilityTable.getProbability(encodedProbability), MAX_ENCODED_PROBABILITY);
+ }
+}
+
+// See comments in ProbabilityUtils::backoff().
+/* static */ int ForgettingCurveUtils::backoff(const int unigramProbability) {
+ if (unigramProbability == NOT_A_PROBABILITY) {
+ return NOT_A_PROBABILITY;
+ } else {
+ return max(unigramProbability - 8, 0);
+ }
+}
+
+ForgettingCurveUtils::ProbabilityTable::ProbabilityTable() : mTable() {
+ // Table entry is as follows:
+ // 1, 1, 1, 2, 3, 5, 6, 9, 13, 18, 25, 34, 48, 66, 91, 127.
+ // Note that first MIN_VALID_ENCODED_PROBABILITY values are not used.
+ mTable.resize(MAX_ENCODED_PROBABILITY + 1);
+ for (int i = 0; i <= MAX_ENCODED_PROBABILITY; ++i) {
+ const int probability = static_cast<int>(powf(static_cast<float>(MAX_COMPUTED_PROBABILITY),
+ static_cast<float>(i) / static_cast<float>(MAX_ENCODED_PROBABILITY)));
+ mTable[i] = min(MAX_COMPUTED_PROBABILITY, max(0, probability));
+ }
+}
+
+} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h
new file mode 100644
index 0000000..d666f22
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_FORGETTING_CURVE_UTILS_H
+#define LATINIME_FORGETTING_CURVE_UTILS_H
+
+#include <vector>
+
+#include "defines.h"
+
+namespace latinime {
+
+// TODO: Check the elapsed time and decrease the probability depending on the time. Time field is
+// required to introduced to each terminal PtNode and bigram entry.
+// TODO: Quit using bigram probability to indicate the delta.
+class ForgettingCurveUtils {
+ public:
+ static const int MAX_UNIGRAM_COUNT;
+ static const int MAX_UNIGRAM_COUNT_AFTER_GC;
+ static const int MAX_BIGRAM_COUNT;
+ static const int MAX_BIGRAM_COUNT_AFTER_GC;
+
+ static int getProbability(const int encodedUnigramProbability,
+ const int encodedBigramProbability);
+
+ static int getUpdatedEncodedProbability(const int originalEncodedProbability,
+ const int newProbability);
+
+ static int isValidEncodedProbability(const int encodedProbability);
+
+ static int getEncodedProbabilityToSave(const int encodedProbability);
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(ForgettingCurveUtils);
+
+ class ProbabilityTable {
+ public:
+ ProbabilityTable();
+
+ int getProbability(const int encodedProbability) const {
+ if (encodedProbability < 0 || encodedProbability > static_cast<int>(mTable.size())) {
+ return NOT_A_PROBABILITY;
+ }
+ return mTable[encodedProbability];
+ }
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ProbabilityTable);
+
+ std::vector<int> mTable;
+ };
+
+ static const int MAX_COMPUTED_PROBABILITY;
+ static const int MAX_ENCODED_PROBABILITY;
+ static const int MIN_VALID_ENCODED_PROBABILITY;
+ static const int ENCODED_PROBABILITY_STEP;
+ static const float MIN_PROBABILITY_TO_DECAY;
+
+ static const ProbabilityTable sProbabilityTable;
+
+ static int decodeProbability(const int encodedProbability);
+
+ static int backoff(const int unigramProbability);
+};
+} // namespace latinime
+#endif /* LATINIME_FORGETTING_CURVE_UTILS_H */
diff --git a/native/jni/src/suggest/policyimpl/typing/typing_traversal.h b/native/jni/src/suggest/policyimpl/typing/typing_traversal.h
index 89e53f4..007c19e 100644
--- a/native/jni/src/suggest/policyimpl/typing/typing_traversal.h
+++ b/native/jni/src/suggest/policyimpl/typing/typing_traversal.h
@@ -101,7 +101,7 @@
}
const int16_t pointIndex = dicNode->getInputIndex(0);
return pointIndex <= inputSize && !dicNode->isTotalInputSizeExceedingLimit()
- && !dicNode->shouldBeFilterdBySafetyNetForBigram();
+ && !dicNode->shouldBeFilteredBySafetyNetForBigram();
}
AK_FORCE_INLINE bool shouldDepthLevelCache(
diff --git a/native/jni/src/utils/char_utils.h b/native/jni/src/utils/char_utils.h
index 2e735a8..41663c8 100644
--- a/native/jni/src/utils/char_utils.h
+++ b/native/jni/src/utils/char_utils.h
@@ -75,6 +75,16 @@
return c;
}
+ static AK_FORCE_INLINE int getSpaceCount(const int *const codePointBuffer, const int length) {
+ int spaceCount = 0;
+ for (int i = 0; i < length; ++i) {
+ if (codePointBuffer[i] == KEYCODE_SPACE) {
+ ++spaceCount;
+ }
+ }
+ return spaceCount;
+ }
+
static unsigned short latin_tolower(const unsigned short c);
private:
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
index cf85d97..b2d31c2 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
@@ -32,6 +32,11 @@
private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
private static final String TEST_LOCALE = "test";
+ // Note that these are corresponding definitions in native code in
+ // latinime::DynamicPatriciaTriePolicy.
+ private static final String SET_NEEDS_TO_DECAY_FOR_TESTING_KEY =
+ "SET_NEEDS_TO_DECAY_FOR_TESTING";
+
private static final int DUMMY_PROBABILITY = 0;
@Override
@@ -45,14 +50,20 @@
}
private void forcePassingShortTime(final BinaryDictionary binaryDictionary) {
- binaryDictionary.flushWithGC();
+ // Entries having low probability would be suppressed once in 2 GCs.
+ final int count = 2;
+ for (int i = 0; i < count; i++) {
+ binaryDictionary.getPropertyForTests(SET_NEEDS_TO_DECAY_FOR_TESTING_KEY);
+ binaryDictionary.flushWithGC();
+ }
}
private void forcePassingLongTime(final BinaryDictionary binaryDictionary) {
// Currently, probabilities are decayed when GC is run. All entries that have never been
- // typed in 32 GCs are removed.
- final int count = 32;
+ // typed in 128 GCs would be removed.
+ final int count = 128;
for (int i = 0; i < count; i++) {
+ binaryDictionary.getPropertyForTests(SET_NEEDS_TO_DECAY_FOR_TESTING_KEY);
binaryDictionary.flushWithGC();
}
}
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
index a4d9426..0cf4ef9 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
@@ -104,7 +104,9 @@
}
sStarBigrams.put(0, new ArrayList<Integer>());
- for (int i = 1; i < sWords.size(); ++i) {
+ // MAX - 1 because we added one above already
+ final int maxBigrams = Math.min(sWords.size(), FormatSpec.MAX_BIGRAMS_IN_A_PTNODE - 1);
+ for (int i = 1; i < maxBigrams; ++i) {
sStarBigrams.get(0).add(i);
}
@@ -544,8 +546,7 @@
}
private long checkGetTerminalPosition(final DictDecoder dictDecoder, final String word,
- int index, boolean contained) {
- final int expectedFrequency = (UNIGRAM_FREQ + index) % 255;
+ final boolean contained) {
long diff = -1;
int position = -1;
try {
@@ -603,7 +604,7 @@
// Test a word that is contained within the dictionary.
long sum = 0;
for (int i = 0; i < sWords.size(); ++i) {
- final long time = checkGetTerminalPosition(dictDecoder, sWords.get(i), i, true);
+ final long time = checkGetTerminalPosition(dictDecoder, sWords.get(i), true);
sum += time == -1 ? 0 : time;
}
Log.d(TAG, "per search : " + (((double)sum) / sWords.size() / 1000000) + " : " + message
@@ -616,11 +617,11 @@
for (int i = 0; i < 1000; ++i) {
final String word = CodePointUtils.generateWord(random, codePointSet);
if (sWords.indexOf(word) != -1) continue;
- checkGetTerminalPosition(dictDecoder, word, i, false);
+ checkGetTerminalPosition(dictDecoder, word, false);
}
}
- private void runGetTerminalPositionTests(final ArrayList<String> results, final int bufferType,
+ private void runGetTerminalPositionTests(final int bufferType,
final FormatOptions formatOptions) {
runGetTerminalPosition(sWords, sEmptyBigrams, bufferType, formatOptions, "unigram");
}
@@ -628,17 +629,17 @@
public void testGetTerminalPosition() {
final ArrayList<String> results = CollectionUtils.newArrayList();
- runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION2);
- runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE);
- runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE);
- runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE);
- runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE);
+ runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION2);
+ runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE);
+ runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE);
+ runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE);
+ runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE);
- runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION2);
- runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE);
- runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE);
- runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE);
- runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE);
+ runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION2);
+ runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE);
+ runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE);
+ runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE);
+ runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE);
for (final String result : results) {
Log.d(TAG, result);
diff --git a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java
index 132483d..aeb8552 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java
@@ -21,10 +21,8 @@
import android.util.Log;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Random;
@@ -36,9 +34,6 @@
public class SparseTableTests extends AndroidTestCase {
private static final String TAG = SparseTableTests.class.getSimpleName();
- private static final int[] SMALL_INDEX = { SparseTable.NOT_EXIST, 0 };
- private static final int[] BIG_INDEX = { SparseTable.NOT_EXIST, 1, 2, 3, 4, 5, 6, 7};
-
private final Random mRandom;
private final ArrayList<Integer> mRandomIndex;
@@ -59,32 +54,21 @@
}
}
- public void testInitializeWithArray() {
- final SparseTable table = new SparseTable(SMALL_INDEX, BIG_INDEX, BLOCK_SIZE);
- for (int i = 0; i < 8; ++i) {
- assertEquals(SparseTable.NOT_EXIST, table.get(i));
- }
- assertEquals(SparseTable.NOT_EXIST, table.get(8));
- for (int i = 9; i < 16; ++i) {
- assertEquals(i - 8, table.get(i));
- }
- }
-
public void testSet() {
- final SparseTable table = new SparseTable(16, BLOCK_SIZE);
- table.set(3, 6);
- table.set(8, 16);
+ final SparseTable table = new SparseTable(16, BLOCK_SIZE, 1);
+ table.set(0, 3, 6);
+ table.set(0, 8, 16);
for (int i = 0; i < 16; ++i) {
if (i == 3 || i == 8) {
- assertEquals(i * 2, table.get(i));
+ assertEquals(i * 2, table.get(0, i));
} else {
- assertEquals(SparseTable.NOT_EXIST, table.get(i));
+ assertEquals(SparseTable.NOT_EXIST, table.get(0, i));
}
}
}
private void generateRandomIndex(final int size, final int prop) {
- for (int i = 0; i < DEFAULT_SIZE; ++i) {
+ for (int i = 0; i < size; ++i) {
if (mRandom.nextInt(100) < prop) {
mRandomIndex.set(i, mRandom.nextInt());
} else {
@@ -94,11 +78,11 @@
}
private void runTestRandomSet() {
- final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE);
+ final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE, 1);
int elementCount = 0;
for (int i = 0; i < DEFAULT_SIZE; ++i) {
if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) {
- table.set(i, mRandomIndex.get(i));
+ table.set(0, i, mRandomIndex.get(i));
elementCount++;
}
}
@@ -107,29 +91,24 @@
+ table.getContentTableSize());
Log.d(TAG, "the table has " + elementCount + " elements");
for (int i = 0; i < DEFAULT_SIZE; ++i) {
- assertEquals(table.get(i), (int)mRandomIndex.get(i));
+ assertEquals(table.get(0, i), (int)mRandomIndex.get(i));
}
// flush and reload
OutputStream lookupOutStream = null;
OutputStream contentOutStream = null;
- InputStream lookupInStream = null;
- InputStream contentInStream = null;
try {
final File lookupIndexFile = File.createTempFile("testRandomSet", ".small");
final File contentFile = File.createTempFile("testRandomSet", ".big");
lookupOutStream = new FileOutputStream(lookupIndexFile);
contentOutStream = new FileOutputStream(contentFile);
- table.write(lookupOutStream, contentOutStream);
- lookupInStream = new FileInputStream(lookupIndexFile);
- contentInStream = new FileInputStream(contentFile);
- final byte[] lookupArray = new byte[(int) lookupIndexFile.length()];
- final byte[] contentArray = new byte[(int) contentFile.length()];
- lookupInStream.read(lookupArray);
- contentInStream.read(contentArray);
- final SparseTable newTable = new SparseTable(lookupArray, contentArray, BLOCK_SIZE);
+ table.write(lookupOutStream, new OutputStream[] { contentOutStream });
+ lookupOutStream.flush();
+ contentOutStream.flush();
+ final SparseTable newTable = SparseTable.readFromFiles(lookupIndexFile,
+ new File[] { contentFile }, BLOCK_SIZE);
for (int i = 0; i < DEFAULT_SIZE; ++i) {
- assertEquals(table.get(i), newTable.get(i));
+ assertEquals(table.get(0, i), newTable.get(0, i));
}
} catch (IOException e) {
Log.d(TAG, "IOException while flushing and realoding", e);
@@ -157,4 +136,60 @@
runTestRandomSet();
}
}
+
+ public void testMultipleContents() {
+ final int numOfContents = 5;
+ generateRandomIndex(DEFAULT_SIZE, 20);
+ final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE, numOfContents);
+ for (int i = 0; i < mRandomIndex.size(); ++i) {
+ if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) {
+ for (int j = 0; j < numOfContents; ++j) {
+ table.set(j, i, mRandomIndex.get(i));
+ }
+ }
+ }
+
+ OutputStream lookupOutStream = null;
+ OutputStream[] contentsOutStream = new OutputStream[numOfContents];
+ try {
+ final File lookupIndexFile = File.createTempFile("testMultipleContents", "small");
+ lookupOutStream = new FileOutputStream(lookupIndexFile);
+ final File[] contentFiles = new File[numOfContents];
+ for (int i = 0; i < numOfContents; ++i) {
+ contentFiles[i] = File.createTempFile("testMultipleContents", "big" + i);
+ contentsOutStream[i] = new FileOutputStream(contentFiles[i]);
+ }
+ table.write(lookupOutStream, contentsOutStream);
+ lookupOutStream.flush();
+ for (int i = 0; i < numOfContents; ++i) {
+ contentsOutStream[i].flush();
+ }
+ final SparseTable newTable = SparseTable.readFromFiles(lookupIndexFile, contentFiles,
+ BLOCK_SIZE);
+ for (int i = 0; i < numOfContents; ++i) {
+ for (int j = 0; j < DEFAULT_SIZE; ++j) {
+ assertEquals(table.get(i, j), newTable.get(i, j));
+ }
+ }
+ } catch (IOException e) {
+ Log.d(TAG, "IOException while flushing and reloading", e);
+ } finally {
+ if (lookupOutStream != null) {
+ try {
+ lookupOutStream.close();
+ } catch (IOException e) {
+ Log.d(TAG, "IOException while closing the stream", e);
+ }
+ }
+ for (int i = 0; i < numOfContents; ++i) {
+ if (contentsOutStream[i] != null) {
+ try {
+ contentsOutStream[i].close();
+ } catch (IOException e) {
+ Log.d(TAG, "IOException while closing the stream.", e);
+ }
+ }
+ }
+ }
+ }
}
diff --git a/tools/make-keyboard-text/res/values-ca/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-ca/donottranslate-more-keys.xml
index 9728c99..4cf7424 100644
--- a/tools/make-keyboard-text/res/values-ca/donottranslate-more-keys.xml
+++ b/tools/make-keyboard-text/res/values-ca/donottranslate-more-keys.xml
@@ -72,7 +72,7 @@
<string name="more_keys_for_l">l·l,ł</string>
<!-- U+00B7: "·" MIDDLE DOT -->
<string name="more_keys_for_punctuation">"!fixedColumnOrder!4,·,!,\\,,\?,:,;,\@"</string>
- <string name="more_keys_for_tablet_period">\?,·</string>
+ <string name="more_keys_for_period">\?,·</string>
<!-- U+00E7: "ç" LATIN SMALL LETTER C WITH CEDILLA -->
<string name="keylabel_for_spanish_row2_10">ç</string>
</resources>
diff --git a/tools/make-keyboard-text/res/values-es/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-es/donottranslate-more-keys.xml
index 8494296..8e6b4ee 100644
--- a/tools/make-keyboard-text/res/values-es/donottranslate-more-keys.xml
+++ b/tools/make-keyboard-text/res/values-es/donottranslate-more-keys.xml
@@ -75,7 +75,7 @@
<!-- U+00A1: "¡" INVERTED EXCLAMATION MARK -->
<string name="more_keys_for_tablet_comma">"!,¡"</string>
<!-- U+00BF: "¿" INVERTED QUESTION MARK -->
- <string name="more_keys_for_tablet_period">"\?,¿"</string>
+ <string name="more_keys_for_period">"\?,¿"</string>
<string name="keylabel_for_apostrophe">\"</string>
<string name="keyhintlabel_for_apostrophe">\'</string>
<string name="more_keys_for_apostrophe">\'</string>
diff --git a/tools/make-keyboard-text/res/values-fa/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-fa/donottranslate-more-keys.xml
index 5a03c80..ab4fbda 100644
--- a/tools/make-keyboard-text/res/values-fa/donottranslate-more-keys.xml
+++ b/tools/make-keyboard-text/res/values-fa/donottranslate-more-keys.xml
@@ -81,8 +81,8 @@
<string name="keylabel_for_tablet_comma">"،"</string>
<string name="keyhintlabel_for_tablet_comma">"!"</string>
<string name="more_keys_for_tablet_comma">"!,\\,"</string>
- <string name="keyhintlabel_for_tablet_period">"؟"</string>
- <string name="more_keys_for_tablet_period">"؟,\?"</string>
+ <string name="keyhintlabel_for_period">"؟"</string>
+ <string name="more_keys_for_period">"؟,\?"</string>
<string name="keylabel_for_apostrophe">،</string>
<string name="keyhintlabel_for_apostrophe">؟</string>
<string name="more_keys_for_apostrophe">"!fixedColumnOrder!4,:,!,؟,؛,-,/,«|»,»|«"</string>
diff --git a/tools/make-keyboard-text/res/values-iw/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-iw/donottranslate-more-keys.xml
index feaed4c..a163331 100644
--- a/tools/make-keyboard-text/res/values-iw/donottranslate-more-keys.xml
+++ b/tools/make-keyboard-text/res/values-iw/donottranslate-more-keys.xml
@@ -55,6 +55,6 @@
<string name="keylabel_for_currency">₪</string>
<string name="keyhintlabel_for_tablet_comma">!</string>
<string name="more_keys_for_tablet_comma">!</string>
- <string name="keyhintlabel_for_tablet_period">\?</string>
- <string name="more_keys_for_tablet_period">\?</string>
+ <string name="keyhintlabel_for_period">\?</string>
+ <string name="more_keys_for_period">\?</string>
</resources>
diff --git a/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml
index cc09f7f..44aa64c 100644
--- a/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml
+++ b/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml
@@ -169,9 +169,9 @@
<string name="keylabel_for_tablet_comma">,</string>
<string name="keyhintlabel_for_tablet_comma"></string>
<string name="more_keys_for_tablet_comma"></string>
- <string name="keyhintlabel_for_tablet_period"></string>
+ <string name="keyhintlabel_for_period"></string>
<!-- U+2026: "…" HORIZONTAL ELLIPSIS -->
- <string name="more_keys_for_tablet_period">…</string>
+ <string name="more_keys_for_period">…</string>
<string name="keylabel_for_apostrophe">\'</string>
<string name="keyhintlabel_for_apostrophe">\"</string>
<string name="more_keys_for_apostrophe">\"</string>
