Merge "Fix revert of committed words"
diff --git a/dictionaries/ru_wordlist.combined.gz b/dictionaries/ru_wordlist.combined.gz
index 1cfab4e..8b67e7c 100644
--- a/dictionaries/ru_wordlist.combined.gz
+++ b/dictionaries/ru_wordlist.combined.gz
Binary files differ
diff --git a/java/res/raw/main_ru.dict b/java/res/raw/main_ru.dict
index 7074416..3e23617 100644
--- a/java/res/raw/main_ru.dict
+++ b/java/res/raw/main_ru.dict
Binary files differ
diff --git a/java/res/values-ru/strings.xml b/java/res/values-ru/strings.xml
index 3ee9869..b00fa65 100644
--- a/java/res/values-ru/strings.xml
+++ b/java/res/values-ru/strings.xml
@@ -33,19 +33,19 @@
<string name="misc_category" msgid="6894192814868233453">"Другие варианты"</string>
<string name="advanced_settings" msgid="362895144495591463">"Расширенные настройки"</string>
<string name="advanced_settings_summary" msgid="4487980456152830271">"Для опытных пользователей"</string>
- <string name="include_other_imes_in_language_switch_list" msgid="4533689960308565519">"Другой способ ввода"</string>
+ <string name="include_other_imes_in_language_switch_list" msgid="4533689960308565519">"Смена способов ввода"</string>
<string name="include_other_imes_in_language_switch_list_summary" msgid="840637129103317635">"Клавиша переключения языков также служит для смены способа ввода"</string>
<string name="show_language_switch_key" msgid="5915478828318774384">"Клавиша смены языка"</string>
<string name="show_language_switch_key_summary" msgid="7343403647474265713">"Показывать, когда включено несколько раскладок"</string>
- <string name="sliding_key_input_preview" msgid="6604262359510068370">"Показывать индикатор перехода"</string>
- <string name="sliding_key_input_preview_summary" msgid="6340524345729093886">"Индикатор перехода между регистрами или цифр. и букв. режимами"</string>
+ <string name="sliding_key_input_preview" msgid="6604262359510068370">"След от переключателя режима"</string>
+ <string name="sliding_key_input_preview_summary" msgid="6340524345729093886">"Показывать след при проведении пальцем от кнопок Shift и \"Символы\""</string>
<string name="key_preview_popup_dismiss_delay" msgid="6213164897443068248">"Задержка закрытия"</string>
<string name="key_preview_popup_dismiss_no_delay" msgid="2096123151571458064">"Без задержки"</string>
<string name="key_preview_popup_dismiss_default_delay" msgid="2166964333903906734">"По умолчанию"</string>
<string name="abbreviation_unit_milliseconds" msgid="8700286094028323363">"<xliff:g id="MILLISECONDS">%s</xliff:g> мс"</string>
<string name="use_contacts_dict" msgid="4435317977804180815">"Подсказывать имена"</string>
<string name="use_contacts_dict_summary" msgid="6599983334507879959">"Подсказывать исправления на основе имен из списка контактов"</string>
- <string name="use_double_space_period" msgid="8781529969425082860">"Точка с пробелом"</string>
+ <string name="use_double_space_period" msgid="8781529969425082860">"Ставить точки автоматически"</string>
<string name="use_double_space_period_summary" msgid="6532892187247952799">"Вводить точку с пробелом двойным нажатием кнопки \"Пробел\"."</string>
<string name="auto_cap" msgid="1719746674854628252">"Заглавные автоматически"</string>
<string name="auto_cap_summary" msgid="7934452761022946874">"Писать первое слово предложения с прописной буквы"</string>
@@ -164,7 +164,7 @@
<string name="prefs_key_longpress_timeout_settings" msgid="6102240298932897873">"Долгое нажатие"</string>
<string name="prefs_keypress_vibration_duration_settings" msgid="7918341459947439226">"Вибросигнал при нажатии клавиш"</string>
<string name="prefs_keypress_sound_volume_settings" msgid="6027007337036891623">"Звук при нажатии клавиш"</string>
- <string name="prefs_read_external_dictionary" msgid="2588931418575013067">"Считывать данные из внешнего словаря"</string>
+ <string name="prefs_read_external_dictionary" msgid="2588931418575013067">"Загрузить словарь из файла"</string>
<string name="read_external_dictionary_no_files_message" msgid="4947420942224623792">"В папке \"Загрузки\" нет словарей"</string>
<string name="read_external_dictionary_multiple_files_title" msgid="7637749044265808628">"Выберите файл словаря"</string>
<string name="read_external_dictionary_confirm_install_message" msgid="6898610163768980870">"Установить этот файл для следующего языка: <xliff:g id="LOCALE_NAME">%s</xliff:g>?"</string>
diff --git a/java/res/values-zh-rCN/strings.xml b/java/res/values-zh-rCN/strings.xml
index 4bb2ce4..a5bbab2 100644
--- a/java/res/values-zh-rCN/strings.xml
+++ b/java/res/values-zh-rCN/strings.xml
@@ -212,9 +212,9 @@
<string name="install_dict" msgid="180852772562189365">"安装"</string>
<string name="cancel_download_dict" msgid="7843340278507019303">"取消"</string>
<string name="delete_dict" msgid="756853268088330054">"删除"</string>
- <string name="should_download_over_metered_prompt" msgid="2878629598667658845">"支持您移动设备上所选语言的词典现已可供下载啦！<br/>建议您<b>下载</b>这部<xliff:g id="LANGUAGE">%1$s</xliff:g>词典，以享受更好的输入体验。<br/><br/>通过 3G 进行下载可能需要 1 到 2 分钟的时间。如果您使用的不是<b>无流量限制的套餐</b>，则可能需要支付一定的费用。<br/>如果您不确定自己使用的是哪种流量套餐，建议您使用 Wi-Fi 连接自动开始下载。<br/><br/>提示：您可以访问移动设备的<b>设置</b>菜单中的<b>语言和输入法</b>，来下载和删除词典。"</string>
+ <string name="should_download_over_metered_prompt" msgid="2878629598667658845">"支持您移动设备上所选语言的词典现已可供下载啦！<br/>建议您<b>下载</b>这部<xliff:g id="LANGUAGE">%1$s</xliff:g>词典，以享受更好的输入体验。<br/><br/>通过 3G 进行下载可能需要 1 到 2 分钟的时间。如果您使用的不是<b>无流量限制的套餐</b>，则可能需要支付一定的费用。<br/>如果您不确定自己使用的是哪种流量套餐，建议您使用 WLAN 连接自动开始下载。<br/><br/>提示：您可以访问移动设备的<b>设置</b>菜单中的<b>语言和输入法</b>，来下载和删除词典。"</string>
<string name="download_over_metered" msgid="1643065851159409546">"立即下载 (<xliff:g id="SIZE_IN_MEGABYTES">%1$.1f</xliff:g>MB)"</string>
- <string name="do_not_download_over_metered" msgid="2176209579313941583">"通过 Wi-Fi 下载"</string>
+ <string name="do_not_download_over_metered" msgid="2176209579313941583">"通过 WLAN 下载"</string>
<string name="dict_available_notification_title" msgid="6514288591959117288">"<xliff:g id="LANGUAGE">%1$s</xliff:g>词典可供下载"</string>
<string name="dict_available_notification_description" msgid="1075194169443163487">"按此通知即可查看和下载"</string>
<string name="toast_downloading_suggestions" msgid="1313027353588566660">"下载中：很快就能启用<xliff:g id="LANGUAGE">%1$s</xliff:g>的词典建议服务了！"</string>
diff --git a/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsFragment.java b/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsFragment.java
index 1e93e7e..4b89d20 100644
--- a/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsFragment.java
+++ b/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsFragment.java
@@ -304,7 +304,7 @@
// the description.
final String key = matchLevelString + "." + description + "." + wordlistId;
final WordListPreference existingPref = prefMap.get(key);
- if (null == existingPref || hasPriority(status, existingPref.mStatus)) {
+ if (null == existingPref || existingPref.hasPriorityOver(status)) {
final WordListPreference oldPreference = mCurrentPreferenceMap.get(key);
final WordListPreference pref;
if (null != oldPreference
@@ -315,7 +315,7 @@
// need to be the same, others have been tested through the key of the
// map. Also, status may differ so we don't want to use #equals() here.
pref = oldPreference;
- pref.mStatus = status;
+ pref.setStatus(status);
} else {
// Otherwise, discard it and create a new one instead.
pref = new WordListPreference(activity, mDictionaryListInterfaceState,
@@ -331,18 +331,6 @@
}
}
- /**
- * Finds out if a given status has priority over another for display order.
- *
- * @param newStatus
- * @param oldStatus
- * @return whether newStatus has priority over oldStatus.
- */
- private static boolean hasPriority(final int newStatus, final int oldStatus) {
- // Both of these should be one of MetadataDbHelper.STATUS_*
- return newStatus > oldStatus;
- }
-
@Override
public boolean onOptionsItemSelected(final MenuItem item) {
switch (item.getItemId()) {
diff --git a/java/src/com/android/inputmethod/dictionarypack/WordListPreference.java b/java/src/com/android/inputmethod/dictionarypack/WordListPreference.java
index a1031c2..7ec7e9c 100644
--- a/java/src/com/android/inputmethod/dictionarypack/WordListPreference.java
+++ b/java/src/com/android/inputmethod/dictionarypack/WordListPreference.java
@@ -61,7 +61,7 @@
public final Locale mLocale;
public final String mDescription;
// The status
- public int mStatus;
+ private int mStatus;
// The size of the dictionary file
private final int mFilesize;
@@ -92,7 +92,7 @@
setKey(wordlistId);
}
- private void setStatus(final int status) {
+ public void setStatus(final int status) {
if (status == mStatus) return;
mStatus = status;
setSummary(getSummary(status));
@@ -106,6 +106,11 @@
return mInterfaceState.addToCacheAndReturnView(newView);
}
+ public boolean hasPriorityOver(final int otherPrefStatus) {
+ // Both of these should be one of MetadataDbHelper.STATUS_*
+ return mStatus > otherPrefStatus;
+ }
+
private String getSummary(final int status) {
switch (status) {
// If we are deleting the word list, for the user it's like it's already deleted.
diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
index 83f1090..7295efa 100644
--- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
+++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
@@ -274,30 +274,30 @@
// Implements {@link KeyboardState.SwitchActions}.
@Override
- public void startDoubleTapTimer() {
+ public void startDoubleTapShiftKeyTimer() {
final MainKeyboardView keyboardView = getMainKeyboardView();
if (keyboardView != null) {
final TimerProxy timer = keyboardView.getTimerProxy();
- timer.startDoubleTapTimer();
+ timer.startDoubleTapShiftKeyTimer();
}
}
// Implements {@link KeyboardState.SwitchActions}.
@Override
- public void cancelDoubleTapTimer() {
+ public void cancelDoubleTapShiftKeyTimer() {
final MainKeyboardView keyboardView = getMainKeyboardView();
if (keyboardView != null) {
final TimerProxy timer = keyboardView.getTimerProxy();
- timer.cancelDoubleTapTimer();
+ timer.cancelDoubleTapShiftKeyTimer();
}
}
// Implements {@link KeyboardState.SwitchActions}.
@Override
- public boolean isInDoubleTapTimeout() {
+ public boolean isInDoubleTapShiftKeyTimeout() {
final MainKeyboardView keyboardView = getMainKeyboardView();
return (keyboardView != null)
- ? keyboardView.getTimerProxy().isInDoubleTapTimeout() : false;
+ ? keyboardView.getTimerProxy().isInDoubleTapShiftKeyTimeout() : false;
}
// Implements {@link KeyboardState.SwitchActions}.
diff --git a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java
index 7493df8..435adb6 100644
--- a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java
+++ b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java
@@ -202,7 +202,7 @@
private static final int MSG_TYPING_STATE_EXPIRED = 0;
private static final int MSG_REPEAT_KEY = 1;
private static final int MSG_LONGPRESS_KEY = 2;
- private static final int MSG_DOUBLE_TAP = 3;
+ private static final int MSG_DOUBLE_TAP_SHIFT_KEY = 3;
private static final int MSG_UPDATE_BATCH_INPUT = 4;
private final int mKeyRepeatStartTimeout;
@@ -392,19 +392,19 @@
}
@Override
- public void startDoubleTapTimer() {
- sendMessageDelayed(obtainMessage(MSG_DOUBLE_TAP),
+ public void startDoubleTapShiftKeyTimer() {
+ sendMessageDelayed(obtainMessage(MSG_DOUBLE_TAP_SHIFT_KEY),
ViewConfiguration.getDoubleTapTimeout());
}
@Override
- public void cancelDoubleTapTimer() {
- removeMessages(MSG_DOUBLE_TAP);
+ public void cancelDoubleTapShiftKeyTimer() {
+ removeMessages(MSG_DOUBLE_TAP_SHIFT_KEY);
}
@Override
- public boolean isInDoubleTapTimeout() {
- return hasMessages(MSG_DOUBLE_TAP);
+ public boolean isInDoubleTapShiftKeyTimeout() {
+ return hasMessages(MSG_DOUBLE_TAP_SHIFT_KEY);
}
@Override
diff --git a/java/src/com/android/inputmethod/keyboard/PointerTracker.java b/java/src/com/android/inputmethod/keyboard/PointerTracker.java
index 5df7011..a28bd1a 100644
--- a/java/src/com/android/inputmethod/keyboard/PointerTracker.java
+++ b/java/src/com/android/inputmethod/keyboard/PointerTracker.java
@@ -94,9 +94,9 @@
public void startLongPressTimer(PointerTracker tracker);
public void startLongPressTimer(int code);
public void cancelLongPressTimer();
- public void startDoubleTapTimer();
- public void cancelDoubleTapTimer();
- public boolean isInDoubleTapTimeout();
+ public void startDoubleTapShiftKeyTimer();
+ public void cancelDoubleTapShiftKeyTimer();
+ public boolean isInDoubleTapShiftKeyTimeout();
public void cancelKeyTimers();
public void startUpdateBatchInputTimer(PointerTracker tracker);
public void cancelUpdateBatchInputTimer(PointerTracker tracker);
@@ -116,11 +116,11 @@
@Override
public void cancelLongPressTimer() {}
@Override
- public void startDoubleTapTimer() {}
+ public void startDoubleTapShiftKeyTimer() {}
@Override
- public void cancelDoubleTapTimer() {}
+ public void cancelDoubleTapShiftKeyTimer() {}
@Override
- public boolean isInDoubleTapTimeout() { return false; }
+ public boolean isInDoubleTapShiftKeyTimeout() { return false; }
@Override
public void cancelKeyTimers() {}
@Override
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
index 9f6374b..bb6dec6 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
@@ -53,9 +53,9 @@
*/
public void requestUpdatingShiftState();
- public void startDoubleTapTimer();
- public boolean isInDoubleTapTimeout();
- public void cancelDoubleTapTimer();
+ public void startDoubleTapShiftKeyTimer();
+ public boolean isInDoubleTapShiftKeyTimeout();
+ public void cancelDoubleTapShiftKeyTimer();
public void startLongPressTimer(int code);
public void cancelLongPressTimer();
}
@@ -325,7 +325,7 @@
} else if (code == Constants.CODE_SWITCH_ALPHA_SYMBOL) {
onPressSymbol();
} else {
- mSwitchActions.cancelDoubleTapTimer();
+ mSwitchActions.cancelDoubleTapShiftKeyTimer();
mSwitchActions.cancelLongPressTimer();
mLongPressShiftLockFired = false;
mShiftKeyState.onOtherKeyPressed();
@@ -450,10 +450,10 @@
// importantly the double tap timer.
if (RecapitalizeStatus.NOT_A_RECAPITALIZE_MODE != mRecapitalizeMode) return;
if (mIsAlphabetMode) {
- mIsInDoubleTapShiftKey = mSwitchActions.isInDoubleTapTimeout();
+ mIsInDoubleTapShiftKey = mSwitchActions.isInDoubleTapShiftKeyTimeout();
if (!mIsInDoubleTapShiftKey) {
// This is first tap.
- mSwitchActions.startDoubleTapTimer();
+ mSwitchActions.startDoubleTapShiftKeyTimer();
}
if (mIsInDoubleTapShiftKey) {
if (mAlphabetShiftState.isManualShifted() || mIsInAlphabetUnshiftedFromShifted) {
diff --git a/java/src/com/android/inputmethod/latin/AssetFileAddress.java b/java/src/com/android/inputmethod/latin/AssetFileAddress.java
index 47c750f..8751925 100644
--- a/java/src/com/android/inputmethod/latin/AssetFileAddress.java
+++ b/java/src/com/android/inputmethod/latin/AssetFileAddress.java
@@ -24,7 +24,7 @@
* the package file. Open it correctly thus requires the name of the package it is in, but
* also the offset in the file and the length of this data. This class encapsulates these three.
*/
-final class AssetFileAddress {
+public final class AssetFileAddress {
public final String mFilename;
public final long mOffset;
public final long mLength;
diff --git a/java/src/com/android/inputmethod/latin/DictionaryFactory.java b/java/src/com/android/inputmethod/latin/DictionaryFactory.java
index 40e5167..4514ec2 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryFactory.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryFactory.java
@@ -21,6 +21,8 @@
import android.content.res.Resources;
import android.util.Log;
+import com.android.inputmethod.annotations.UsedForTesting;
+
import java.io.File;
import java.util.ArrayList;
import java.util.LinkedList;
@@ -126,21 +128,22 @@
/**
* Create a dictionary from passed data. This is intended for unit tests only.
- * @param dictionary the file to read
- * @param startOffset the offset in the file where the data starts
- * @param length the length of the data
+ * @param dictionaryList the list of files to read, with their offsets and lengths
* @param useFullEditDistance whether to use the full edit distance in suggestions
* @return the created dictionary, or null.
*/
- public static Dictionary createDictionaryForTest(File dictionary, long startOffset, long length,
+ @UsedForTesting
+ public static Dictionary createDictionaryForTest(final AssetFileAddress[] dictionaryList,
final boolean useFullEditDistance, Locale locale) {
- if (dictionary.isFile()) {
- return new BinaryDictionary(dictionary.getAbsolutePath(), startOffset, length,
- useFullEditDistance, locale, Dictionary.TYPE_MAIN);
- } else {
- Log.e(TAG, "Could not find the file. path=" + dictionary.getAbsolutePath());
- return null;
+ final DictionaryCollection dictionaryCollection =
+ new DictionaryCollection(Dictionary.TYPE_MAIN);
+ for (final AssetFileAddress address : dictionaryList) {
+ final BinaryDictionary binaryDictionary = new BinaryDictionary(address.mFilename,
+ address.mOffset, address.mLength, useFullEditDistance, locale,
+ Dictionary.TYPE_MAIN);
+ dictionaryCollection.addDictionary(binaryDictionary);
}
+ return dictionaryCollection;
}
/**
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index c9a42a3..98008ac 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -1637,8 +1637,10 @@
public void onStartBatchInput(final LatinIME latinIme) {
synchronized (mLock) {
mHandler.removeMessages(MSG_UPDATE_GESTURE_PREVIEW_AND_SUGGESTION_STRIP);
- mLatinIme = latinIme;
mInBatchInput = true;
+ mLatinIme = latinIme;
+ mLatinIme.mHandler.showGesturePreviewAndSuggestionStrip(
+ SuggestedWords.EMPTY, false /* dismissGestureFloatingPreviewText */);
}
}
@@ -1797,8 +1799,6 @@
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
final String word = mWordComposer.getTypedWord();
ResearchLogger.latinIME_handleBackspace_batch(word, 1);
- ResearchLogger.getInstance().uncommitCurrentLogUnit(
- word, false /* dumpCurrentLogUnit */);
}
final String rejectedSuggestion = mWordComposer.getTypedWord();
mWordComposer.reset();
@@ -1825,6 +1825,9 @@
// like the smiley key or the .com key.
final int length = mEnteredText.length();
mConnection.deleteSurroundingText(length, 0);
+ if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
+ ResearchLogger.latinIME_handleBackspace_cancelTextInput(mEnteredText);
+ }
mEnteredText = null;
// If we have mEnteredText, then we know that mHasUncommittedTypedChars == false.
// In addition we know that spaceState is false, and that we should not be
@@ -1858,7 +1861,8 @@
mLastSelectionEnd = mLastSelectionStart;
mConnection.deleteSurroundingText(numCharsDeleted, 0);
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
- ResearchLogger.latinIME_handleBackspace(numCharsDeleted);
+ ResearchLogger.latinIME_handleBackspace(numCharsDeleted,
+ false /* shouldUncommitLogUnit */);
}
} else {
// There is no selection, just delete one character.
@@ -1876,12 +1880,13 @@
mConnection.deleteSurroundingText(1, 0);
}
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
- ResearchLogger.latinIME_handleBackspace(1);
+ ResearchLogger.latinIME_handleBackspace(1, true /* shouldUncommitLogUnit */);
}
if (mDeleteCount > DELETE_ACCELERATE_AT) {
mConnection.deleteSurroundingText(1, 0);
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
- ResearchLogger.latinIME_handleBackspace(1);
+ ResearchLogger.latinIME_handleBackspace(1,
+ true /* shouldUncommitLogUnit */);
}
}
}
diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java
index 5d580f2..e783e6d 100644
--- a/java/src/com/android/inputmethod/latin/Suggest.java
+++ b/java/src/com/android/inputmethod/latin/Suggest.java
@@ -23,7 +23,6 @@
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
-import java.io.File;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
@@ -77,9 +76,9 @@
}
@UsedForTesting
- Suggest(final File dictionary, final long startOffset, final long length, final Locale locale) {
- final Dictionary mainDict = DictionaryFactory.createDictionaryForTest(dictionary,
- startOffset, length /* useFullEditDistance */, false, locale);
+ Suggest(final AssetFileAddress[] dictionaryList, final Locale locale) {
+ final Dictionary mainDict = DictionaryFactory.createDictionaryForTest(dictionaryList,
+ false /* useFullEditDistance */, locale);
mLocale = locale;
mMainDictionary = mainDict;
addOrReplaceDictionary(mDictionaries, Dictionary.TYPE_MAIN, mainDict);
diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java
index 2f2ec35..e4c5a06 100644
--- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java
+++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java
@@ -208,6 +208,9 @@
private CharSequence getStyledSuggestedWord(final SuggestedWords suggestedWords,
final int indexInSuggestedWords) {
+ if (indexInSuggestedWords >= suggestedWords.size()) {
+ return null;
+ }
final String word = suggestedWords.getWord(indexInSuggestedWords);
final boolean isAutoCorrect = indexInSuggestedWords == 1
&& suggestedWords.willAutoCorrect();
diff --git a/java/src/com/android/inputmethod/research/ResearchLogger.java b/java/src/com/android/inputmethod/research/ResearchLogger.java
index 2da6d01..57b740b 100644
--- a/java/src/com/android/inputmethod/research/ResearchLogger.java
+++ b/java/src/com/android/inputmethod/research/ResearchLogger.java
@@ -865,7 +865,10 @@
// Check that expected word matches.
if (oldLogUnit != null) {
final String oldLogUnitWords = oldLogUnit.getWordsAsString();
- if (oldLogUnitWords != null && !oldLogUnitWords.equals(expectedWord)) {
+ // Because the word is stored in the LogUnit with digits scrubbed, the comparison must
+ // be made on a scrubbed version of the expectedWord as well.
+ if (oldLogUnitWords != null && !oldLogUnitWords.equals(
+ scrubDigitsFromString(expectedWord))) {
return;
}
}
@@ -1276,6 +1279,16 @@
}
/**
+ * Log a revert of onTextInput() (known in the IME as "EnteredText").
+ *
+ * SystemResponse: Remove the LogUnit recording the textInput
+ */
+ public static void latinIME_handleBackspace_cancelTextInput(final String text) {
+ final ResearchLogger researchLogger = getInstance();
+ researchLogger.uncommitCurrentLogUnit(text, true /* dumpCurrentLogUnit */);
+ }
+
+ /**
* Log a call to LatinIME.pickSuggestionManually().
*
* UserAction: The user has chosen a specific word from the suggestion strip.
@@ -1595,7 +1608,12 @@
private static final LogStatement LOGSTATEMENT_RICHINPUTCONNECTION_REVERTDOUBLESPACEPERIOD =
new LogStatement("RichInputConnectionRevertDoubleSpacePeriod", false, false);
public static void richInputConnection_revertDoubleSpacePeriod() {
- getInstance().enqueueEvent(LOGSTATEMENT_RICHINPUTCONNECTION_REVERTDOUBLESPACEPERIOD);
+ final ResearchLogger researchLogger = getInstance();
+ // An extra LogUnit is added for the period; this is removed here because of the revert.
+ researchLogger.uncommitCurrentLogUnit(null, true /* dumpCurrentLogUnit */);
+ // TODO: This will probably be lost as the user backspaces further. Figure out how to put
+ // it into the right logUnit.
+ researchLogger.enqueueEvent(LOGSTATEMENT_RICHINPUTCONNECTION_REVERTDOUBLESPACEPERIOD);
}
/**
@@ -1826,17 +1844,26 @@
SystemClock.uptimeMillis());
}
+ private static final LogStatement LOGSTATEMENT_LATINIME_HANDLEBACKSPACE =
+ new LogStatement("LatinIMEHandleBackspace", true, false, "numCharacters");
/**
* Log a call to LatinIME.handleBackspace() that is not a batch delete.
*
* UserInput: The user is deleting one or more characters by hitting the backspace key once.
* The covers single character deletes as well as deleting selections.
+ *
+ * @param numCharacters how many characters the backspace operation deleted
+ * @param shouldUncommitLogUnit whether to uncommit the last {@code LogUnit} in the
+ * {@code LogBuffer}
*/
- private static final LogStatement LOGSTATEMENT_LATINIME_HANDLEBACKSPACE =
- new LogStatement("LatinIMEHandleBackspace", true, false, "numCharacters");
- public static void latinIME_handleBackspace(final int numCharacters) {
+ public static void latinIME_handleBackspace(final int numCharacters,
+ final boolean shouldUncommitLogUnit) {
final ResearchLogger researchLogger = getInstance();
researchLogger.enqueueEvent(LOGSTATEMENT_LATINIME_HANDLEBACKSPACE, numCharacters);
+ if (shouldUncommitLogUnit) {
+ ResearchLogger.getInstance().uncommitCurrentLogUnit(
+ null, true /* dumpCurrentLogUnit */);
+ }
}
/**
@@ -1854,6 +1881,8 @@
numCharacters);
researchLogger.mStatistics.recordGestureDelete(deletedText.length(),
SystemClock.uptimeMillis());
+ researchLogger.uncommitCurrentLogUnit(deletedText.toString(),
+ false /* dumpCurrentLogUnit */);
}
/**
diff --git a/native/jni/Android.mk b/native/jni/Android.mk
index 34b3524..7ca4057 100644
--- a/native/jni/Android.mk
+++ b/native/jni/Android.mk
@@ -46,18 +46,17 @@
jni_common.cpp
LATIN_IME_CORE_SRC_FILES := \
- bigram_dictionary.cpp \
- char_utils.cpp \
- correction.cpp \
- dic_traverse_wrapper.cpp \
- unigram_dictionary.cpp \
- words_priority_queue.cpp \
suggest/core/suggest.cpp \
+ $(addprefix obsolete/, \
+ correction.cpp) \
$(addprefix suggest/core/dicnode/, \
dic_node.cpp \
dic_node_utils.cpp \
dic_nodes_cache.cpp) \
$(addprefix suggest/core/dictionary/, \
+ bigram_dictionary.cpp \
+ binary_dictionary_format.cpp \
+ byte_array_utils.cpp \
dictionary.cpp \
digraph_utils.cpp) \
$(addprefix suggest/core/layout/, \
@@ -74,7 +73,8 @@
typing_scoring.cpp \
typing_suggest_policy.cpp \
typing_traversal.cpp \
- typing_weighting.cpp)
+ typing_weighting.cpp) \
+ utils/char_utils.cpp
LOCAL_SRC_FILES := \
$(LATIN_IME_JNI_SRC_FILES) \
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 9f5e2ae..34764c3 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -31,12 +31,13 @@
#include "com_android_inputmethod_latin_BinaryDictionary.h"
-#include "correction.h"
#include "jni.h"
#include "jni_common.h"
-#include "suggest_options.h"
-#include "suggest/core/dictionary/binary_format.h"
+#include "obsolete/correction.h"
+#include "suggest/core/dictionary/binary_dictionary_format.h"
+#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/dictionary.h"
+#include "suggest/core/suggest_options.h"
namespace latinime {
@@ -110,8 +111,8 @@
return 0;
}
Dictionary *dictionary = 0;
- if (BinaryFormat::UNKNOWN_FORMAT
- == BinaryFormat::detectFormat(static_cast<uint8_t *>(dictBuf),
+ if (BinaryDictionaryFormat::UNKNOWN_VERSION
+ == BinaryDictionaryFormat::detectFormatVersion(static_cast<uint8_t *>(dictBuf),
static_cast<int>(dictSize))) {
AKLOGE("DICT: dictionary format is unknown, bad magic number");
#ifdef USE_MMAP_FOR_DICTIONARY
@@ -136,7 +137,8 @@
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return 0;
ProximityInfo *pInfo = reinterpret_cast<ProximityInfo *>(proximityInfo);
- void *traverseSession = reinterpret_cast<void *>(dicTraverseSession);
+ DicTraverseSession *traverseSession =
+ reinterpret_cast<DicTraverseSession *>(dicTraverseSession);
// Input values
int xCoordinates[inputSize];
@@ -260,7 +262,7 @@
static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return;
- const void *dictBuf = dictionary->getDict();
+ const void *dictBuf = dictionary->getBinaryDictionaryInfo()->getDictBuf();
if (!dictBuf) return;
#ifdef USE_MMAP_FOR_DICTIONARY
releaseDictBuf(static_cast<const char *>(dictBuf) - dictionary->getDictBufAdjust(),
diff --git a/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp b/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp
index 08124ff..d454150 100644
--- a/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp
+++ b/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp
@@ -17,36 +17,37 @@
#define LOG_TAG "LatinIME: jni: Session"
#include "com_android_inputmethod_latin_DicTraverseSession.h"
+
#include "defines.h"
-#include "dic_traverse_wrapper.h"
#include "jni.h"
#include "jni_common.h"
+#include "suggest/core/session/dic_traverse_session.h"
namespace latinime {
class Dictionary;
static jlong latinime_setDicTraverseSession(JNIEnv *env, jclass clazz, jstring localeJStr) {
- void *traverseSession = DicTraverseWrapper::getDicTraverseSession(env, localeJStr);
+ void *traverseSession = DicTraverseSession::getSessionInstance(env, localeJStr);
return reinterpret_cast<jlong>(traverseSession);
}
static void latinime_initDicTraverseSession(JNIEnv *env, jclass clazz, jlong traverseSession,
jlong dictionary, jintArray previousWord, jint previousWordLength) {
- void *ts = reinterpret_cast<void *>(traverseSession);
+ DicTraverseSession *ts = reinterpret_cast<DicTraverseSession *>(traverseSession);
Dictionary *dict = reinterpret_cast<Dictionary *>(dictionary);
if (!previousWord) {
- DicTraverseWrapper::initDicTraverseSession(
+ DicTraverseSession::initSessionInstance(
ts, dict, 0 /* prevWord */, 0 /* prevWordLength*/, 0 /* suggestOptions */);
return;
}
int prevWord[previousWordLength];
env->GetIntArrayRegion(previousWord, 0, previousWordLength, prevWord);
- DicTraverseWrapper::initDicTraverseSession(
+ DicTraverseSession::initSessionInstance(
ts, dict, prevWord, previousWordLength, 0 /* suggestOptions */);
}
static void latinime_releaseDicTraverseSession(JNIEnv *env, jclass clazz, jlong traverseSession) {
- void *ts = reinterpret_cast<void *>(traverseSession);
- DicTraverseWrapper::releaseDicTraverseSession(ts);
+ DicTraverseSession *ts = reinterpret_cast<DicTraverseSession *>(traverseSession);
+ DicTraverseSession::releaseSessionInstance(ts);
}
static JNINativeMethod sMethods[] = {
diff --git a/native/jni/src/char_utils.h b/native/jni/src/char_utils.h
deleted file mode 100644
index b429f40..0000000
--- a/native/jni/src/char_utils.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright (C) 2010 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_CHAR_UTILS_H
-#define LATINIME_CHAR_UTILS_H
-
-#include <cctype>
-
-#include "defines.h"
-
-namespace latinime {
-
-inline static bool isAsciiUpper(int c) {
- // Note: isupper(...) reports false positives for some Cyrillic characters, causing them to
- // be incorrectly lower-cased using toAsciiLower(...) rather than latin_tolower(...).
- return (c >= 'A' && c <= 'Z');
-}
-
-inline static int toAsciiLower(int c) {
- return c - 'A' + 'a';
-}
-
-inline static bool isAscii(int c) {
- return isascii(c) != 0;
-}
-
-unsigned short latin_tolower(const unsigned short c);
-
-/**
- * Table mapping most combined Latin, Greek, and Cyrillic characters
- * to their base characters. If c is in range, BASE_CHARS[c] == c
- * if c is not a combined character, or the base character if it
- * is combined.
- */
-static const int BASE_CHARS_SIZE = 0x0500;
-extern const unsigned short BASE_CHARS[BASE_CHARS_SIZE];
-
-inline static int toBaseCodePoint(int c) {
- if (c < BASE_CHARS_SIZE) {
- return static_cast<int>(BASE_CHARS[c]);
- }
- return c;
-}
-
-AK_FORCE_INLINE static int toLowerCase(const int c) {
- if (isAsciiUpper(c)) {
- return toAsciiLower(c);
- }
- if (isAscii(c)) {
- return c;
- }
- return static_cast<int>(latin_tolower(static_cast<unsigned short>(c)));
-}
-
-AK_FORCE_INLINE static int toBaseLowerCase(const int c) {
- return toLowerCase(toBaseCodePoint(c));
-}
-
-inline static bool isIntentionalOmissionCodePoint(const int codePoint) {
- // TODO: Do not hardcode here
- return codePoint == KEYCODE_SINGLE_QUOTE || codePoint == KEYCODE_HYPHEN_MINUS;
-}
-
-inline static int getCodePointCount(const int arraySize, const int *const codePoints) {
- int size = 0;
- for (; size < arraySize; ++size) {
- if (codePoints[size] == '\0') {
- break;
- }
- }
- return size;
-}
-
-} // namespace latinime
-#endif // LATINIME_CHAR_UTILS_H
diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h
index eb59744..4e3a7b2 100644
--- a/native/jni/src/defines.h
+++ b/native/jni/src/defines.h
@@ -203,14 +203,12 @@
#define DEBUG_DICT true
#define DEBUG_DICT_FULL false
#define DEBUG_EDIT_DISTANCE false
-#define DEBUG_SHOW_FOUND_WORD false
#define DEBUG_NODE DEBUG_DICT_FULL
#define DEBUG_TRACE DEBUG_DICT_FULL
#define DEBUG_PROXIMITY_INFO false
#define DEBUG_PROXIMITY_CHARS false
#define DEBUG_CORRECTION false
#define DEBUG_CORRECTION_FREQ false
-#define DEBUG_WORDS_PRIORITY_QUEUE false
#define DEBUG_SAMPLING_POINTS false
#define DEBUG_POINTS_PROBABILITY false
#define DEBUG_DOUBLE_LETTER false
@@ -229,14 +227,12 @@
#define DEBUG_DICT false
#define DEBUG_DICT_FULL false
#define DEBUG_EDIT_DISTANCE false
-#define DEBUG_SHOW_FOUND_WORD false
#define DEBUG_NODE false
#define DEBUG_TRACE false
#define DEBUG_PROXIMITY_INFO false
#define DEBUG_PROXIMITY_CHARS false
#define DEBUG_CORRECTION false
#define DEBUG_CORRECTION_FREQ false
-#define DEBUG_WORDS_PRIORITY_QUEUE false
#define DEBUG_SAMPLING_POINTS false
#define DEBUG_POINTS_PROBABILITY false
#define DEBUG_DOUBLE_LETTER false
@@ -289,7 +285,6 @@
#define CALIBRATE_SCORE_BY_TOUCH_COORDINATES true
#define SUGGEST_MULTIPLE_WORDS true
-#define USE_SUGGEST_INTERFACE_FOR_TYPING true
#define SUGGEST_INTERFACE_OUTPUT_SCALE 1000000.0f
// The following "rate"s are used as a multiplier before dividing by 100, so they are in percent.
@@ -311,39 +306,19 @@
#define FIRST_CHAR_DIFFERENT_DEMOTION_RATE 96
#define TWO_WORDS_CAPITALIZED_DEMOTION_RATE 50
#define TWO_WORDS_CORRECTION_DEMOTION_BASE 80
-#define TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER 1
#define ZERO_DISTANCE_PROMOTION_RATE 110.0f
#define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f
#define HALF_SCORE_SQUARED_RADIUS 32.0f
#define MAX_PROBABILITY 255
#define MAX_BIGRAM_ENCODED_PROBABILITY 15
+#define MULTIPLE_WORDS_DEMOTION_RATE 80
// Assuming locale strings such as en_US, sr-Latn etc.
#define MAX_LOCALE_STRING_LENGTH 10
-// Word limit for sub queues used in WordsPriorityQueuePool. Sub queues are temporary queues used
-// for better performance.
-// Holds up to 1 candidate for each word
-#define SUB_QUEUE_MAX_WORDS 1
-#define SUB_QUEUE_MAX_COUNT 10
-#define SUB_QUEUE_MIN_WORD_LENGTH 4
-// TODO: Extend this limitation
-#define MULTIPLE_WORDS_SUGGESTION_MAX_WORDS 5
-// TODO: Remove this limitation
-#define MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH 12
-// TODO: Remove this limitation
-#define MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT 45
-#define MULTIPLE_WORDS_DEMOTION_RATE 80
-#define MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION 6
-
-#define TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD 0.35f
-#define START_TWO_WORDS_CORRECTION_THRESHOLD 0.185f
/* heuristic... This should be changed if we change the unit of the probability. */
#define SUPPRESS_SHORT_MULTIPLE_WORDS_THRESHOLD_FREQ (MAX_PROBABILITY * 58 / 100)
-#define MAX_DEPTH_MULTIPLIER 3
-#define FIRST_WORD_INDEX 0
-
// Max value for length, distance and probability which are used in weighting
// TODO: Remove
#define MAX_VALUE_FOR_WEIGHTING 10000000
@@ -351,12 +326,6 @@
// The max number of the keys in one keyboard layout
#define MAX_KEY_COUNT_IN_A_KEYBOARD 64
-// TODO: Reduce this constant if possible; check the maximum number of digraphs in the same
-// word in the dictionary for languages with digraphs, like German and French
-#define DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH 5
-
-#define MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION 3
-
// TODO: Remove
#define MAX_POINTER_COUNT 1
#define MAX_POINTER_COUNT_G 2
diff --git a/native/jni/src/dic_traverse_wrapper.cpp b/native/jni/src/dic_traverse_wrapper.cpp
deleted file mode 100644
index ec8c62d..0000000
--- a/native/jni/src/dic_traverse_wrapper.cpp
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright (C) 2012, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#define LOG_TAG "LatinIME: jni: Session"
-
-#include "dic_traverse_wrapper.h"
-
-namespace latinime {
-void *(*DicTraverseWrapper::sDicTraverseSessionFactoryMethod)(JNIEnv *, jstring) = 0;
-void (*DicTraverseWrapper::sDicTraverseSessionReleaseMethod)(void *) = 0;
-void (*DicTraverseWrapper::sDicTraverseSessionInitMethod)(
- void *, const Dictionary *const, const int *, const int, const SuggestOptions *const) = 0;
-} // namespace latinime
diff --git a/native/jni/src/dic_traverse_wrapper.h b/native/jni/src/dic_traverse_wrapper.h
deleted file mode 100644
index 43b4c9a..0000000
--- a/native/jni/src/dic_traverse_wrapper.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_DIC_TRAVERSE_WRAPPER_H
-#define LATINIME_DIC_TRAVERSE_WRAPPER_H
-
-#include "defines.h"
-#include "jni.h"
-
-namespace latinime {
-class Dictionary;
-class SuggestOptions;
-// TODO: Remove
-class DicTraverseWrapper {
- public:
- static void *getDicTraverseSession(JNIEnv *env, jstring locale) {
- if (sDicTraverseSessionFactoryMethod) {
- return sDicTraverseSessionFactoryMethod(env, locale);
- }
- return 0;
- }
- static void initDicTraverseSession(void *traverseSession, const Dictionary *const dictionary,
- const int *prevWord, const int prevWordLength,
- const SuggestOptions *const suggestOptions) {
- if (sDicTraverseSessionInitMethod) {
- sDicTraverseSessionInitMethod(
- traverseSession, dictionary, prevWord, prevWordLength, suggestOptions);
- }
- }
- static void releaseDicTraverseSession(void *traverseSession) {
- if (sDicTraverseSessionReleaseMethod) {
- sDicTraverseSessionReleaseMethod(traverseSession);
- }
- }
- static void setTraverseSessionFactoryMethod(void *(*factoryMethod)(JNIEnv *, jstring)) {
- sDicTraverseSessionFactoryMethod = factoryMethod;
- }
- static void setTraverseSessionInitMethod(
- void (*initMethod)(void *, const Dictionary *const, const int *, const int,
- const SuggestOptions *const)) {
- sDicTraverseSessionInitMethod = initMethod;
- }
- static void setTraverseSessionReleaseMethod(void (*releaseMethod)(void *)) {
- sDicTraverseSessionReleaseMethod = releaseMethod;
- }
-
- private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(DicTraverseWrapper);
- static void *(*sDicTraverseSessionFactoryMethod)(JNIEnv *, jstring);
- static void (*sDicTraverseSessionInitMethod)(
- void *, const Dictionary *const, const int *, const int, const SuggestOptions *const);
- static void (*sDicTraverseSessionReleaseMethod)(void *);
-};
-} // namespace latinime
-#endif // LATINIME_DIC_TRAVERSE_WRAPPER_H
diff --git a/native/jni/src/correction.cpp b/native/jni/src/obsolete/correction.cpp
similarity index 98%
rename from native/jni/src/correction.cpp
rename to native/jni/src/obsolete/correction.cpp
index e2ad557..e583dfa 100644
--- a/native/jni/src/correction.cpp
+++ b/native/jni/src/obsolete/correction.cpp
@@ -18,13 +18,13 @@
#include <cmath>
-#include "char_utils.h"
-#include "correction.h"
#include "defines.h"
+#include "obsolete/correction.h"
#include "suggest/core/layout/proximity_info_state.h"
#include "suggest/core/layout/touch_position_correction_utils.h"
#include "suggest/policyimpl/utils/edit_distance.h"
#include "suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h"
+#include "utils/char_utils.h"
namespace latinime {
@@ -528,7 +528,7 @@
}
inline static bool isUpperCase(unsigned short c) {
- return isAsciiUpper(toBaseCodePoint(c));
+ return CharUtils::isAsciiUpper(CharUtils::toBaseCodePoint(c));
}
//////////////////////
@@ -918,11 +918,15 @@
// In dictionary.cpp, getSuggestion() method,
// When USE_SUGGEST_INTERFACE_FOR_TYPING is true:
+//
+// // TODO: Revise the following logic thoroughly by referring to the logic
+// // marked as "Otherwise" below.
// SUGGEST_INTERFACE_OUTPUT_SCALE was multiplied to the original suggestion scores to convert
// them to integers.
// score = (int)((original score) * SUGGEST_INTERFACE_OUTPUT_SCALE)
// Undo the scaling here to recover the original score.
// normalizedScore = ((float)score) / SUGGEST_INTERFACE_OUTPUT_SCALE
+//
// Otherwise: suggestion scores are computed using the below formula.
// original score
// := powf(mTypedLetterMultiplier (this is defined 2),
@@ -965,9 +969,11 @@
// so, 0 <= distance / afterLength <= 1
const float weight = 1.0f - static_cast<float>(distance) / static_cast<float>(afterLength);
- if (USE_SUGGEST_INTERFACE_FOR_TYPING) {
+ // TODO: Revise the following logic thoroughly by referring to...
+ if (true /* USE_SUGGEST_INTERFACE_FOR_TYPING */) {
return (static_cast<float>(score) / SUGGEST_INTERFACE_OUTPUT_SCALE) * weight;
}
+ // ...this logic.
const float maxScore = score >= S_INT_MAX ? static_cast<float>(S_INT_MAX)
: static_cast<float>(MAX_INITIAL_SCORE)
* powf(static_cast<float>(TYPED_LETTER_MULTIPLIER),
diff --git a/native/jni/src/correction.h b/native/jni/src/obsolete/correction.h
similarity index 96%
rename from native/jni/src/correction.h
rename to native/jni/src/obsolete/correction.h
index 75b4995..710220d 100644
--- a/native/jni/src/correction.h
+++ b/native/jni/src/obsolete/correction.h
@@ -19,9 +19,10 @@
#include <cstring> // for memset()
-#include "correction_state.h"
#include "defines.h"
+#include "obsolete/correction_state.h"
#include "suggest/core/layout/proximity_info_state.h"
+#include "utils/char_utils.h"
namespace latinime {
@@ -342,13 +343,13 @@
const int *const prevprev =
outputLength >= 2 ? editDistanceTable + (outputLength - 2) * (inputSize + 1) : 0;
current[0] = outputLength;
- const int co = toBaseLowerCase(output[outputLength - 1]);
- const int prevCO = outputLength >= 2 ? toBaseLowerCase(output[outputLength - 2]) : 0;
+ const int co = CharUtils::toBaseLowerCase(output[outputLength - 1]);
+ const int prevCO = outputLength >= 2 ? CharUtils::toBaseLowerCase(output[outputLength - 2]) : 0;
for (int i = 1; i <= inputSize; ++i) {
- const int ci = toBaseLowerCase(input[i - 1]);
+ const int ci = CharUtils::toBaseLowerCase(input[i - 1]);
const int cost = (ci == co) ? 0 : 1;
current[i] = min(current[i - 1] + 1, min(prev[i] + 1, prev[i - 1] + cost));
- if (i >= 2 && prevprev && ci == prevCO && co == toBaseLowerCase(input[i - 2])) {
+ if (i >= 2 && prevprev && ci == prevCO && co == CharUtils::toBaseLowerCase(input[i - 2])) {
current[i] = min(current[i], prevprev[i - 2] + 1);
}
}
diff --git a/native/jni/src/correction_state.h b/native/jni/src/obsolete/correction_state.h
similarity index 100%
rename from native/jni/src/correction_state.h
rename to native/jni/src/obsolete/correction_state.h
diff --git a/native/jni/src/suggest/core/dicnode/dic_node.h b/native/jni/src/suggest/core/dicnode/dic_node.h
index 1510e3d..3f64d07 100644
--- a/native/jni/src/suggest/core/dicnode/dic_node.h
+++ b/native/jni/src/suggest/core/dicnode/dic_node.h
@@ -17,13 +17,13 @@
#ifndef LATINIME_DIC_NODE_H
#define LATINIME_DIC_NODE_H
-#include "char_utils.h"
#include "defines.h"
#include "suggest/core/dicnode/dic_node_state.h"
#include "suggest/core/dicnode/dic_node_profiler.h"
#include "suggest/core/dicnode/dic_node_properties.h"
#include "suggest/core/dicnode/dic_node_release_listener.h"
#include "suggest/core/dictionary/digraph_utils.h"
+#include "utils/char_utils.h"
#if DEBUG_DICT
#define LOGI_SHOW_ADD_COST_PROP \
@@ -221,7 +221,7 @@
bool isFirstCharUppercase() const {
const int c = getOutputWordBuf()[0];
- return isAsciiUpper(c);
+ return CharUtils::isAsciiUpper(c);
}
bool isFirstWord() const {
@@ -375,7 +375,7 @@
// Whether the current codepoint can be an intentional omission, in which case the traversal
// algorithm will always check for a possible omission here.
bool canBeIntentionalOmission() const {
- return isIntentionalOmissionCodePoint(getNodeCodePoint());
+ return CharUtils::isIntentionalOmissionCodePoint(getNodeCodePoint());
}
// Whether the omission is so frequent that it should incur zero cost.
diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp
index 7f0d0ed..3deee1a 100644
--- a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp
+++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp
@@ -20,10 +20,13 @@
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_utils.h"
#include "suggest/core/dicnode/dic_node_vector.h"
+#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
#include "suggest/core/dictionary/multi_bigram_map.h"
+#include "suggest/core/dictionary/probability_utils.h"
#include "suggest/core/layout/proximity_info.h"
#include "suggest/core/layout/proximity_info_state.h"
+#include "utils/char_utils.h"
namespace latinime {
@@ -31,20 +34,23 @@
// Node initialization utils //
///////////////////////////////
-/* static */ void DicNodeUtils::initAsRoot(const int rootPos, const uint8_t *const dicRoot,
- const int prevWordNodePos, DicNode *newRootNode) {
- int curPos = rootPos;
+/* static */ void DicNodeUtils::initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo,
+ const int prevWordNodePos, DicNode *const newRootNode) {
+ int curPos = binaryDictionaryInfo->getRootPosition();
const int pos = curPos;
- const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &curPos);
+ const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(
+ binaryDictionaryInfo->getDictRoot(), &curPos);
const int childrenPos = curPos;
newRootNode->initAsRoot(pos, childrenPos, childrenCount, prevWordNodePos);
}
-/*static */ void DicNodeUtils::initAsRootWithPreviousWord(const int rootPos,
- const uint8_t *const dicRoot, DicNode *prevWordLastNode, DicNode *newRootNode) {
- int curPos = rootPos;
+/*static */ void DicNodeUtils::initAsRootWithPreviousWord(
+ const BinaryDictionaryInfo *const binaryDictionaryInfo,
+ DicNode *const prevWordLastNode, DicNode *const newRootNode) {
+ int curPos = binaryDictionaryInfo->getRootPosition();
const int pos = curPos;
- const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &curPos);
+ const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(
+ binaryDictionaryInfo->getDictRoot(), &curPos);
const int childrenPos = curPos;
newRootNode->initAsRootWithPreviousWord(prevWordLastNode, pos, childrenPos, childrenCount);
}
@@ -62,24 +68,27 @@
DicNodeVector *childDicNodes) {
// Passing multiple chars node. No need to traverse child
const int codePoint = dicNode->getNodeTypedCodePoint();
- const int baseLowerCaseCodePoint = toBaseLowerCase(codePoint);
+ const int baseLowerCaseCodePoint = CharUtils::toBaseLowerCase(codePoint);
const bool isMatch = isMatchedNodeCodePoint(pInfoState, pointIndex, exactOnly, codePoint);
- if (isMatch || isIntentionalOmissionCodePoint(baseLowerCaseCodePoint)) {
+ if (isMatch || CharUtils::isIntentionalOmissionCodePoint(baseLowerCaseCodePoint)) {
childDicNodes->pushPassingChild(dicNode);
}
}
/* static */ int DicNodeUtils::createAndGetLeavingChildNode(DicNode *dicNode, int pos,
- const uint8_t *const dicRoot, const int terminalDepth, const ProximityInfoState *pInfoState,
- const int pointIndex, const bool exactOnly, const std::vector<int> *const codePointsFilter,
- const ProximityInfo *const pInfo, DicNodeVector *childDicNodes) {
+ const BinaryDictionaryInfo *const binaryDictionaryInfo, const int terminalDepth,
+ const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly,
+ const std::vector<int> *const codePointsFilter, const ProximityInfo *const pInfo,
+ DicNodeVector *childDicNodes) {
int nextPos = pos;
- const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(dicRoot, &pos);
+ const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(
+ binaryDictionaryInfo->getDictRoot(), &pos);
const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags));
const bool isTerminal = (0 != (BinaryFormat::FLAG_IS_TERMINAL & flags));
const bool hasChildren = BinaryFormat::hasChildrenInFlags(flags);
- int codePoint = BinaryFormat::getCodePointAndForwardPointer(dicRoot, &pos);
+ int codePoint = BinaryFormat::getCodePointAndForwardPointer(
+ binaryDictionaryInfo->getDictRoot(), &pos);
ASSERT(NOT_A_CODE_POINT != codePoint);
const int nodeCodePoint = codePoint;
// TODO: optimize this
@@ -89,7 +98,8 @@
do {
const int nextCodePoint = hasMultipleChars
- ? BinaryFormat::getCodePointAndForwardPointer(dicRoot, &pos) : NOT_A_CODE_POINT;
+ ? BinaryFormat::getCodePointAndForwardPointer(
+ binaryDictionaryInfo->getDictRoot(), &pos) : NOT_A_CODE_POINT;
const bool isLastChar = (NOT_A_CODE_POINT == nextCodePoint);
if (!isLastChar) {
additionalWordBuf[additionalSubwordLength++] = nextCodePoint;
@@ -97,12 +107,14 @@
codePoint = nextCodePoint;
} while (NOT_A_CODE_POINT != codePoint);
- const int probability =
- isTerminal ? BinaryFormat::readProbabilityWithoutMovingPointer(dicRoot, pos) : -1;
+ const int probability = isTerminal ? BinaryFormat::readProbabilityWithoutMovingPointer(
+ binaryDictionaryInfo->getDictRoot(), pos) : -1;
pos = BinaryFormat::skipProbability(flags, pos);
- int childrenPos = hasChildren ? BinaryFormat::readChildrenPosition(dicRoot, flags, pos) : 0;
+ int childrenPos = hasChildren ? BinaryFormat::readChildrenPosition(
+ binaryDictionaryInfo->getDictRoot(), flags, pos) : 0;
const int attributesPos = BinaryFormat::skipChildrenPosition(flags, pos);
- const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(dicRoot, flags, pos);
+ const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(
+ binaryDictionaryInfo->getDictRoot(), flags, pos);
if (isDicNodeFilteredOut(nodeCodePoint, pInfo, codePointsFilter)) {
return siblingPos;
@@ -110,8 +122,8 @@
if (!isMatchedNodeCodePoint(pInfoState, pointIndex, exactOnly, nodeCodePoint)) {
return siblingPos;
}
- const int childrenCount = hasChildren
- ? BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &childrenPos) : 0;
+ const int childrenCount = hasChildren ? BinaryFormat::getGroupCountAndForwardPointer(
+ binaryDictionaryInfo->getDictRoot(), &childrenPos) : 0;
childDicNodes->pushLeavingChild(dicNode, nextPos, flags, childrenPos, attributesPos, siblingPos,
nodeCodePoint, childrenCount, probability, -1 /* bigramProbability */, isTerminal,
hasMultipleChars, hasChildren, additionalSubwordLength, additionalWordBuf);
@@ -125,13 +137,13 @@
return false;
}
if (pInfo && (pInfo->getKeyIndexOf(nodeCodePoint) == NOT_AN_INDEX
- || isIntentionalOmissionCodePoint(nodeCodePoint))) {
+ || CharUtils::isIntentionalOmissionCodePoint(nodeCodePoint))) {
// If normalized nodeCodePoint is not on the keyboard or skippable, this child is never
// filtered.
return false;
}
- const int lowerCodePoint = toLowerCase(nodeCodePoint);
- const int baseLowerCodePoint = toBaseCodePoint(lowerCodePoint);
+ const int lowerCodePoint = CharUtils::toLowerCase(nodeCodePoint);
+ const int baseLowerCodePoint = CharUtils::toBaseCodePoint(lowerCodePoint);
// TODO: Avoid linear search
for (int i = 0; i < filterSize; ++i) {
// Checking if a normalized code point is in filter characters when pInfo is not
@@ -147,16 +159,18 @@
}
/* static */ void DicNodeUtils::createAndGetAllLeavingChildNodes(DicNode *dicNode,
- const uint8_t *const dicRoot, const ProximityInfoState *pInfoState, const int pointIndex,
- const bool exactOnly, const std::vector<int> *const codePointsFilter,
- const ProximityInfo *const pInfo, DicNodeVector *childDicNodes) {
+ const BinaryDictionaryInfo *const binaryDictionaryInfo,
+ const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly,
+ const std::vector<int> *const codePointsFilter, const ProximityInfo *const pInfo,
+ DicNodeVector *childDicNodes) {
const int terminalDepth = dicNode->getLeavingDepth();
const int childCount = dicNode->getChildrenCount();
int nextPos = dicNode->getChildrenPos();
for (int i = 0; i < childCount; i++) {
const int filterSize = codePointsFilter ? codePointsFilter->size() : 0;
- nextPos = createAndGetLeavingChildNode(dicNode, nextPos, dicRoot, terminalDepth, pInfoState,
- pointIndex, exactOnly, codePointsFilter, pInfo, childDicNodes);
+ nextPos = createAndGetLeavingChildNode(dicNode, nextPos, binaryDictionaryInfo,
+ terminalDepth, pInfoState, pointIndex, exactOnly, codePointsFilter, pInfo,
+ childDicNodes);
if (!pInfo && filterSize > 0 && childDicNodes->exceeds(filterSize)) {
// All code points have been found.
break;
@@ -164,14 +178,15 @@
}
}
-/* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot,
- DicNodeVector *childDicNodes) {
- getProximityChildDicNodes(dicNode, dicRoot, 0, 0, false, childDicNodes);
+/* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode,
+ const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes) {
+ getProximityChildDicNodes(dicNode, binaryDictionaryInfo, 0, 0, false, childDicNodes);
}
/* static */ void DicNodeUtils::getProximityChildDicNodes(DicNode *dicNode,
- const uint8_t *const dicRoot, const ProximityInfoState *pInfoState, const int pointIndex,
- bool exactOnly, DicNodeVector *childDicNodes) {
+ const BinaryDictionaryInfo *const binaryDictionaryInfo,
+ const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly,
+ DicNodeVector *childDicNodes) {
if (dicNode->isTotalInputSizeExceedingLimit()) {
return;
}
@@ -179,9 +194,9 @@
DicNodeUtils::createAndGetPassingChildNode(dicNode, pInfoState, pointIndex, exactOnly,
childDicNodes);
} else {
- DicNodeUtils::createAndGetAllLeavingChildNodes(dicNode, dicRoot, pInfoState, pointIndex,
- exactOnly, 0 /* codePointsFilter */, 0 /* pInfo */,
- childDicNodes);
+ DicNodeUtils::createAndGetAllLeavingChildNodes(
+ dicNode, binaryDictionaryInfo, pInfoState, pointIndex, exactOnly,
+ 0 /* codePointsFilter */, 0 /* pInfo */, childDicNodes);
}
}
@@ -191,32 +206,35 @@
/**
* Computes the combined bigram / unigram cost for the given dicNode.
*/
-/* static */ float DicNodeUtils::getBigramNodeImprobability(const uint8_t *const dicRoot,
+/* static */ float DicNodeUtils::getBigramNodeImprobability(
+ const BinaryDictionaryInfo *const binaryDictionaryInfo,
const DicNode *const node, MultiBigramMap *multiBigramMap) {
if (node->isImpossibleBigramWord()) {
return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
}
- const int probability = getBigramNodeProbability(dicRoot, node, multiBigramMap);
+ const int probability = getBigramNodeProbability(binaryDictionaryInfo, node, multiBigramMap);
// TODO: This equation to calculate the improbability looks unreasonable. Investigate this.
const float cost = static_cast<float>(MAX_PROBABILITY - probability)
/ static_cast<float>(MAX_PROBABILITY);
return cost;
}
-/* static */ int DicNodeUtils::getBigramNodeProbability(const uint8_t *const dicRoot,
+/* static */ int DicNodeUtils::getBigramNodeProbability(
+ const BinaryDictionaryInfo *const binaryDictionaryInfo,
const DicNode *const node, MultiBigramMap *multiBigramMap) {
const int unigramProbability = node->getProbability();
const int wordPos = node->getPos();
const int prevWordPos = node->getPrevWordPos();
if (NOT_VALID_WORD == wordPos || NOT_VALID_WORD == prevWordPos) {
// Note: Normally wordPos comes from the dictionary and should never equal NOT_VALID_WORD.
- return backoff(unigramProbability);
+ return ProbabilityUtils::backoff(unigramProbability);
}
if (multiBigramMap) {
return multiBigramMap->getBigramProbability(
- dicRoot, prevWordPos, wordPos, unigramProbability);
+ binaryDictionaryInfo, prevWordPos, wordPos, unigramProbability);
}
- return BinaryFormat::getBigramProbability(dicRoot, prevWordPos, wordPos, unigramProbability);
+ return BinaryFormat::getBigramProbability(
+ binaryDictionaryInfo->getDictRoot(), prevWordPos, wordPos, unigramProbability);
}
///////////////////////////////////////
diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.h b/native/jni/src/suggest/core/dicnode/dic_node_utils.h
index 5bc542d..e198d61 100644
--- a/native/jni/src/suggest/core/dicnode/dic_node_utils.h
+++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.h
@@ -24,6 +24,7 @@
namespace latinime {
+class BinaryDictionaryInfo;
class DicNode;
class DicNodeVector;
class ProximityInfo;
@@ -34,19 +35,20 @@
public:
static int appendTwoWords(const int *src0, const int16_t length0, const int *src1,
const int16_t length1, int *dest);
- static void initAsRoot(const int rootPos, const uint8_t *const dicRoot,
+ static void initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo,
const int prevWordNodePos, DicNode *newRootNode);
- static void initAsRootWithPreviousWord(const int rootPos, const uint8_t *const dicRoot,
+ static void initAsRootWithPreviousWord(const BinaryDictionaryInfo *const binaryDictionaryInfo,
DicNode *prevWordLastNode, DicNode *newRootNode);
static void initByCopy(DicNode *srcNode, DicNode *destNode);
- static void getAllChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot,
- DicNodeVector *childDicNodes);
- static float getBigramNodeImprobability(const uint8_t *const dicRoot,
+ static void getAllChildDicNodes(DicNode *dicNode,
+ const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes);
+ static float getBigramNodeImprobability(const BinaryDictionaryInfo *const binaryDictionaryInfo,
const DicNode *const node, MultiBigramMap *const multiBigramMap);
static bool isDicNodeFilteredOut(const int nodeCodePoint, const ProximityInfo *const pInfo,
const std::vector<int> *const codePointsFilter);
// TODO: Move to private
- static void getProximityChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot,
+ static void getProximityChildDicNodes(DicNode *dicNode,
+ const BinaryDictionaryInfo *const binaryDictionaryInfo,
const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly,
DicNodeVector *childDicNodes);
@@ -60,16 +62,18 @@
// Max number of bigrams to look up
static const int MAX_BIGRAMS_CONSIDERED_PER_CONTEXT = 500;
- static int getBigramNodeProbability(const uint8_t *const dicRoot, const DicNode *const node,
- MultiBigramMap *multiBigramMap);
+ static int getBigramNodeProbability(const BinaryDictionaryInfo *const binaryDictionaryInfo,
+ const DicNode *const node, MultiBigramMap *multiBigramMap);
static void createAndGetPassingChildNode(DicNode *dicNode, const ProximityInfoState *pInfoState,
const int pointIndex, const bool exactOnly, DicNodeVector *childDicNodes);
- static void createAndGetAllLeavingChildNodes(DicNode *dicNode, const uint8_t *const dicRoot,
+ static void createAndGetAllLeavingChildNodes(DicNode *dicNode,
+ const BinaryDictionaryInfo *const binaryDictionaryInfo,
const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly,
const std::vector<int> *const codePointsFilter,
const ProximityInfo *const pInfo, DicNodeVector *childDicNodes);
- static int createAndGetLeavingChildNode(DicNode *dicNode, int pos, const uint8_t *const dicRoot,
- const int terminalDepth, const ProximityInfoState *pInfoState, const int pointIndex,
+ static int createAndGetLeavingChildNode(DicNode *dicNode, int pos,
+ const BinaryDictionaryInfo *const binaryDictionaryInfo, const int terminalDepth,
+ const ProximityInfoState *pInfoState, const int pointIndex,
const bool exactOnly, const std::vector<int> *const codePointsFilter,
const ProximityInfo *const pInfo, DicNodeVector *childDicNodes);
diff --git a/native/jni/src/bigram_dictionary.cpp b/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp
similarity index 91%
rename from native/jni/src/bigram_dictionary.cpp
rename to native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp
index c592542..59d1b19 100644
--- a/native/jni/src/bigram_dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp
@@ -20,15 +20,18 @@
#include "bigram_dictionary.h"
-#include "char_utils.h"
#include "defines.h"
+#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
#include "suggest/core/dictionary/bloom_filter.h"
#include "suggest/core/dictionary/dictionary.h"
+#include "suggest/core/dictionary/probability_utils.h"
+#include "utils/char_utils.h"
namespace latinime {
-BigramDictionary::BigramDictionary(const uint8_t *const streamStart) : DICT_ROOT(streamStart) {
+BigramDictionary::BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo)
+ : mBinaryDictionaryInfo(binaryDictionaryInfo) {
if (DEBUG_DICT) {
AKLOGI("BigramDictionary - constructor");
}
@@ -52,7 +55,7 @@
int insertAt = 0;
while (insertAt < MAX_RESULTS) {
if (probability > bigramProbability[insertAt] || (bigramProbability[insertAt] == probability
- && length < getCodePointCount(MAX_WORD_LENGTH,
+ && length < CharUtils::getCodePointCount(MAX_WORD_LENGTH,
bigramCodePoints + insertAt * MAX_WORD_LENGTH))) {
break;
}
@@ -103,7 +106,7 @@
// TODO: remove unused arguments, and refrain from storing stuff in members of this class
// TODO: have "in" arguments before "out" ones, and make out args explicit in the name
- const uint8_t *const root = DICT_ROOT;
+ const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot();
int pos = getBigramListPositionForWord(prevWord, prevWordLength,
false /* forceLowerCaseSearch */);
// getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams
@@ -134,7 +137,7 @@
// resulting probability is 8 - although in the practice it's never bigger than 3 or 4
// in very bad cases. This means that sometimes, we'll see some bigrams interverted
// here, but it can't get too bad.
- const int probability = BinaryFormat::computeProbabilityForBigram(
+ const int probability = ProbabilityUtils::computeProbabilityForBigram(
unigramProbability, bigramProbabilityTemp);
addWordBigram(bigramBuffer, length, probability, bigramProbability, bigramCodePoints,
outputTypes);
@@ -149,7 +152,7 @@
int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const int prevWordLength,
const bool forceLowerCaseSearch) const {
if (0 >= prevWordLength) return 0;
- const uint8_t *const root = DICT_ROOT;
+ const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot();
int pos = BinaryFormat::getTerminalPosition(root, prevWord, prevWordLength,
forceLowerCaseSearch);
@@ -170,7 +173,7 @@
void BigramDictionary::fillBigramAddressToProbabilityMapAndFilter(const int *prevWord,
const int prevWordLength, std::map<int, int> *map, uint8_t *filter) const {
memset(filter, 0, BIGRAM_FILTER_BYTE_SIZE);
- const uint8_t *const root = DICT_ROOT;
+ const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot();
int pos = getBigramListPositionForWord(prevWord, prevWordLength,
false /* forceLowerCaseSearch */);
if (0 == pos) {
@@ -196,9 +199,9 @@
// what user typed.
int maxAlt = MAX_ALTERNATIVES;
- const int firstBaseLowerCodePoint = toBaseLowerCase(*word);
+ const int firstBaseLowerCodePoint = CharUtils::toBaseLowerCase(*word);
while (maxAlt > 0) {
- if (toBaseLowerCase(*inputCodePoints) == firstBaseLowerCodePoint) {
+ if (CharUtils::toBaseLowerCase(*inputCodePoints) == firstBaseLowerCodePoint) {
return true;
}
inputCodePoints++;
@@ -209,7 +212,7 @@
bool BigramDictionary::isValidBigram(const int *word1, int length1, const int *word2,
int length2) const {
- const uint8_t *const root = DICT_ROOT;
+ const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot();
int pos = getBigramListPositionForWord(word1, length1, false /* forceLowerCaseSearch */);
// getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams
if (0 == pos) return false;
diff --git a/native/jni/src/bigram_dictionary.h b/native/jni/src/suggest/core/dictionary/bigram_dictionary.h
similarity index 91%
rename from native/jni/src/bigram_dictionary.h
rename to native/jni/src/suggest/core/dictionary/bigram_dictionary.h
index b86e564..8b7a253 100644
--- a/native/jni/src/bigram_dictionary.h
+++ b/native/jni/src/suggest/core/dictionary/bigram_dictionary.h
@@ -24,9 +24,12 @@
namespace latinime {
+class BinaryDictionaryInfo;
+
class BigramDictionary {
public:
- BigramDictionary(const uint8_t *const streamStart);
+ BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo);
+
int getBigrams(const int *word, int length, int *inputCodePoints, int inputSize, int *outWords,
int *frequencies, int *outputTypes) const;
void fillBigramAddressToProbabilityMapAndFilter(const int *prevWord, const int prevWordLength,
@@ -35,13 +38,14 @@
~BigramDictionary();
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BigramDictionary);
+
void addWordBigram(int *word, int length, int probability, int *bigramProbability,
int *bigramCodePoints, int *outputTypes) const;
bool checkFirstCharacter(int *word, int *inputCodePoints) const;
int getBigramListPositionForWord(const int *prevWord, const int prevWordLength,
const bool forceLowerCaseSearch) const;
- const uint8_t *const DICT_ROOT;
+ const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
// TODO: Re-implement proximity correction for bigram correction
static const int MAX_ALTERNATIVES = 1;
};
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.cpp
new file mode 100644
index 0000000..50e0211
--- /dev/null
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.cpp
@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/core/dictionary/binary_dictionary_format.h"
+
+namespace latinime {
+
+/**
+ * Dictionary size
+ */
+// Any file smaller than this is not a dictionary.
+const int BinaryDictionaryFormat::DICTIONARY_MINIMUM_SIZE = 4;
+
+/**
+ * Format versions
+ */
+// Originally, format version 1 had a 16-bit magic number, then the version number `01'
+// then options that must be 0. Hence the first 32-bits of the format are always as follow
+// and it's okay to consider them a magic number as a whole.
+const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100;
+const int BinaryDictionaryFormat::FORMAT_VERSION_1_HEADER_SIZE = 5;
+
+// The versions of Latin IME that only handle format version 1 only test for the magic
+// number, so we had to change it so that version 2 files would be rejected by older
+// implementations. On this occasion, we made the magic number 32 bits long.
+const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE;
+// Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12
+const int BinaryDictionaryFormat::FORMAT_VERSION_2_MINIMUM_SIZE = 12;
+const int BinaryDictionaryFormat::VERSION_2_MAGIC_NUMBER_SIZE = 4;
+const int BinaryDictionaryFormat::VERSION_2_DICTIONARY_VERSION_SIZE = 2;
+const int BinaryDictionaryFormat::VERSION_2_DICTIONARY_FLAG_SIZE = 2;
+
+/* static */ BinaryDictionaryFormat::FORMAT_VERSION BinaryDictionaryFormat::detectFormatVersion(
+ const uint8_t *const dict, const int dictSize) {
+ // The magic number is stored big-endian.
+ // If the dictionary is less than 4 bytes, we can't even read the magic number, so we don't
+ // understand this format.
+ if (dictSize < DICTIONARY_MINIMUM_SIZE) {
+ return UNKNOWN_VERSION;
+ }
+ const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0);
+ switch (magicNumber) {
+ case FORMAT_VERSION_1_MAGIC_NUMBER:
+ // Format 1 header is exactly 5 bytes long and looks like:
+ // Magic number (2 bytes) 0x78 0xB1
+ // Version number (1 byte) 0x01
+ // Options (2 bytes) must be 0x00 0x00
+ return VERSION_1;
+ case FORMAT_VERSION_2_MAGIC_NUMBER:
+ // Version 2 dictionaries are at least 12 bytes long.
+ // If this dictionary has the version 2 magic number but is less than 12 bytes long,
+ // then it's an unknown format and we need to avoid confidently reading the next bytes.
+ if (dictSize < FORMAT_VERSION_2_MINIMUM_SIZE) {
+ return UNKNOWN_VERSION;
+ }
+ // Format 2 header is as follows:
+ // Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE
+ // Version number (2 bytes) 0x00 0x02
+ // Options (2 bytes)
+ // Header size (4 bytes) : integer, big endian
+ if (ByteArrayUtils::readUint16(dict, 4) == 2) {
+ return VERSION_2;
+ } else {
+ return UNKNOWN_VERSION;
+ }
+ default:
+ return UNKNOWN_VERSION;
+ }
+}
+
+} // namespace latinime
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.h
new file mode 100644
index 0000000..3aa1662
--- /dev/null
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_BINARY_DICTIONARY_FORMAT_H
+#define LATINIME_BINARY_DICTIONARY_FORMAT_H
+
+#include <stdint.h>
+
+#include "defines.h"
+#include "suggest/core/dictionary/byte_array_utils.h"
+
+namespace latinime {
+
+/**
+ * Methods to handle binary dictionary format version.
+ *
+ * Currently, we have a file with a similar name, binary_format.h. binary_format.h contains binary
+ * reading methods and utility methods for various purposes.
+ * On the other hand, this file deals with only about dictionary format version.
+ */
+class BinaryDictionaryFormat {
+ public:
+ // TODO: Remove obsolete version logic
+ enum FORMAT_VERSION {
+ VERSION_1,
+ VERSION_2,
+ UNKNOWN_VERSION
+ };
+
+ static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize);
+
+ static AK_FORCE_INLINE int getHeaderSize(
+ const uint8_t *const dict, const FORMAT_VERSION format) {
+ switch (format) {
+ case VERSION_1:
+ return FORMAT_VERSION_1_HEADER_SIZE;
+ case VERSION_2:
+ // See the format of the header in the comment in detectFormat() above
+ return ByteArrayUtils::readUint32(dict, 8);
+ default:
+ return S_INT_MAX;
+ }
+ }
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormat);
+
+ static const int DICTIONARY_MINIMUM_SIZE;
+ static const uint32_t FORMAT_VERSION_1_MAGIC_NUMBER;
+ static const int FORMAT_VERSION_1_HEADER_SIZE;
+ static const uint32_t FORMAT_VERSION_2_MAGIC_NUMBER;
+ static const int FORMAT_VERSION_2_MINIMUM_SIZE;
+ static const int VERSION_2_MAGIC_NUMBER_SIZE;
+ static const int VERSION_2_DICTIONARY_VERSION_SIZE ;
+ static const int VERSION_2_DICTIONARY_FLAG_SIZE;
+};
+} // namespace latinime
+#endif /* LATINIME_BINARY_DICTIONARY_FORMAT_H */
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h
new file mode 100644
index 0000000..8508c67
--- /dev/null
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_BINARY_DICTIONARY_INFO_H
+#define LATINIME_BINARY_DICTIONARY_INFO_H
+
+#include <stdint.h>
+
+#include "defines.h"
+#include "suggest/core/dictionary/binary_dictionary_format.h"
+
+namespace latinime {
+
+class BinaryDictionaryInfo {
+ public:
+ BinaryDictionaryInfo(const uint8_t *const dictBuf, const int dictSize)
+ : mDictBuf(dictBuf),
+ mFormat(BinaryDictionaryFormat::detectFormatVersion(mDictBuf, dictSize)),
+ mDictRoot(mDictBuf + BinaryDictionaryFormat::getHeaderSize(mDictBuf, mFormat)) {}
+
+ AK_FORCE_INLINE const uint8_t *getDictBuf() const {
+ return mDictBuf;
+ }
+
+ AK_FORCE_INLINE const uint8_t *getDictRoot() const {
+ return mDictRoot;
+ }
+
+ AK_FORCE_INLINE BinaryDictionaryFormat::FORMAT_VERSION getFormat() const {
+ return mFormat;
+ }
+
+ AK_FORCE_INLINE int getRootPosition() const {
+ return 0;
+ }
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(BinaryDictionaryInfo);
+
+ const uint8_t *const mDictBuf;
+ const BinaryDictionaryFormat::FORMAT_VERSION mFormat;
+ const uint8_t *const mDictRoot;
+};
+}
+#endif /* LATINIME_BINARY_DICTIONARY_INFO_H */
diff --git a/native/jni/src/suggest/core/dictionary/binary_format.h b/native/jni/src/suggest/core/dictionary/binary_format.h
index 65c2e91..c82065f 100644
--- a/native/jni/src/suggest/core/dictionary/binary_format.h
+++ b/native/jni/src/suggest/core/dictionary/binary_format.h
@@ -18,12 +18,12 @@
#define LATINIME_BINARY_FORMAT_H
#include <cstdlib>
-#include <map>
#include <stdint.h>
-#include "char_utils.h"
-#include "hash_map_compat.h"
#include "suggest/core/dictionary/bloom_filter.h"
+#include "suggest/core/dictionary/probability_utils.h"
+#include "utils/char_utils.h"
+#include "utils/hash_map_compat.h"
namespace latinime {
@@ -91,10 +91,6 @@
const int length, const bool forceLowerCaseSearch);
static int getWordAtAddress(const uint8_t *const root, const int address, const int maxDepth,
int *outWord, int *outUnigramProbability);
- static int computeProbabilityForBigram(
- const int unigramProbability, const int bigramProbability);
- static int getProbability(const int position, const std::map<int, int> *bigramMap,
- const uint8_t *bigramFilter, const int unigramProbability);
static int getBigramProbabilityFromHashMap(const int position,
const hash_map_compat<int, int> *bigramMap, const int unigramProbability);
static float getMultiWordCostMultiplier(const uint8_t *const dict, const int dictSize);
@@ -271,8 +267,7 @@
inline float BinaryFormat::getMultiWordCostMultiplier(const uint8_t *const dict,
const int dictSize) {
- const int headerValue = readHeaderValueInt(dict, dictSize,
- "MULTIPLE_WORDS_DEMOTION_RATE");
+ const int headerValue = readHeaderValueInt(dict, dictSize, "MULTIPLE_WORDS_DEMOTION_RATE");
if (headerValue == S_INT_MIN) {
return 1.0f;
}
@@ -473,7 +468,8 @@
// there was no match (or we would have found it).
if (wordPos >= length) return NOT_VALID_WORD;
int charGroupCount = BinaryFormat::getGroupCountAndForwardPointer(root, &pos);
- const int wChar = forceLowerCaseSearch ? toLowerCase(inWord[wordPos]) : inWord[wordPos];
+ const int wChar = forceLowerCaseSearch
+ ? CharUtils::toLowerCase(inWord[wordPos]) : inWord[wordPos];
while (true) {
// If there are no more character groups in this node, it means we could not
// find a matching character for this depth, therefore there is no match.
@@ -677,51 +673,18 @@
return 0;
}
-static inline int backoff(const int unigramProbability) {
- return unigramProbability;
- // For some reason, applying the backoff weight gives bad results in tests. To apply the
- // backoff weight, we divide the probability by 2, which in our storing format means
- // decreasing the score by 8.
- // TODO: figure out what's wrong with this.
- // return unigramProbability > 8 ? unigramProbability - 8 : (0 == unigramProbability ? 0 : 8);
-}
-
-inline int BinaryFormat::computeProbabilityForBigram(
- const int unigramProbability, const int bigramProbability) {
- // We divide the range [unigramProbability..255] in 16.5 steps - in other words, we want the
- // unigram probability to be the median value of the 17th step from the top. A value of
- // 0 for the bigram probability represents the middle of the 16th step from the top,
- // while a value of 15 represents the middle of the top step.
- // See makedict.BinaryDictInputOutput for details.
- const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability)
- / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY);
- return unigramProbability
- + static_cast<int>(static_cast<float>(bigramProbability + 1) * stepSize);
-}
-
-// This returns a probability in log space.
-inline int BinaryFormat::getProbability(const int position, const std::map<int, int> *bigramMap,
- const uint8_t *bigramFilter, const int unigramProbability) {
- if (!bigramMap || !bigramFilter) return backoff(unigramProbability);
- if (!isInFilter(bigramFilter, position)) return backoff(unigramProbability);
- const std::map<int, int>::const_iterator bigramProbabilityIt = bigramMap->find(position);
- if (bigramProbabilityIt != bigramMap->end()) {
- const int bigramProbability = bigramProbabilityIt->second;
- return computeProbabilityForBigram(unigramProbability, bigramProbability);
- }
- return backoff(unigramProbability);
-}
-
// This returns a probability in log space.
inline int BinaryFormat::getBigramProbabilityFromHashMap(const int position,
const hash_map_compat<int, int> *bigramMap, const int unigramProbability) {
- if (!bigramMap) return backoff(unigramProbability);
+ if (!bigramMap) {
+ return ProbabilityUtils::backoff(unigramProbability);
+ }
const hash_map_compat<int, int>::const_iterator bigramProbabilityIt = bigramMap->find(position);
if (bigramProbabilityIt != bigramMap->end()) {
const int bigramProbability = bigramProbabilityIt->second;
- return computeProbabilityForBigram(unigramProbability, bigramProbability);
+ return ProbabilityUtils::computeProbabilityForBigram(unigramProbability, bigramProbability);
}
- return backoff(unigramProbability);
+ return ProbabilityUtils::backoff(unigramProbability);
}
AK_FORCE_INLINE void BinaryFormat::fillBigramProbabilityToHashMap(
@@ -742,7 +705,9 @@
AK_FORCE_INLINE int BinaryFormat::getBigramProbability(const uint8_t *const root, int position,
const int nextPosition, const int unigramProbability) {
position = getBigramListPositionForWordPosition(root, position);
- if (0 == position) return backoff(unigramProbability);
+ if (0 == position) {
+ return ProbabilityUtils::backoff(unigramProbability);
+ }
uint8_t bigramFlags;
do {
@@ -751,10 +716,11 @@
root, bigramFlags, &position);
if (bigramPos == nextPosition) {
const int bigramProbability = MASK_ATTRIBUTE_PROBABILITY & bigramFlags;
- return computeProbabilityForBigram(unigramProbability, bigramProbability);
+ return ProbabilityUtils::computeProbabilityForBigram(
+ unigramProbability, bigramProbability);
}
} while (FLAG_ATTRIBUTE_HAS_NEXT & bigramFlags);
- return backoff(unigramProbability);
+ return ProbabilityUtils::backoff(unigramProbability);
}
// Returns a pointer to the start of the bigram list.
diff --git a/native/jni/src/suggest/core/dictionary/byte_array_utils.cpp b/native/jni/src/suggest/core/dictionary/byte_array_utils.cpp
new file mode 100644
index 0000000..68b1d5d
--- /dev/null
+++ b/native/jni/src/suggest/core/dictionary/byte_array_utils.cpp
@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/core/dictionary/byte_array_utils.h"
+
+namespace latinime {
+
+const uint8_t ByteArrayUtils::MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
+const uint8_t ByteArrayUtils::CHARACTER_ARRAY_TERMINATOR = 0x1F;
+
+} // namespace latinime
diff --git a/native/jni/src/suggest/core/dictionary/byte_array_utils.h b/native/jni/src/suggest/core/dictionary/byte_array_utils.h
new file mode 100644
index 0000000..832b747
--- /dev/null
+++ b/native/jni/src/suggest/core/dictionary/byte_array_utils.h
@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_BYTE_ARRAY_UTILS_H
+#define LATINIME_BYTE_ARRAY_UTILS_H
+
+#include <stdint.h>
+
+#include "defines.h"
+
+namespace latinime {
+
+/**
+ * Utility methods for reading byte arrays.
+ */
+class ByteArrayUtils {
+ public:
+ /**
+ * Integer
+ *
+ * Each method read a corresponding size integer in a big endian manner.
+ */
+ static AK_FORCE_INLINE uint32_t readUint32(const uint8_t *const buffer, const int pos) {
+ return (buffer[pos] << 24) ^ (buffer[pos + 1] << 16)
+ ^ (buffer[pos + 2] << 8) ^ buffer[pos + 3];
+ }
+
+ static AK_FORCE_INLINE uint32_t readUint24(const uint8_t *const buffer, const int pos) {
+ return (buffer[pos] << 16) ^ (buffer[pos + 1] << 8) ^ buffer[pos + 2];
+ }
+
+ static AK_FORCE_INLINE uint16_t readUint16(const uint8_t *const buffer, const int pos) {
+ return (buffer[pos] << 8) ^ buffer[pos + 1];
+ }
+
+ static AK_FORCE_INLINE uint8_t readUint8(const uint8_t *const buffer, const int pos) {
+ return buffer[pos];
+ }
+
+ static AK_FORCE_INLINE uint32_t readUint32andAdvancePosition(
+ const uint8_t *const buffer, int *const pos) {
+ const uint32_t value = readUint32(buffer, *pos);
+ *pos += 4;
+ return value;
+ }
+
+ static AK_FORCE_INLINE uint32_t readUint24andAdvancePosition(
+ const uint8_t *const buffer, int *const pos) {
+ const uint32_t value = readUint24(buffer, *pos);
+ *pos += 3;
+ return value;
+ }
+
+ static AK_FORCE_INLINE uint16_t readUint16andAdvancePosition(
+ const uint8_t *const buffer, int *const pos) {
+ const uint16_t value = readUint16(buffer, *pos);
+ *pos += 2;
+ return value;
+ }
+
+ static AK_FORCE_INLINE uint8_t readUint8andAdvancePosition(
+ const uint8_t *const buffer, int *const pos) {
+ return buffer[(*pos)++];
+ }
+
+ /**
+ * Code Point
+ *
+ * 1 byte = bbbbbbbb match
+ * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
+ * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
+ * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
+ * 00011111 would be outside unicode.
+ * else: iso-latin-1 code
+ * This allows for the whole unicode range to be encoded, including chars outside of
+ * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
+ * characters which should never happen anyway (and still work, but take 3 bytes).
+ */
+ static AK_FORCE_INLINE int readCodePoint(const uint8_t *const buffer, const int pos) {
+ int p = pos;
+ return readCodePointAndAdvancePosition(buffer, &p);
+ }
+
+ static AK_FORCE_INLINE int readCodePointAndAdvancePosition(
+ const uint8_t *const buffer, int *const pos) {
+ const uint8_t firstByte = readUint8(buffer, *pos);
+ if (firstByte < MINIMAL_ONE_BYTE_CHARACTER_VALUE) {
+ if (firstByte == CHARACTER_ARRAY_TERMINATOR) {
+ *pos += 1;
+ return NOT_A_CODE_POINT;
+ } else {
+ return readUint24andAdvancePosition(buffer, pos);
+ }
+ } else {
+ *pos += 1;
+ return firstByte;
+ }
+ }
+
+ /**
+ * String (array of code points)
+ *
+ * Reads code points until the terminator is found.
+ */
+ // Returns the length of the string.
+ static int readStringAndAdvancePosition(const uint8_t *const buffer, int *const pos,
+ int *const outBuffer, const int maxLength) {
+ int length = 0;
+ int codePoint = readCodePointAndAdvancePosition(buffer, pos);
+ while (NOT_A_CODE_POINT != codePoint && length < maxLength) {
+ outBuffer[length++] = codePoint;
+ codePoint = readCodePointAndAdvancePosition(buffer, pos);
+ }
+ return length;
+ }
+
+ // Advances the position and returns the length of the string.
+ static int advancePositionToBehindString(
+ const uint8_t *const buffer, int *const pos, const int maxLength) {
+ int length = 0;
+ int codePoint = readCodePointAndAdvancePosition(buffer, pos);
+ while (NOT_A_CODE_POINT != codePoint && length < maxLength) {
+ codePoint = readCodePointAndAdvancePosition(buffer, pos);
+ }
+ return length;
+ }
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(ByteArrayUtils);
+
+ static const uint8_t MINIMAL_ONE_BYTE_CHARACTER_VALUE;
+ static const uint8_t CHARACTER_ARRAY_TERMINATOR;
+};
+} // namespace latinime
+#endif /* LATINIME_BYTE_ARRAY_UTILS_H */
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp
index 1939c74..2d4ad5d 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp
@@ -21,45 +21,41 @@
#include <map> // TODO: remove
#include <stdint.h>
-#include "bigram_dictionary.h"
#include "defines.h"
-#include "dic_traverse_wrapper.h"
-#include "suggest_options.h"
-#include "suggest/core/suggest.h"
+#include "suggest/core/dictionary/bigram_dictionary.h"
#include "suggest/core/dictionary/binary_format.h"
+#include "suggest/core/session/dic_traverse_session.h"
+#include "suggest/core/suggest.h"
+#include "suggest/core/suggest_options.h"
#include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h"
#include "suggest/policyimpl/typing/typing_suggest_policy_factory.h"
-#include "unigram_dictionary.h"
namespace latinime {
Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust)
- : mDict(static_cast<unsigned char *>(dict)),
- mOffsetDict((static_cast<unsigned char *>(dict))
- + BinaryFormat::getHeaderSize(mDict, dictSize)),
- mDictSize(dictSize), mMmapFd(mmapFd), mDictBufAdjust(dictBufAdjust),
- mUnigramDictionary(new UnigramDictionary(mOffsetDict,
- BinaryFormat::getFlags(mDict, dictSize))),
- mBigramDictionary(new BigramDictionary(mOffsetDict)),
+ : mBinaryDicitonaryInfo(static_cast<const uint8_t *>(dict), dictSize),
+ mDictSize(dictSize),
+ mDictFlags(BinaryFormat::getFlags(mBinaryDicitonaryInfo.getDictBuf(), dictSize)),
+ mMmapFd(mmapFd), mDictBufAdjust(dictBufAdjust),
+ mBigramDictionary(new BigramDictionary(&mBinaryDicitonaryInfo)),
mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())),
mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) {
}
Dictionary::~Dictionary() {
- delete mUnigramDictionary;
delete mBigramDictionary;
delete mGestureSuggest;
delete mTypingSuggest;
}
-int Dictionary::getSuggestions(ProximityInfo *proximityInfo, void *traverseSession,
+int Dictionary::getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,
int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
int inputSize, int *prevWordCodePoints, int prevWordLength, int commitPoint,
const SuggestOptions *const suggestOptions, int *outWords, int *frequencies,
int *spaceIndices, int *outputTypes) const {
int result = 0;
if (suggestOptions->isGesture()) {
- DicTraverseWrapper::initDicTraverseSession(
+ DicTraverseSession::initSessionInstance(
traverseSession, this, prevWordCodePoints, prevWordLength, suggestOptions);
result = mGestureSuggest->getSuggestions(proximityInfo, traverseSession, xcoordinates,
ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint, outWords,
@@ -69,26 +65,15 @@
}
return result;
} else {
- if (USE_SUGGEST_INTERFACE_FOR_TYPING) {
- DicTraverseWrapper::initDicTraverseSession(
- traverseSession, this, prevWordCodePoints, prevWordLength, suggestOptions);
- result = mTypingSuggest->getSuggestions(proximityInfo, traverseSession, xcoordinates,
- ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint,
- outWords, frequencies, spaceIndices, outputTypes);
- if (DEBUG_DICT) {
- DUMP_RESULT(outWords, frequencies);
- }
- return result;
- } else {
- std::map<int, int> bigramMap;
- uint8_t bigramFilter[BIGRAM_FILTER_BYTE_SIZE];
- mBigramDictionary->fillBigramAddressToProbabilityMapAndFilter(prevWordCodePoints,
- prevWordLength, &bigramMap, bigramFilter);
- result = mUnigramDictionary->getSuggestions(proximityInfo, xcoordinates, ycoordinates,
- inputCodePoints, inputSize, &bigramMap, bigramFilter,
- suggestOptions->useFullEditDistance(), outWords, frequencies, outputTypes);
- return result;
+ DicTraverseSession::initSessionInstance(
+ traverseSession, this, prevWordCodePoints, prevWordLength, suggestOptions);
+ result = mTypingSuggest->getSuggestions(proximityInfo, traverseSession, xcoordinates,
+ ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint,
+ outWords, frequencies, spaceIndices, outputTypes);
+ if (DEBUG_DICT) {
+ DUMP_RESULT(outWords, frequencies);
}
+ return result;
}
}
@@ -100,7 +85,27 @@
}
int Dictionary::getProbability(const int *word, int length) const {
- return mUnigramDictionary->getProbability(word, length);
+ const uint8_t *const root = mBinaryDicitonaryInfo.getDictRoot();
+ int pos = BinaryFormat::getTerminalPosition(root, word, length,
+ false /* forceLowerCaseSearch */);
+ if (NOT_VALID_WORD == pos) {
+ return NOT_A_PROBABILITY;
+ }
+ const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
+ if (flags & (BinaryFormat::FLAG_IS_BLACKLISTED | BinaryFormat::FLAG_IS_NOT_A_WORD)) {
+ // If this is not a word, or if it's a blacklisted entry, it should behave as
+ // having no probability outside of the suggestion process (where it should be used
+ // for shortcuts).
+ return NOT_A_PROBABILITY;
+ }
+ const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags));
+ if (hasMultipleChars) {
+ pos = BinaryFormat::skipOtherCharacters(root, pos);
+ } else {
+ BinaryFormat::getCodePointAndForwardPointer(root, &pos);
+ }
+ const int unigramProbability = BinaryFormat::readProbabilityWithoutMovingPointer(root, pos);
+ return unigramProbability;
}
bool Dictionary::isValidBigram(const int *word1, int length1, const int *word2, int length2) const {
@@ -108,7 +113,7 @@
}
int Dictionary::getDictFlags() const {
- return mUnigramDictionary->getDictFlags();
+ return mDictFlags;
}
} // namespace latinime
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h
index e6861a3..1f25080 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.h
+++ b/native/jni/src/suggest/core/dictionary/dictionary.h
@@ -20,14 +20,15 @@
#include <stdint.h>
#include "defines.h"
+#include "suggest/core/dictionary/binary_dictionary_info.h"
namespace latinime {
class BigramDictionary;
+class DicTraverseSession;
class ProximityInfo;
class SuggestInterface;
class SuggestOptions;
-class UnigramDictionary;
class Dictionary {
public:
@@ -53,9 +54,9 @@
Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust);
- int getSuggestions(ProximityInfo *proximityInfo, void *traverseSession, int *xcoordinates,
- int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints, int inputSize,
- int *prevWordCodePoints, int prevWordLength, int commitPoint,
+ int getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,
+ int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
+ int inputSize, int *prevWordCodePoints, int prevWordLength, int commitPoint,
const SuggestOptions *const suggestOptions, int *outWords, int *frequencies,
int *spaceIndices, int *outputTypes) const;
@@ -64,11 +65,8 @@
int getProbability(const int *word, int length) const;
bool isValidBigram(const int *word1, int length1, const int *word2, int length2) const;
- const uint8_t *getDict() const { // required to release dictionary buffer
- return mDict;
- }
- const uint8_t *getOffsetDict() const {
- return mOffsetDict;
+ const BinaryDictionaryInfo *getBinaryDictionaryInfo() const {
+ return &mBinaryDicitonaryInfo;
}
int getDictSize() const { return mDictSize; }
int getMmapFd() const { return mMmapFd; }
@@ -78,16 +76,15 @@
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
- const uint8_t *mDict;
- const uint8_t *mOffsetDict;
+ const BinaryDictionaryInfo mBinaryDicitonaryInfo;
// Used only for the mmap version of dictionary loading, but we use these as dummy variables
// also for the malloc version.
const int mDictSize;
+ const int mDictFlags;
const int mMmapFd;
const int mDictBufAdjust;
- const UnigramDictionary *mUnigramDictionary;
const BigramDictionary *mBigramDictionary;
SuggestInterface *mGestureSuggest;
SuggestInterface *mTypingSuggest;
diff --git a/native/jni/src/suggest/core/dictionary/digraph_utils.cpp b/native/jni/src/suggest/core/dictionary/digraph_utils.cpp
index 7a0f755..f53e56e 100644
--- a/native/jni/src/suggest/core/dictionary/digraph_utils.cpp
+++ b/native/jni/src/suggest/core/dictionary/digraph_utils.cpp
@@ -16,9 +16,9 @@
#include "suggest/core/dictionary/digraph_utils.h"
-#include "char_utils.h"
#include "defines.h"
#include "suggest/core/dictionary/binary_format.h"
+#include "utils/char_utils.h"
namespace latinime {
@@ -122,7 +122,7 @@
/* static */ const DigraphUtils::digraph_t *DigraphUtils::getDigraphForDigraphTypeAndCodePoint(
const DigraphUtils::DigraphType digraphType, const int compositeGlyphCodePoint) {
const DigraphUtils::digraph_t *digraphs = 0;
- const int compositeGlyphLowerCodePoint = toLowerCase(compositeGlyphCodePoint);
+ const int compositeGlyphLowerCodePoint = CharUtils::toLowerCase(compositeGlyphCodePoint);
const int digraphsSize =
DigraphUtils::getAllDigraphsForDictionaryAndReturnSize(digraphType, &digraphs);
for (int i = 0; i < digraphsSize; i++) {
diff --git a/native/jni/src/suggest/core/dictionary/multi_bigram_map.h b/native/jni/src/suggest/core/dictionary/multi_bigram_map.h
index fcac98f..ba97e58 100644
--- a/native/jni/src/suggest/core/dictionary/multi_bigram_map.h
+++ b/native/jni/src/suggest/core/dictionary/multi_bigram_map.h
@@ -17,11 +17,10 @@
#ifndef LATINIME_MULTI_BIGRAM_MAP_H
#define LATINIME_MULTI_BIGRAM_MAP_H
-#include <stdint.h>
-
#include "defines.h"
-#include "hash_map_compat.h"
+#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
+#include "utils/hash_map_compat.h"
namespace latinime {
@@ -35,20 +34,20 @@
// Look up the bigram probability for the given word pair from the cached bigram maps.
// Also caches the bigrams if there is space remaining and they have not been cached already.
- int getBigramProbability(const uint8_t *const dicRoot, const int wordPosition,
- const int nextWordPosition, const int unigramProbability) {
+ int getBigramProbability(const BinaryDictionaryInfo *const binaryDicitonaryInfo,
+ const int wordPosition, const int nextWordPosition, const int unigramProbability) {
hash_map_compat<int, BigramMap>::const_iterator mapPosition =
mBigramMaps.find(wordPosition);
if (mapPosition != mBigramMaps.end()) {
return mapPosition->second.getBigramProbability(nextWordPosition, unigramProbability);
}
if (mBigramMaps.size() < MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP) {
- addBigramsForWordPosition(dicRoot, wordPosition);
+ addBigramsForWordPosition(binaryDicitonaryInfo, wordPosition);
return mBigramMaps[wordPosition].getBigramProbability(
nextWordPosition, unigramProbability);
}
- return BinaryFormat::getBigramProbability(
- dicRoot, wordPosition, nextWordPosition, unigramProbability);
+ return BinaryFormat::getBigramProbability(binaryDicitonaryInfo->getDictRoot(),
+ wordPosition, nextWordPosition, unigramProbability);
}
void clear() {
@@ -63,8 +62,9 @@
BigramMap() : mBigramMap(DEFAULT_HASH_MAP_SIZE_FOR_EACH_BIGRAM_MAP) {}
~BigramMap() {}
- void init(const uint8_t *const dicRoot, int position) {
- BinaryFormat::fillBigramProbabilityToHashMap(dicRoot, position, &mBigramMap);
+ void init(const BinaryDictionaryInfo *const binaryDicitonaryInfo, const int position) {
+ BinaryFormat::fillBigramProbabilityToHashMap(
+ binaryDicitonaryInfo->getDictRoot(), position, &mBigramMap);
}
inline int getBigramProbability(const int nextWordPosition, const int unigramProbability)
@@ -78,8 +78,9 @@
hash_map_compat<int, int> mBigramMap;
};
- void addBigramsForWordPosition(const uint8_t *const dicRoot, const int position) {
- mBigramMaps[position].init(dicRoot, position);
+ void addBigramsForWordPosition(const BinaryDictionaryInfo *const binaryDicitonaryInfo,
+ const int position) {
+ mBigramMaps[position].init(binaryDicitonaryInfo, position);
}
hash_map_compat<int, BigramMap> mBigramMaps;
diff --git a/native/jni/src/suggest/core/dictionary/probability_utils.h b/native/jni/src/suggest/core/dictionary/probability_utils.h
new file mode 100644
index 0000000..14d2f84
--- /dev/null
+++ b/native/jni/src/suggest/core/dictionary/probability_utils.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_PROBABILITY_UTILS_H
+#define LATINIME_PROBABILITY_UTILS_H
+
+#include <map>
+#include <stdint.h>
+
+#include "defines.h"
+
+namespace latinime {
+
+class ProbabilityUtils {
+ public:
+ static AK_FORCE_INLINE int backoff(const int unigramProbability) {
+ return unigramProbability;
+ // For some reason, applying the backoff weight gives bad results in tests. To apply the
+ // backoff weight, we divide the probability by 2, which in our storing format means
+ // decreasing the score by 8.
+ // TODO: figure out what's wrong with this.
+ // return unigramProbability > 8 ?
+ // unigramProbability - 8 : (0 == unigramProbability ? 0 : 8);
+ }
+
+ static AK_FORCE_INLINE int computeProbabilityForBigram(
+ const int unigramProbability, const int bigramProbability) {
+ // We divide the range [unigramProbability..255] in 16.5 steps - in other words, we want
+ // the unigram probability to be the median value of the 17th step from the top. A value of
+ // 0 for the bigram probability represents the middle of the 16th step from the top,
+ // while a value of 15 represents the middle of the top step.
+ // See makedict.BinaryDictInputOutput for details.
+ const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability)
+ / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY);
+ return unigramProbability
+ + static_cast<int>(static_cast<float>(bigramProbability + 1) * stepSize);
+ }
+
+ // This returns a probability in log space.
+ static AK_FORCE_INLINE int getProbability(const int position,
+ const std::map<int, int> *const bigramMap,
+ const uint8_t *bigramFilter, const int unigramProbability) {
+ if (!bigramMap || !bigramFilter) {
+ return backoff(unigramProbability);
+ }
+ if (!isInFilter(bigramFilter, position)){
+ return backoff(unigramProbability);
+ }
+ const std::map<int, int>::const_iterator bigramProbabilityIt = bigramMap->find(position);
+ if (bigramProbabilityIt != bigramMap->end()) {
+ const int bigramProbability = bigramProbabilityIt->second;
+ return computeProbabilityForBigram(unigramProbability, bigramProbability);
+ }
+ return backoff(unigramProbability);
+ }
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityUtils);
+};
+}
+#endif /* LATINIME_PROBABILITY_UTILS_H */
diff --git a/native/jni/src/suggest/core/dictionary/terminal_attributes.h b/native/jni/src/suggest/core/dictionary/terminal_attributes.h
index 8377c60..bbd9af0 100644
--- a/native/jni/src/suggest/core/dictionary/terminal_attributes.h
+++ b/native/jni/src/suggest/core/dictionary/terminal_attributes.h
@@ -19,6 +19,7 @@
#include <stdint.h>
+#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
namespace latinime {
@@ -32,8 +33,9 @@
public:
class ShortcutIterator {
public:
- ShortcutIterator(const uint8_t *dict, const int pos, const uint8_t flags)
- : mDict(dict), mPos(pos),
+ ShortcutIterator(const BinaryDictionaryInfo *const binaryDictionaryInfo, const int pos,
+ const uint8_t flags)
+ : mBinaryDicitionaryInfo(binaryDictionaryInfo), mPos(pos),
mHasNextShortcutTarget(0 != (flags & BinaryFormat::FLAG_HAS_SHORTCUT_TARGETS)) {
}
@@ -44,11 +46,13 @@
// Gets the shortcut target itself as an int string. For parameters and return value
// see BinaryFormat::getWordAtAddress.
inline int getNextShortcutTarget(const int maxDepth, int *outWord, int *outFreq) {
- const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer(mDict, &mPos);
+ const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer(
+ mBinaryDicitionaryInfo->getDictRoot(), &mPos);
mHasNextShortcutTarget = 0 != (shortcutFlags & BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT);
unsigned int i;
for (i = 0; i < MAX_WORD_LENGTH; ++i) {
- const int codePoint = BinaryFormat::getCodePointAndForwardPointer(mDict, &mPos);
+ const int codePoint = BinaryFormat::getCodePointAndForwardPointer(
+ mBinaryDicitionaryInfo->getDictRoot(), &mPos);
if (NOT_A_CODE_POINT == codePoint) break;
outWord[i] = codePoint;
}
@@ -57,19 +61,21 @@
}
private:
- const uint8_t *const mDict;
+ const BinaryDictionaryInfo *const mBinaryDicitionaryInfo;
int mPos;
bool mHasNextShortcutTarget;
};
- TerminalAttributes(const uint8_t *const dict, const uint8_t flags, const int pos)
- : mDict(dict), mFlags(flags), mStartPos(pos) {
+ TerminalAttributes(const BinaryDictionaryInfo *const binaryDicitonaryInfo,
+ const uint8_t flags, const int pos)
+ : mBinaryDicitionaryInfo(binaryDicitonaryInfo), mFlags(flags), mStartPos(pos) {
}
inline ShortcutIterator getShortcutIterator() const {
// The size of the shortcuts is stored here so that the whole shortcut chunk can be
// skipped quickly, so we ignore it.
- return ShortcutIterator(mDict, mStartPos + BinaryFormat::SHORTCUT_LIST_SIZE_SIZE, mFlags);
+ return ShortcutIterator(
+ mBinaryDicitionaryInfo, mStartPos + BinaryFormat::SHORTCUT_LIST_SIZE_SIZE, mFlags);
}
bool isBlacklistedOrNotAWord() const {
@@ -78,7 +84,7 @@
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TerminalAttributes);
- const uint8_t *const mDict;
+ const BinaryDictionaryInfo *const mBinaryDicitionaryInfo;
const uint8_t mFlags;
const int mStartPos;
};
diff --git a/native/jni/src/suggest/core/layout/proximity_info.cpp b/native/jni/src/suggest/core/layout/proximity_info.cpp
index 6dd8805..80355c1 100644
--- a/native/jni/src/suggest/core/layout/proximity_info.cpp
+++ b/native/jni/src/suggest/core/layout/proximity_info.cpp
@@ -21,12 +21,12 @@
#include <cstring>
#include <cmath>
-#include "char_utils.h"
#include "defines.h"
#include "jni.h"
#include "suggest/core/layout/additional_proximity_chars.h"
#include "suggest/core/layout/geometry_utils.h"
#include "suggest/core/layout/proximity_info_params.h"
+#include "utils/char_utils.h"
namespace latinime {
@@ -165,7 +165,7 @@
// TODO: Optimize
for (int i = 0; i < KEY_COUNT; ++i) {
const int code = mKeyCodePoints[i];
- const int lowerCode = toLowerCase(code);
+ const int lowerCode = CharUtils::toLowerCase(code);
mCenterXsG[i] = mKeyXCoordinates[i] + mKeyWidths[i] / 2;
mCenterYsG[i] = mKeyYCoordinates[i] + mKeyHeights[i] / 2;
mCodeToKeyMap[lowerCode] = i;
diff --git a/native/jni/src/suggest/core/layout/proximity_info.h b/native/jni/src/suggest/core/layout/proximity_info.h
index 6d2ddd4..6ca2fdd 100644
--- a/native/jni/src/suggest/core/layout/proximity_info.h
+++ b/native/jni/src/suggest/core/layout/proximity_info.h
@@ -18,9 +18,9 @@
#define LATINIME_PROXIMITY_INFO_H
#include "defines.h"
-#include "hash_map_compat.h"
#include "jni.h"
#include "suggest/core/layout/proximity_info_utils.h"
+#include "utils/hash_map_compat.h"
namespace latinime {
diff --git a/native/jni/src/suggest/core/layout/proximity_info_state.cpp b/native/jni/src/suggest/core/layout/proximity_info_state.cpp
index 2bd3ceb..4e53992 100644
--- a/native/jni/src/suggest/core/layout/proximity_info_state.cpp
+++ b/native/jni/src/suggest/core/layout/proximity_info_state.cpp
@@ -26,6 +26,7 @@
#include "suggest/core/layout/geometry_utils.h"
#include "suggest/core/layout/proximity_info.h"
#include "suggest/core/layout/proximity_info_state_utils.h"
+#include "utils/char_utils.h"
namespace latinime {
@@ -175,7 +176,7 @@
const int index = inputIndex * mProximityInfo->getKeyCount() + keyId;
return min(mSampledNormalizedSquaredLengthCache[index], mMaxPointToKeyLength);
}
- if (isIntentionalOmissionCodePoint(codePoint)) {
+ if (CharUtils::isIntentionalOmissionCodePoint(codePoint)) {
return 0.0f;
}
// If the char is not a key on the keyboard then return the max length.
@@ -203,7 +204,7 @@
const bool checkProximityChars, int *proximityIndex) const {
const int *currentCodePoints = getProximityCodePointsAt(index);
const int firstCodePoint = currentCodePoints[0];
- const int baseLowerC = toBaseLowerCase(codePoint);
+ const int baseLowerC = CharUtils::toBaseLowerCase(codePoint);
// The first char in the array is what user typed. If it matches right away, that means the
// user typed that same char for this pos.
@@ -215,7 +216,7 @@
// If the non-accented, lowercased version of that first character matches c, then we have a
// non-accented version of the accented character the user typed. Treat it as a close char.
- if (toBaseLowerCase(firstCodePoint) == baseLowerC) {
+ if (CharUtils::toBaseLowerCase(firstCodePoint) == baseLowerC) {
return PROXIMITY_CHAR;
}
@@ -257,8 +258,8 @@
if (!isUsed()) {
return UNRELATED_CHAR;
}
- const int lowerCodePoint = toLowerCase(codePoint);
- const int baseLowerCodePoint = toBaseCodePoint(lowerCodePoint);
+ const int lowerCodePoint = CharUtils::toLowerCase(codePoint);
+ const int baseLowerCodePoint = CharUtils::toBaseCodePoint(lowerCodePoint);
for (int i = 0; i < static_cast<int>(mSampledSearchKeyVectors[index].size()); ++i) {
if (mSampledSearchKeyVectors[index][i] == lowerCodePoint
|| mSampledSearchKeyVectors[index][i] == baseLowerCodePoint) {
diff --git a/native/jni/src/suggest/core/layout/proximity_info_state.h b/native/jni/src/suggest/core/layout/proximity_info_state.h
index fd09307..0079ab5 100644
--- a/native/jni/src/suggest/core/layout/proximity_info_state.h
+++ b/native/jni/src/suggest/core/layout/proximity_info_state.h
@@ -20,11 +20,10 @@
#include <cstring> // for memset()
#include <vector>
-#include "char_utils.h"
#include "defines.h"
-#include "hash_map_compat.h"
#include "suggest/core/layout/proximity_info_params.h"
#include "suggest/core/layout/proximity_info_state_utils.h"
+#include "utils/hash_map_compat.h"
namespace latinime {
diff --git a/native/jni/src/suggest/core/layout/proximity_info_state_utils.h b/native/jni/src/suggest/core/layout/proximity_info_state_utils.h
index 1837c7a..66fe079 100644
--- a/native/jni/src/suggest/core/layout/proximity_info_state_utils.h
+++ b/native/jni/src/suggest/core/layout/proximity_info_state_utils.h
@@ -21,7 +21,7 @@
#include <vector>
#include "defines.h"
-#include "hash_map_compat.h"
+#include "utils/hash_map_compat.h"
namespace latinime {
class ProximityInfo;
diff --git a/native/jni/src/suggest/core/layout/proximity_info_utils.h b/native/jni/src/suggest/core/layout/proximity_info_utils.h
index c3a275b..54f7539 100644
--- a/native/jni/src/suggest/core/layout/proximity_info_utils.h
+++ b/native/jni/src/suggest/core/layout/proximity_info_utils.h
@@ -19,11 +19,11 @@
#include <cmath>
-#include "char_utils.h"
#include "defines.h"
-#include "hash_map_compat.h"
#include "suggest/core/layout/additional_proximity_chars.h"
#include "suggest/core/layout/geometry_utils.h"
+#include "utils/char_utils.h"
+#include "utils/hash_map_compat.h"
namespace latinime {
class ProximityInfoUtils {
@@ -37,7 +37,7 @@
if (c == NOT_A_CODE_POINT) {
return NOT_AN_INDEX;
}
- const int lowerCode = toLowerCase(c);
+ const int lowerCode = CharUtils::toLowerCase(c);
hash_map_compat<int, int>::const_iterator mapPos = codeToKeyMap->find(lowerCode);
if (mapPos != codeToKeyMap->end()) {
return mapPos->second;
diff --git a/native/jni/src/suggest/core/policy/weighting.cpp b/native/jni/src/suggest/core/policy/weighting.cpp
index d01531f..0c57ca0 100644
--- a/native/jni/src/suggest/core/policy/weighting.cpp
+++ b/native/jni/src/suggest/core/policy/weighting.cpp
@@ -16,7 +16,6 @@
#include "suggest/core/policy/weighting.h"
-#include "char_utils.h"
#include "defines.h"
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_profiler.h"
@@ -143,7 +142,7 @@
case CT_TERMINAL: {
const float languageImprobability =
DicNodeUtils::getBigramNodeImprobability(
- traverseSession->getOffsetDict(), dicNode, multiBigramMap);
+ traverseSession->getBinaryDictionaryInfo(), dicNode, multiBigramMap);
return weighting->getTerminalLanguageCost(traverseSession, dicNode, languageImprobability);
}
case CT_NEW_WORD_SPACE_SUBSTITUTION:
diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.cpp b/native/jni/src/suggest/core/session/dic_traverse_session.cpp
index 4e63450..c398cae 100644
--- a/native/jni/src/suggest/core/session/dic_traverse_session.cpp
+++ b/native/jni/src/suggest/core/session/dic_traverse_session.cpp
@@ -17,55 +17,19 @@
#include "suggest/core/session/dic_traverse_session.h"
#include "defines.h"
-#include "dic_traverse_wrapper.h"
#include "jni.h"
#include "suggest/core/dicnode/dic_node_utils.h"
+#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
#include "suggest/core/dictionary/dictionary.h"
namespace latinime {
-const int DicTraverseSession::CACHE_START_INPUT_LENGTH_THRESHOLD = 20;
-
-// A factory method for DicTraverseSession
-static void *getSessionInstance(JNIEnv *env, jstring localeStr) {
- return new DicTraverseSession(env, localeStr);
-}
-
-// TODO: Pass "DicTraverseSession *traverseSession" when the source code structure settles down.
-static void initSessionInstance(void *traverseSession, const Dictionary *const dictionary,
- const int *prevWord, const int prevWordLength,
- const SuggestOptions *const suggestOptions) {
- if (traverseSession) {
- DicTraverseSession *tSession = static_cast<DicTraverseSession *>(traverseSession);
- tSession->init(dictionary, prevWord, prevWordLength, suggestOptions);
- }
-}
-
-// TODO: Pass "DicTraverseSession *traverseSession" when the source code structure settles down.
-static void releaseSessionInstance(void *traverseSession) {
- delete static_cast<DicTraverseSession *>(traverseSession);
-}
-
-// An ad-hoc internal class to register the factory method defined above
-class TraverseSessionFactoryRegisterer {
- public:
- TraverseSessionFactoryRegisterer() {
- DicTraverseWrapper::setTraverseSessionFactoryMethod(getSessionInstance);
- DicTraverseWrapper::setTraverseSessionInitMethod(initSessionInstance);
- DicTraverseWrapper::setTraverseSessionReleaseMethod(releaseSessionInstance);
- }
- private:
- DISALLOW_COPY_AND_ASSIGN(TraverseSessionFactoryRegisterer);
-};
-
-// To invoke the TraverseSessionFactoryRegisterer constructor in the global constructor.
-static TraverseSessionFactoryRegisterer traverseSessionFactoryRegisterer;
-
void DicTraverseSession::init(const Dictionary *const dictionary, const int *prevWord,
int prevWordLength, const SuggestOptions *const suggestOptions) {
mDictionary = dictionary;
- mMultiWordCostMultiplier = BinaryFormat::getMultiWordCostMultiplier(mDictionary->getDict(),
+ mMultiWordCostMultiplier = BinaryFormat::getMultiWordCostMultiplier(
+ mDictionary->getBinaryDictionaryInfo()->getDictBuf(),
mDictionary->getDictSize());
mSuggestOptions = suggestOptions;
if (!prevWord) {
@@ -73,12 +37,14 @@
return;
}
// TODO: merge following similar calls to getTerminalPosition into one case-insensitive call.
- mPrevWordPos = BinaryFormat::getTerminalPosition(dictionary->getOffsetDict(), prevWord,
+ mPrevWordPos = BinaryFormat::getTerminalPosition(
+ dictionary->getBinaryDictionaryInfo()->getDictRoot(), prevWord,
prevWordLength, false /* forceLowerCaseSearch */);
if (mPrevWordPos == NOT_VALID_WORD) {
// Check bigrams for lower-cased previous word if original was not found. Useful for
// auto-capitalized words like "The [current_word]".
- mPrevWordPos = BinaryFormat::getTerminalPosition(dictionary->getOffsetDict(), prevWord,
+ mPrevWordPos = BinaryFormat::getTerminalPosition(
+ dictionary->getBinaryDictionaryInfo()->getDictRoot(), prevWord,
prevWordLength, true /* forceLowerCaseSearch */);
}
}
@@ -93,8 +59,8 @@
maxSpatialDistance, maxPointerCount);
}
-const uint8_t *DicTraverseSession::getOffsetDict() const {
- return mDictionary->getOffsetDict();
+const BinaryDictionaryInfo *DicTraverseSession::getBinaryDictionaryInfo() const {
+ return mDictionary->getBinaryDictionaryInfo();
}
int DicTraverseSession::getDictFlags() const {
diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.h b/native/jni/src/suggest/core/session/dic_traverse_session.h
index e5c7f8e..630b3b5 100644
--- a/native/jni/src/suggest/core/session/dic_traverse_session.h
+++ b/native/jni/src/suggest/core/session/dic_traverse_session.h
@@ -28,12 +28,32 @@
namespace latinime {
+class BinaryDictionaryInfo;
class Dictionary;
class ProximityInfo;
class SuggestOptions;
class DicTraverseSession {
public:
+
+ // A factory method for DicTraverseSession
+ static AK_FORCE_INLINE void *getSessionInstance(JNIEnv *env, jstring localeStr) {
+ return new DicTraverseSession(env, localeStr);
+ }
+
+ static AK_FORCE_INLINE void initSessionInstance(DicTraverseSession *traverseSession,
+ const Dictionary *const dictionary, const int *prevWord, const int prevWordLength,
+ const SuggestOptions *const suggestOptions) {
+ if (traverseSession) {
+ DicTraverseSession *tSession = static_cast<DicTraverseSession *>(traverseSession);
+ tSession->init(dictionary, prevWord, prevWordLength, suggestOptions);
+ }
+ }
+
+ static AK_FORCE_INLINE void releaseSessionInstance(DicTraverseSession *traverseSession) {
+ delete traverseSession;
+ }
+
AK_FORCE_INLINE DicTraverseSession(JNIEnv *env, jstring localeStr)
: mPrevWordPos(NOT_VALID_WORD), mProximityInfo(0),
mDictionary(0), mSuggestOptions(0), mDicNodesCache(), mMultiBigramMap(),
@@ -56,7 +76,7 @@
void resetCache(const int nextActiveCacheSize, const int maxWords);
// TODO: Remove
- const uint8_t *getOffsetDict() const;
+ const BinaryDictionaryInfo *getBinaryDictionaryInfo() const;
int getDictFlags() const;
//--------------------
diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp
index 9444187..1f108e4 100644
--- a/native/jni/src/suggest/core/suggest.cpp
+++ b/native/jni/src/suggest/core/suggest.cpp
@@ -16,7 +16,6 @@
#include "suggest/core/suggest.h"
-#include "char_utils.h"
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_priority_queue.h"
#include "suggest/core/dicnode/dic_node_vector.h"
@@ -106,8 +105,8 @@
traverseSession->resetCache(TRAVERSAL->getMaxCacheSize(), MAX_RESULTS);
// Create a new dic node here
DicNode rootNode;
- DicNodeUtils::initAsRoot(traverseSession->getDicRootPos(),
- traverseSession->getOffsetDict(), traverseSession->getPrevWordPos(), &rootNode);
+ DicNodeUtils::initAsRoot(traverseSession->getBinaryDictionaryInfo(),
+ traverseSession->getPrevWordPos(), &rootNode);
traverseSession->getDicTraverseCache()->copyPushActive(&rootNode);
}
}
@@ -159,7 +158,7 @@
terminalIndex, doubleLetterTerminalIndex, doubleLetterLevel);
const float compoundDistance = terminalDicNode->getCompoundDistance(languageWeight)
+ doubleLetterCost;
- const TerminalAttributes terminalAttributes(traverseSession->getOffsetDict(),
+ const TerminalAttributes terminalAttributes(traverseSession->getBinaryDictionaryInfo(),
terminalDicNode->getFlags(), terminalDicNode->getAttributesPos());
const bool isPossiblyOffensiveWord = terminalDicNode->getProbability() <= 0;
const bool isExactMatch = terminalDicNode->isExactMatch();
@@ -285,7 +284,7 @@
}
DicNodeUtils::getAllChildDicNodes(
- &dicNode, traverseSession->getOffsetDict(), &childDicNodes);
+ &dicNode, traverseSession->getBinaryDictionaryInfo(), &childDicNodes);
const int childDicNodesSize = childDicNodes.getSizeAndLock();
for (int i = 0; i < childDicNodesSize; ++i) {
@@ -432,7 +431,8 @@
void Suggest::processDicNodeAsOmission(
DicTraverseSession *traverseSession, DicNode *dicNode) const {
DicNodeVector childDicNodes;
- DicNodeUtils::getAllChildDicNodes(dicNode, traverseSession->getOffsetDict(), &childDicNodes);
+ DicNodeUtils::getAllChildDicNodes(
+ dicNode, traverseSession->getBinaryDictionaryInfo(), &childDicNodes);
const int size = childDicNodes.getSizeAndLock();
for (int i = 0; i < size; i++) {
@@ -457,7 +457,7 @@
DicNode *dicNode) const {
const int16_t pointIndex = dicNode->getInputIndex(0);
DicNodeVector childDicNodes;
- DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getOffsetDict(),
+ DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getBinaryDictionaryInfo(),
traverseSession->getProximityInfoState(0), pointIndex + 1, true, &childDicNodes);
const int size = childDicNodes.getSizeAndLock();
for (int i = 0; i < size; i++) {
@@ -475,14 +475,14 @@
DicNode *dicNode) const {
const int16_t pointIndex = dicNode->getInputIndex(0);
DicNodeVector childDicNodes1;
- DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getOffsetDict(),
+ DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getBinaryDictionaryInfo(),
traverseSession->getProximityInfoState(0), pointIndex + 1, false, &childDicNodes1);
const int childSize1 = childDicNodes1.getSizeAndLock();
for (int i = 0; i < childSize1; i++) {
if (childDicNodes1[i]->hasChildren()) {
DicNodeVector childDicNodes2;
DicNodeUtils::getProximityChildDicNodes(
- childDicNodes1[i], traverseSession->getOffsetDict(),
+ childDicNodes1[i], traverseSession->getBinaryDictionaryInfo(),
traverseSession->getProximityInfoState(0), pointIndex, false, &childDicNodes2);
const int childSize2 = childDicNodes2.getSizeAndLock();
for (int j = 0; j < childSize2; j++) {
@@ -522,8 +522,8 @@
// Create a non-cached node here.
DicNode newDicNode;
- DicNodeUtils::initAsRootWithPreviousWord(traverseSession->getDicRootPos(),
- traverseSession->getOffsetDict(), dicNode, &newDicNode);
+ DicNodeUtils::initAsRootWithPreviousWord(
+ traverseSession->getBinaryDictionaryInfo(), dicNode, &newDicNode);
const CorrectionType correctionType = spaceSubstitution ?
CT_NEW_WORD_SPACE_SUBSTITUTION : CT_NEW_WORD_SPACE_OMITTION;
Weighting::addCostAndForwardInputIndex(WEIGHTING, correctionType, traverseSession, dicNode,
diff --git a/native/jni/src/suggest_options.h b/native/jni/src/suggest/core/suggest_options.h
similarity index 100%
rename from native/jni/src/suggest_options.h
rename to native/jni/src/suggest/core/suggest_options.h
diff --git a/native/jni/src/suggest/policyimpl/typing/typing_traversal.h b/native/jni/src/suggest/policyimpl/typing/typing_traversal.h
index b212fe1..e21b318 100644
--- a/native/jni/src/suggest/policyimpl/typing/typing_traversal.h
+++ b/native/jni/src/suggest/policyimpl/typing/typing_traversal.h
@@ -19,7 +19,6 @@
#include <stdint.h>
-#include "char_utils.h"
#include "defines.h"
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_vector.h"
@@ -27,6 +26,7 @@
#include "suggest/core/policy/traversal.h"
#include "suggest/core/session/dic_traverse_session.h"
#include "suggest/policyimpl/typing/scoring_params.h"
+#include "utils/char_utils.h"
namespace latinime {
class TypingTraversal : public Traversal {
@@ -64,9 +64,9 @@
}
const int point0Index = dicNode->getInputIndex(0);
const int currentBaseLowerCodePoint =
- toBaseLowerCase(childDicNode->getNodeCodePoint());
+ CharUtils::toBaseLowerCase(childDicNode->getNodeCodePoint());
const int typedBaseLowerCodePoint =
- toBaseLowerCase(traverseSession->getProximityInfoState(0)
+ CharUtils::toBaseLowerCase(traverseSession->getProximityInfoState(0)
->getPrimaryCodePointAt(point0Index));
return (currentBaseLowerCodePoint != typedBaseLowerCodePoint);
}
@@ -172,7 +172,7 @@
}
const int c = dicNode->getOutputWordBuf()[0];
const bool shortCappedWord = dicNode->getDepth()
- < ScoringParams::THRESHOLD_SHORT_WORD_LENGTH && isAsciiUpper(c);
+ < ScoringParams::THRESHOLD_SHORT_WORD_LENGTH && CharUtils::isAsciiUpper(c);
return !shortCappedWord
|| probability >= ScoringParams::THRESHOLD_NEXT_WORD_PROBABILITY_FOR_CAPPED;
}
diff --git a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h
index cb6abd5..17fa110 100644
--- a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h
+++ b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h
@@ -23,6 +23,7 @@
#include "suggest/core/policy/weighting.h"
#include "suggest/core/session/dic_traverse_session.h"
#include "suggest/policyimpl/typing/scoring_params.h"
+#include "utils/char_utils.h"
namespace latinime {
@@ -98,9 +99,9 @@
bool isProximityDicNode(const DicTraverseSession *const traverseSession,
const DicNode *const dicNode) const {
const int pointIndex = dicNode->getInputIndex(0);
- const int primaryCodePoint = toBaseLowerCase(
+ const int primaryCodePoint = CharUtils::toBaseLowerCase(
traverseSession->getProximityInfoState(0)->getPrimaryCodePointAt(pointIndex));
- const int dicNodeChar = toBaseLowerCase(dicNode->getNodeCodePoint());
+ const int dicNodeChar = CharUtils::toBaseLowerCase(dicNode->getNodeCodePoint());
return primaryCodePoint != dicNodeChar;
}
@@ -145,7 +146,7 @@
float getNewWordBigramCost(const DicTraverseSession *const traverseSession,
const DicNode *const dicNode,
MultiBigramMap *const multiBigramMap) const {
- return DicNodeUtils::getBigramNodeImprobability(traverseSession->getOffsetDict(),
+ return DicNodeUtils::getBigramNodeImprobability(traverseSession->getBinaryDictionaryInfo(),
dicNode, multiBigramMap) * ScoringParams::DISTANCE_WEIGHT_LANGUAGE;
}
diff --git a/native/jni/src/suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h b/native/jni/src/suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h
index ec14574..81614bc 100644
--- a/native/jni/src/suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h
+++ b/native/jni/src/suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h
@@ -17,8 +17,8 @@
#ifndef LATINIME_DAEMARU_LEVENSHTEIN_EDIT_DISTANCE_POLICY_H
#define LATINIME_DAEMARU_LEVENSHTEIN_EDIT_DISTANCE_POLICY_H
-#include "char_utils.h"
#include "suggest/policyimpl/utils/edit_distance_policy.h"
+#include "utils/char_utils.h"
namespace latinime {
@@ -31,8 +31,8 @@
~DamerauLevenshteinEditDistancePolicy() {}
AK_FORCE_INLINE float getSubstitutionCost(const int index0, const int index1) const {
- const int c0 = toBaseLowerCase(mString0[index0]);
- const int c1 = toBaseLowerCase(mString1[index1]);
+ const int c0 = CharUtils::toBaseLowerCase(mString0[index0]);
+ const int c1 = CharUtils::toBaseLowerCase(mString1[index1]);
return (c0 == c1) ? 0.0f : 1.0f;
}
@@ -45,10 +45,10 @@
}
AK_FORCE_INLINE bool allowTransposition(const int index0, const int index1) const {
- const int c0 = toBaseLowerCase(mString0[index0]);
- const int c1 = toBaseLowerCase(mString1[index1]);
- if (index0 > 0 && index1 > 0 && c0 == toBaseLowerCase(mString1[index1 - 1])
- && c1 == toBaseLowerCase(mString0[index0 - 1])) {
+ const int c0 = CharUtils::toBaseLowerCase(mString0[index0]);
+ const int c1 = CharUtils::toBaseLowerCase(mString1[index1]);
+ if (index0 > 0 && index1 > 0 && c0 == CharUtils::toBaseLowerCase(mString1[index1 - 1])
+ && c1 == CharUtils::toBaseLowerCase(mString0[index0 - 1])) {
return true;
}
return false;
diff --git a/native/jni/src/unigram_dictionary.cpp b/native/jni/src/unigram_dictionary.cpp
deleted file mode 100644
index 1133256..0000000
--- a/native/jni/src/unigram_dictionary.cpp
+++ /dev/null
@@ -1,988 +0,0 @@
-/*
- * Copyright (C) 2010, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <cstring>
-
-#define LOG_TAG "LatinIME: unigram_dictionary.cpp"
-
-#include "char_utils.h"
-#include "defines.h"
-#include "suggest/core/dictionary/binary_format.h"
-#include "suggest/core/dictionary/dictionary.h"
-#include "suggest/core/dictionary/digraph_utils.h"
-#include "suggest/core/dictionary/terminal_attributes.h"
-#include "suggest/core/layout/proximity_info.h"
-#include "unigram_dictionary.h"
-#include "words_priority_queue.h"
-#include "words_priority_queue_pool.h"
-
-namespace latinime {
-
-// TODO: check the header
-UnigramDictionary::UnigramDictionary(const uint8_t *const streamStart, const unsigned int dictFlags)
- : DICT_ROOT(streamStart), ROOT_POS(0),
- MAX_DIGRAPH_SEARCH_DEPTH(DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH), DICT_FLAGS(dictFlags) {
- if (DEBUG_DICT) {
- AKLOGI("UnigramDictionary - constructor");
- }
-}
-
-UnigramDictionary::~UnigramDictionary() {
-}
-
-// TODO: This needs to take a const int* and not tinker with its contents
-static void addWord(int *word, int length, int probability, WordsPriorityQueue *queue, int type) {
- queue->push(probability, word, length, type);
-}
-
-// Return the replacement code point for a digraph, or 0 if none.
-int UnigramDictionary::getDigraphReplacement(const int *codes, const int i, const int inputSize,
- const DigraphUtils::digraph_t *const digraphs, const unsigned int digraphsSize) const {
-
- // There can't be a digraph if we don't have at least 2 characters to examine
- if (i + 2 > inputSize) return false;
-
- // Search for the first char of some digraph
- int lastDigraphIndex = -1;
- const int thisChar = codes[i];
- for (lastDigraphIndex = digraphsSize - 1; lastDigraphIndex >= 0; --lastDigraphIndex) {
- if (thisChar == digraphs[lastDigraphIndex].first) break;
- }
- // No match: return early
- if (lastDigraphIndex < 0) return 0;
-
- // It's an interesting digraph if the second char matches too.
- if (digraphs[lastDigraphIndex].second == codes[i + 1]) {
- return digraphs[lastDigraphIndex].compositeGlyph;
- } else {
- return 0;
- }
-}
-
-// Mostly the same arguments as the non-recursive version, except:
-// codes is the original value. It points to the start of the work buffer, and gets passed as is.
-// inputSize is the size of the user input (thus, it is the size of codesSrc).
-// codesDest is the current point in the work buffer.
-// codesSrc is the current point in the user-input, original, content-unmodified buffer.
-// codesRemain is the remaining size in codesSrc.
-void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
- const int *xcoordinates, const int *ycoordinates, const int *codesBuffer,
- int *xCoordinatesBuffer, int *yCoordinatesBuffer,
- const int codesBufferSize, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
- const bool useFullEditDistance, const int *codesSrc,
- const int codesRemain, const int currentDepth, int *codesDest, Correction *correction,
- WordsPriorityQueuePool *queuePool,
- const DigraphUtils::digraph_t *const digraphs, const unsigned int digraphsSize) const {
- ASSERT(sizeof(codesDest[0]) == sizeof(codesSrc[0]));
- ASSERT(sizeof(xCoordinatesBuffer[0]) == sizeof(xcoordinates[0]));
- ASSERT(sizeof(yCoordinatesBuffer[0]) == sizeof(ycoordinates[0]));
-
- const int startIndex = static_cast<int>(codesDest - codesBuffer);
- if (currentDepth < MAX_DIGRAPH_SEARCH_DEPTH) {
- for (int i = 0; i < codesRemain; ++i) {
- xCoordinatesBuffer[startIndex + i] = xcoordinates[codesBufferSize - codesRemain + i];
- yCoordinatesBuffer[startIndex + i] = ycoordinates[codesBufferSize - codesRemain + i];
- const int replacementCodePoint =
- getDigraphReplacement(codesSrc, i, codesRemain, digraphs, digraphsSize);
- if (0 != replacementCodePoint) {
- // Found a digraph. We will try both spellings. eg. the word is "pruefen"
-
- // Copy the word up to the first char of the digraph, including proximity chars,
- // and overwrite the primary code with the replacement code point. Then, continue
- // processing on the remaining part of the word, skipping the second char of the
- // digraph.
- // In our example, copy "pru", replace "u" with the version with the diaeresis and
- // continue running on "fen".
- // Make i the index of the second char of the digraph for simplicity. Forgetting
- // to do that results in an infinite recursion so take care!
- ++i;
- memcpy(codesDest, codesSrc, i * sizeof(codesDest[0]));
- codesDest[i - 1] = replacementCodePoint;
- getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
- codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesBufferSize,
- bigramMap, bigramFilter, useFullEditDistance, codesSrc + i + 1,
- codesRemain - i - 1, currentDepth + 1, codesDest + i, correction,
- queuePool, digraphs, digraphsSize);
-
- // Copy the second char of the digraph in place, then continue processing on
- // the remaining part of the word.
- // In our example, after "pru" in the buffer copy the "e", and continue on "fen"
- memcpy(codesDest + i, codesSrc + i, sizeof(codesDest[0]));
- getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
- codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesBufferSize,
- bigramMap, bigramFilter, useFullEditDistance, codesSrc + i, codesRemain - i,
- currentDepth + 1, codesDest + i, correction, queuePool, digraphs,
- digraphsSize);
- return;
- }
- }
- }
-
- // If we come here, we hit the end of the word: let's check it against the dictionary.
- // In our example, we'll come here once for "prufen" and then once for "pruefen".
- // If the word contains several digraphs, we'll come it for the product of them.
- // eg. if the word is "ueberpruefen" we'll test, in order, against
- // "uberprufen", "uberpruefen", "ueberprufen", "ueberpruefen".
- const unsigned int remainingBytes = sizeof(codesDest[0]) * codesRemain;
- if (0 != remainingBytes) {
- memcpy(codesDest, codesSrc, remainingBytes);
- memcpy(&xCoordinatesBuffer[startIndex], &xcoordinates[codesBufferSize - codesRemain],
- sizeof(xCoordinatesBuffer[0]) * codesRemain);
- memcpy(&yCoordinatesBuffer[startIndex], &ycoordinates[codesBufferSize - codesRemain],
- sizeof(yCoordinatesBuffer[0]) * codesRemain);
- }
-
- getWordSuggestions(proximityInfo, xCoordinatesBuffer, yCoordinatesBuffer, codesBuffer,
- startIndex + codesRemain, bigramMap, bigramFilter, useFullEditDistance, correction,
- queuePool);
-}
-
-// bigramMap contains the association <bigram address> -> <bigram probability>
-// bigramFilter is a bloom filter for fast rejection: see functions setInFilter and isInFilter
-// in bigram_dictionary.cpp
-int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *inputCodePoints, const int inputSize,
- const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
- const bool useFullEditDistance, int *outWords, int *frequencies, int *outputTypes) const {
- WordsPriorityQueuePool queuePool(MAX_RESULTS, SUB_QUEUE_MAX_WORDS);
- queuePool.clearAll();
- Correction masterCorrection;
- masterCorrection.resetCorrection();
- const DigraphUtils::digraph_t *digraphs = 0;
- const int digraphsSize =
- DigraphUtils::getAllDigraphsForDictionaryAndReturnSize(DICT_FLAGS, &digraphs);
- if (digraphsSize > 0)
- { // Incrementally tune the word and try all possibilities
- int codesBuffer[sizeof(*inputCodePoints) * inputSize];
- int xCoordinatesBuffer[inputSize];
- int yCoordinatesBuffer[inputSize];
- getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
- xCoordinatesBuffer, yCoordinatesBuffer, inputSize, bigramMap, bigramFilter,
- useFullEditDistance, inputCodePoints, inputSize, 0, codesBuffer, &masterCorrection,
- &queuePool, digraphs, digraphsSize);
- } else { // Normal processing
- getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, inputCodePoints, inputSize,
- bigramMap, bigramFilter, useFullEditDistance, &masterCorrection, &queuePool);
- }
-
- PROF_START(20);
- if (DEBUG_DICT) {
- float ns = queuePool.getMasterQueue()->getHighestNormalizedScore(
- masterCorrection.getPrimaryInputWord(), inputSize, 0, 0, 0);
- ns += 0;
- AKLOGI("Max normalized score = %f", ns);
- }
- const int suggestedWordsCount =
- queuePool.getMasterQueue()->outputSuggestions(masterCorrection.getPrimaryInputWord(),
- inputSize, frequencies, outWords, outputTypes);
-
- if (DEBUG_DICT) {
- float ns = queuePool.getMasterQueue()->getHighestNormalizedScore(
- masterCorrection.getPrimaryInputWord(), inputSize, 0, 0, 0);
- ns += 0;
- AKLOGI("Returning %d words", suggestedWordsCount);
- /// Print the returned words
- for (int j = 0; j < suggestedWordsCount; ++j) {
- int *w = outWords + j * MAX_WORD_LENGTH;
- char s[MAX_WORD_LENGTH];
- for (int i = 0; i <= MAX_WORD_LENGTH; i++) s[i] = w[i];
- (void)s; // To suppress compiler warning
- AKLOGI("%s %i", s, frequencies[j]);
- }
- }
- PROF_END(20);
- PROF_CLOSE;
- return suggestedWordsCount;
-}
-
-void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *inputCodePoints, const int inputSize,
- const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
- const bool useFullEditDistance, Correction *correction, WordsPriorityQueuePool *queuePool)
- const {
- PROF_OPEN;
- PROF_START(0);
- PROF_END(0);
-
- PROF_START(1);
- getOneWordSuggestions(proximityInfo, xcoordinates, ycoordinates, inputCodePoints, bigramMap,
- bigramFilter, useFullEditDistance, inputSize, correction, queuePool);
- PROF_END(1);
-
- PROF_START(2);
- // Note: This line is intentionally left blank
- PROF_END(2);
-
- PROF_START(3);
- // Note: This line is intentionally left blank
- PROF_END(3);
-
- PROF_START(4);
- bool hasAutoCorrectionCandidate = false;
- WordsPriorityQueue *masterQueue = queuePool->getMasterQueue();
- if (masterQueue->size() > 0) {
- float nsForMaster = masterQueue->getHighestNormalizedScore(
- correction->getPrimaryInputWord(), inputSize, 0, 0, 0);
- hasAutoCorrectionCandidate = (nsForMaster > START_TWO_WORDS_CORRECTION_THRESHOLD);
- }
- PROF_END(4);
-
- PROF_START(5);
- // Multiple word suggestions
- if (SUGGEST_MULTIPLE_WORDS
- && inputSize >= MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION) {
- getSplitMultipleWordsSuggestions(proximityInfo, xcoordinates, ycoordinates, inputCodePoints,
- useFullEditDistance, inputSize, correction, queuePool,
- hasAutoCorrectionCandidate);
- }
- PROF_END(5);
-
- PROF_START(6);
- // Note: This line is intentionally left blank
- PROF_END(6);
-
- if (DEBUG_DICT) {
- queuePool->dumpSubQueue1TopSuggestions();
- for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) {
- WordsPriorityQueue *queue = queuePool->getSubQueue(FIRST_WORD_INDEX, i);
- if (queue->size() > 0) {
- WordsPriorityQueue::SuggestedWord *sw = queue->top();
- const int score = sw->mScore;
- const int *word = sw->mWord;
- const int wordLength = sw->mWordLength;
- float ns = Correction::RankingAlgorithm::calcNormalizedScore(
- correction->getPrimaryInputWord(), i, word, wordLength, score);
- ns += 0;
- AKLOGI("--- TOP SUB WORDS for %d --- %d %f [%d]", i, score, ns,
- (ns > TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD));
- DUMP_WORD(correction->getPrimaryInputWord(), i);
- DUMP_WORD(word, wordLength);
- }
- }
- }
-}
-
-void UnigramDictionary::initSuggestions(ProximityInfo *proximityInfo, const int *xCoordinates,
- const int *yCoordinates, const int *codes, const int inputSize,
- Correction *correction) const {
- if (DEBUG_DICT) {
- AKLOGI("initSuggest");
- DUMP_WORD(codes, inputSize);
- }
- correction->initInputParams(proximityInfo, codes, inputSize, xCoordinates, yCoordinates);
- const int maxDepth = min(inputSize * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH);
- correction->initCorrection(proximityInfo, inputSize, maxDepth);
-}
-
-void UnigramDictionary::getOneWordSuggestions(ProximityInfo *proximityInfo,
- const int *xcoordinates, const int *ycoordinates, const int *codes,
- const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
- const bool useFullEditDistance, const int inputSize,
- Correction *correction, WordsPriorityQueuePool *queuePool) const {
- initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, inputSize, correction);
- getSuggestionCandidates(useFullEditDistance, inputSize, bigramMap, bigramFilter, correction,
- queuePool, true /* doAutoCompletion */, DEFAULT_MAX_ERRORS, FIRST_WORD_INDEX);
-}
-
-void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance,
- const int inputSize, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
- Correction *correction, WordsPriorityQueuePool *queuePool,
- const bool doAutoCompletion, const int maxErrors, const int currentWordIndex) const {
- uint8_t totalTraverseCount = correction->pushAndGetTotalTraverseCount();
- if (DEBUG_DICT) {
- AKLOGI("Traverse count %d", totalTraverseCount);
- }
- if (totalTraverseCount > MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT) {
- if (DEBUG_DICT) {
- AKLOGI("Abort traversing %d", totalTraverseCount);
- }
- return;
- }
- // TODO: Remove setCorrectionParams
- correction->setCorrectionParams(0, 0, 0,
- -1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance,
- doAutoCompletion, maxErrors);
- int rootPosition = ROOT_POS;
- // Get the number of children of root, then increment the position
- int childCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &rootPosition);
- int outputIndex = 0;
-
- correction->initCorrectionState(rootPosition, childCount, (inputSize <= 0));
-
- // Depth first search
- while (outputIndex >= 0) {
- if (correction->initProcessState(outputIndex)) {
- int siblingPos = correction->getTreeSiblingPos(outputIndex);
- int firstChildPos;
-
- const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos,
- bigramMap, bigramFilter, correction, &childCount, &firstChildPos, &siblingPos,
- queuePool, currentWordIndex);
- // Update next sibling pos
- correction->setTreeSiblingPos(outputIndex, siblingPos);
-
- if (needsToTraverseChildrenNodes) {
- // Goes to child node
- outputIndex = correction->goDownTree(outputIndex, childCount, firstChildPos);
- }
- } else {
- // Goes to parent sibling node
- outputIndex = correction->getTreeParentIndex(outputIndex);
- }
- }
-}
-
-void UnigramDictionary::onTerminal(const int probability,
- const TerminalAttributes &terminalAttributes, Correction *correction,
- WordsPriorityQueuePool *queuePool, const bool addToMasterQueue,
- const int currentWordIndex) const {
- const int inputIndex = correction->getInputIndex();
- const bool addToSubQueue = inputIndex < SUB_QUEUE_MAX_COUNT;
-
- int wordLength;
- int *wordPointer;
-
- if ((currentWordIndex == FIRST_WORD_INDEX) && addToMasterQueue) {
- WordsPriorityQueue *masterQueue = queuePool->getMasterQueue();
- const int finalProbability =
- correction->getFinalProbability(probability, &wordPointer, &wordLength);
-
- if (0 != finalProbability && !terminalAttributes.isBlacklistedOrNotAWord()) {
- // If the probability is 0, we don't want to add this word. However we still
- // want to add its shortcuts (including a possible whitelist entry) if any.
- // Furthermore, if this is not a word (shortcut only for example) or a blacklisted
- // entry then we never want to suggest this.
- addWord(wordPointer, wordLength, finalProbability, masterQueue,
- Dictionary::KIND_CORRECTION);
- }
-
- const int shortcutProbability = finalProbability > 0 ? finalProbability - 1 : 0;
- // Please note that the shortcut candidates will be added to the master queue only.
- TerminalAttributes::ShortcutIterator iterator = terminalAttributes.getShortcutIterator();
- while (iterator.hasNextShortcutTarget()) {
- // TODO: addWord only supports weak ordering, meaning we have no means
- // to control the order of the shortcuts relative to one another or to the word.
- // We need to either modulate the probability of each shortcut according
- // to its own shortcut probability or to make the queue
- // so that the insert order is protected inside the queue for words
- // with the same score. For the moment we use -1 to make sure the shortcut will
- // never be in front of the word.
- int shortcutTarget[MAX_WORD_LENGTH];
- int shortcutFrequency;
- const int shortcutTargetStringLength = iterator.getNextShortcutTarget(
- MAX_WORD_LENGTH, shortcutTarget, &shortcutFrequency);
- int shortcutScore;
- int kind;
- if (shortcutFrequency == BinaryFormat::WHITELIST_SHORTCUT_PROBABILITY
- && correction->sameAsTyped()) {
- shortcutScore = S_INT_MAX;
- kind = Dictionary::KIND_WHITELIST;
- } else {
- shortcutScore = shortcutProbability;
- kind = Dictionary::KIND_CORRECTION;
- }
- addWord(shortcutTarget, shortcutTargetStringLength, shortcutScore,
- masterQueue, kind);
- }
- }
-
- // We only allow two words + other error correction for words with SUB_QUEUE_MIN_WORD_LENGTH
- // or more length.
- if (inputIndex >= SUB_QUEUE_MIN_WORD_LENGTH && addToSubQueue) {
- WordsPriorityQueue *subQueue;
- subQueue = queuePool->getSubQueue(currentWordIndex, inputIndex);
- if (!subQueue) {
- return;
- }
- const int finalProbability = correction->getFinalProbabilityForSubQueue(
- probability, &wordPointer, &wordLength, inputIndex);
- addWord(wordPointer, wordLength, finalProbability, subQueue, Dictionary::KIND_CORRECTION);
- }
-}
-
-int UnigramDictionary::getSubStringSuggestion(
- ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates,
- const int *codes, const bool useFullEditDistance, Correction *correction,
- WordsPriorityQueuePool *queuePool, const int inputSize,
- const bool hasAutoCorrectionCandidate, const int currentWordIndex,
- const int inputWordStartPos, const int inputWordLength,
- const int outputWordStartPos, const bool isSpaceProximity, int *freqArray,
- int *wordLengthArray, int *outputWord, int *outputWordLength) const {
- if (inputWordLength > MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH) {
- return FLAG_MULTIPLE_SUGGEST_ABORT;
- }
-
- /////////////////////////////////////////////
- // safety net for multiple word suggestion //
- // TODO: Remove this safety net //
- /////////////////////////////////////////////
- int smallWordCount = 0;
- int singleLetterWordCount = 0;
- if (inputWordLength == 1) {
- ++singleLetterWordCount;
- }
- if (inputWordLength <= 2) {
- // small word == single letter or 2-letter word
- ++smallWordCount;
- }
- for (int i = 0; i < currentWordIndex; ++i) {
- const int length = wordLengthArray[i];
- if (length == 1) {
- ++singleLetterWordCount;
- // Safety net to avoid suggesting sequential single letter words
- if (i < (currentWordIndex - 1)) {
- if (wordLengthArray[i + 1] == 1) {
- return FLAG_MULTIPLE_SUGGEST_ABORT;
- }
- } else if (inputWordLength == 1) {
- return FLAG_MULTIPLE_SUGGEST_ABORT;
- }
- }
- if (length <= 2) {
- ++smallWordCount;
- }
- // Safety net to avoid suggesting multiple words with many (4 or more, for now) small words
- if (singleLetterWordCount >= 3 || smallWordCount >= 4) {
- return FLAG_MULTIPLE_SUGGEST_ABORT;
- }
- }
- //////////////////////////////////////////////
- // TODO: Remove the safety net above //
- //////////////////////////////////////////////
-
- int *tempOutputWord = 0;
- int nextWordLength = 0;
- // TODO: Optimize init suggestion
- initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes,
- inputSize, correction);
-
- int word[MAX_WORD_LENGTH];
- int freq = getMostProbableWordLike(
- inputWordStartPos, inputWordLength, correction, word);
- if (freq > 0) {
- nextWordLength = inputWordLength;
- tempOutputWord = word;
- } else if (!hasAutoCorrectionCandidate) {
- if (inputWordStartPos > 0) {
- const int offset = inputWordStartPos;
- initSuggestions(proximityInfo, &xcoordinates[offset], &ycoordinates[offset],
- codes + offset, inputWordLength, correction);
- queuePool->clearSubQueue(currentWordIndex);
- // TODO: pass the bigram list for substring suggestion
- getSuggestionCandidates(useFullEditDistance, inputWordLength,
- 0 /* bigramMap */, 0 /* bigramFilter */, correction, queuePool,
- false /* doAutoCompletion */, MAX_ERRORS_FOR_TWO_WORDS, currentWordIndex);
- if (DEBUG_DICT) {
- if (currentWordIndex < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS) {
- AKLOGI("Dump word candidates(%d) %d", currentWordIndex, inputWordLength);
- for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) {
- queuePool->getSubQueue(currentWordIndex, i)->dumpTopWord();
- }
- }
- }
- }
- WordsPriorityQueue *queue = queuePool->getSubQueue(currentWordIndex, inputWordLength);
- // TODO: Return the correct value depending on doAutoCompletion
- if (!queue || queue->size() <= 0) {
- return FLAG_MULTIPLE_SUGGEST_ABORT;
- }
- int score = 0;
- const float ns = queue->getHighestNormalizedScore(
- correction->getPrimaryInputWord(), inputWordLength,
- &tempOutputWord, &score, &nextWordLength);
- if (DEBUG_DICT) {
- AKLOGI("NS(%d) = %f, Score = %d", currentWordIndex, ns, score);
- }
- // Two words correction won't be done if the score of the first word doesn't exceed the
- // threshold.
- if (ns < TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD
- || nextWordLength < SUB_QUEUE_MIN_WORD_LENGTH) {
- return FLAG_MULTIPLE_SUGGEST_SKIP;
- }
- freq = score >> (nextWordLength + TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER);
- }
- if (DEBUG_DICT) {
- AKLOGI("Freq(%d): %d, length: %d, input length: %d, input start: %d (%d)",
- currentWordIndex, freq, nextWordLength, inputWordLength, inputWordStartPos,
- (currentWordIndex > 0) ? wordLengthArray[0] : 0);
- }
- if (freq <= 0 || nextWordLength <= 0
- || MAX_WORD_LENGTH <= (outputWordStartPos + nextWordLength)) {
- return FLAG_MULTIPLE_SUGGEST_SKIP;
- }
- for (int i = 0; i < nextWordLength; ++i) {
- outputWord[outputWordStartPos + i] = tempOutputWord[i];
- }
-
- // Put output values
- freqArray[currentWordIndex] = freq;
- // TODO: put output length instead of input length
- wordLengthArray[currentWordIndex] = inputWordLength;
- const int tempOutputWordLength = outputWordStartPos + nextWordLength;
- if (outputWordLength) {
- *outputWordLength = tempOutputWordLength;
- }
-
- if ((inputWordStartPos + inputWordLength) < inputSize) {
- if (outputWordStartPos + nextWordLength >= MAX_WORD_LENGTH) {
- return FLAG_MULTIPLE_SUGGEST_SKIP;
- }
- outputWord[tempOutputWordLength] = KEYCODE_SPACE;
- if (outputWordLength) {
- ++*outputWordLength;
- }
- } else if (currentWordIndex >= 1) {
- // TODO: Handle 3 or more words
- const int pairFreq = correction->getFreqForSplitMultipleWords(
- freqArray, wordLengthArray, currentWordIndex + 1, isSpaceProximity, outputWord);
- if (DEBUG_DICT) {
- DUMP_WORD(outputWord, tempOutputWordLength);
- for (int i = 0; i < currentWordIndex + 1; ++i) {
- AKLOGI("Split %d,%d words: freq = %d, length = %d", i, currentWordIndex + 1,
- freqArray[i], wordLengthArray[i]);
- }
- AKLOGI("Split two words: freq = %d, length = %d, %d, isSpace ? %d", pairFreq,
- inputSize, tempOutputWordLength, isSpaceProximity);
- }
- addWord(outputWord, tempOutputWordLength, pairFreq, queuePool->getMasterQueue(),
- Dictionary::KIND_CORRECTION);
- }
- return FLAG_MULTIPLE_SUGGEST_CONTINUE;
-}
-
-void UnigramDictionary::getMultiWordsSuggestionRec(ProximityInfo *proximityInfo,
- const int *xcoordinates, const int *ycoordinates, const int *codes,
- const bool useFullEditDistance, const int inputSize, Correction *correction,
- WordsPriorityQueuePool *queuePool, const bool hasAutoCorrectionCandidate,
- const int startInputPos, const int startWordIndex, const int outputWordLength,
- int *freqArray, int *wordLengthArray, int *outputWord) const {
- if (startWordIndex >= (MULTIPLE_WORDS_SUGGESTION_MAX_WORDS - 1)) {
- // Return if the last word index
- return;
- }
- if (startWordIndex >= 1
- && (hasAutoCorrectionCandidate
- || inputSize < MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION)) {
- // Do not suggest 3+ words if already has auto correction candidate
- return;
- }
- for (int i = startInputPos + 1; i < inputSize; ++i) {
- if (DEBUG_CORRECTION_FREQ) {
- AKLOGI("Multi words(%d), start in %d sep %d start out %d",
- startWordIndex, startInputPos, i, outputWordLength);
- DUMP_WORD(outputWord, outputWordLength);
- }
- int tempOutputWordLength = 0;
- // Current word
- int inputWordStartPos = startInputPos;
- int inputWordLength = i - startInputPos;
- const int suggestionFlag = getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates,
- codes, useFullEditDistance, correction, queuePool, inputSize,
- hasAutoCorrectionCandidate, startWordIndex, inputWordStartPos, inputWordLength,
- outputWordLength, true /* not used */, freqArray, wordLengthArray, outputWord,
- &tempOutputWordLength);
- if (suggestionFlag == FLAG_MULTIPLE_SUGGEST_ABORT) {
- // TODO: break here
- continue;
- } else if (suggestionFlag == FLAG_MULTIPLE_SUGGEST_SKIP) {
- continue;
- }
-
- if (DEBUG_CORRECTION_FREQ) {
- AKLOGI("Do missing space correction");
- }
- // Next word
- // Missing space
- inputWordStartPos = i;
- inputWordLength = inputSize - i;
- if (getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, codes,
- useFullEditDistance, correction, queuePool, inputSize, hasAutoCorrectionCandidate,
- startWordIndex + 1, inputWordStartPos, inputWordLength, tempOutputWordLength,
- false /* missing space */, freqArray, wordLengthArray, outputWord, 0)
- != FLAG_MULTIPLE_SUGGEST_CONTINUE) {
- getMultiWordsSuggestionRec(proximityInfo, xcoordinates, ycoordinates, codes,
- useFullEditDistance, inputSize, correction, queuePool,
- hasAutoCorrectionCandidate, inputWordStartPos, startWordIndex + 1,
- tempOutputWordLength, freqArray, wordLengthArray, outputWord);
- }
-
- // Mistyped space
- ++inputWordStartPos;
- --inputWordLength;
-
- if (inputWordLength <= 0) {
- continue;
- }
-
- const int x = xcoordinates[inputWordStartPos - 1];
- const int y = ycoordinates[inputWordStartPos - 1];
- if (!proximityInfo->hasSpaceProximity(x, y)) {
- continue;
- }
-
- if (DEBUG_CORRECTION_FREQ) {
- AKLOGI("Do mistyped space correction");
- }
- getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, codes,
- useFullEditDistance, correction, queuePool, inputSize, hasAutoCorrectionCandidate,
- startWordIndex + 1, inputWordStartPos, inputWordLength, tempOutputWordLength,
- true /* mistyped space */, freqArray, wordLengthArray, outputWord, 0);
- }
-}
-
-void UnigramDictionary::getSplitMultipleWordsSuggestions(ProximityInfo *proximityInfo,
- const int *xcoordinates, const int *ycoordinates, const int *codes,
- const bool useFullEditDistance, const int inputSize,
- Correction *correction, WordsPriorityQueuePool *queuePool,
- const bool hasAutoCorrectionCandidate) const {
- if (inputSize >= MAX_WORD_LENGTH) return;
- if (DEBUG_DICT) {
- AKLOGI("--- Suggest multiple words");
- }
-
- // Allocating fixed length array on stack
- int outputWord[MAX_WORD_LENGTH];
- int freqArray[MULTIPLE_WORDS_SUGGESTION_MAX_WORDS];
- int wordLengthArray[MULTIPLE_WORDS_SUGGESTION_MAX_WORDS];
- const int outputWordLength = 0;
- const int startInputPos = 0;
- const int startWordIndex = 0;
- getMultiWordsSuggestionRec(proximityInfo, xcoordinates, ycoordinates, codes,
- useFullEditDistance, inputSize, correction, queuePool, hasAutoCorrectionCandidate,
- startInputPos, startWordIndex, outputWordLength, freqArray, wordLengthArray,
- outputWord);
-}
-
-// Wrapper for getMostProbableWordLikeInner, which matches it to the previous
-// interface.
-int UnigramDictionary::getMostProbableWordLike(const int startInputIndex, const int inputSize,
- Correction *correction, int *word) const {
- int inWord[inputSize];
- for (int i = 0; i < inputSize; ++i) {
- inWord[i] = correction->getPrimaryCodePointAt(startInputIndex + i);
- }
- return getMostProbableWordLikeInner(inWord, inputSize, word);
-}
-
-// This function will take the position of a character array within a CharGroup,
-// and check it actually like-matches the word in inWord starting at startInputIndex,
-// that is, it matches it with case and accents squashed.
-// The function returns true if there was a full match, false otherwise.
-// The function will copy on-the-fly the characters in the CharGroup to outNewWord.
-// It will also place the end position of the array in outPos; in outInputIndex,
-// it will place the index of the first char AFTER the match if there was a match,
-// and the initial position if there was not. It makes sense because if there was
-// a match we want to continue searching, but if there was not, we want to go to
-// the next CharGroup.
-// In and out parameters may point to the same location. This function takes care
-// not to use any input parameters after it wrote into its outputs.
-static inline bool testCharGroupForContinuedLikeness(const uint8_t flags,
- const uint8_t *const root, const int startPos, const int *const inWord,
- const int startInputIndex, const int inputSize, int *outNewWord, int *outInputIndex,
- int *outPos) {
- const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags));
- int pos = startPos;
- int codePoint = BinaryFormat::getCodePointAndForwardPointer(root, &pos);
- int baseChar = toBaseLowerCase(codePoint);
- const int wChar = toBaseLowerCase(inWord[startInputIndex]);
-
- if (baseChar != wChar) {
- *outPos = hasMultipleChars ? BinaryFormat::skipOtherCharacters(root, pos) : pos;
- *outInputIndex = startInputIndex;
- return false;
- }
- int inputIndex = startInputIndex;
- outNewWord[inputIndex] = codePoint;
- if (hasMultipleChars) {
- codePoint = BinaryFormat::getCodePointAndForwardPointer(root, &pos);
- while (NOT_A_CODE_POINT != codePoint) {
- baseChar = toBaseLowerCase(codePoint);
- if (inputIndex + 1 >= inputSize || toBaseLowerCase(inWord[++inputIndex]) != baseChar) {
- *outPos = BinaryFormat::skipOtherCharacters(root, pos);
- *outInputIndex = startInputIndex;
- return false;
- }
- outNewWord[inputIndex] = codePoint;
- codePoint = BinaryFormat::getCodePointAndForwardPointer(root, &pos);
- }
- }
- *outInputIndex = inputIndex + 1;
- *outPos = pos;
- return true;
-}
-
-// This function is invoked when a word like the word searched for is found.
-// It will compare the probability to the max probability, and if greater, will
-// copy the word into the output buffer. In output value maxFreq, it will
-// write the new maximum probability if it changed.
-static inline void onTerminalWordLike(const int freq, int *newWord, const int length, int *outWord,
- int *maxFreq) {
- if (freq > *maxFreq) {
- for (int q = 0; q < length; ++q) {
- outWord[q] = newWord[q];
- }
- outWord[length] = 0;
- *maxFreq = freq;
- }
-}
-
-// Will find the highest probability of the words like the one passed as an argument,
-// that is, everything that only differs by case/accents.
-int UnigramDictionary::getMostProbableWordLikeInner(const int *const inWord, const int inputSize,
- int *outWord) const {
- int newWord[MAX_WORD_LENGTH];
- int depth = 0;
- int maxFreq = -1;
- const uint8_t *const root = DICT_ROOT;
- int stackChildCount[MAX_WORD_LENGTH];
- int stackInputIndex[MAX_WORD_LENGTH];
- int stackSiblingPos[MAX_WORD_LENGTH];
-
- int startPos = 0;
- stackChildCount[0] = BinaryFormat::getGroupCountAndForwardPointer(root, &startPos);
- stackInputIndex[0] = 0;
- stackSiblingPos[0] = startPos;
- while (depth >= 0) {
- const int charGroupCount = stackChildCount[depth];
- int pos = stackSiblingPos[depth];
- for (int charGroupIndex = charGroupCount - 1; charGroupIndex >= 0; --charGroupIndex) {
- int inputIndex = stackInputIndex[depth];
- const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
- // Test whether all chars in this group match with the word we are searching for. If so,
- // we want to traverse its children (or if the inputSize match, evaluate its
- // probability). Note that this function will output the position regardless, but will
- // only write into inputIndex if there is a match.
- const bool isAlike = testCharGroupForContinuedLikeness(flags, root, pos, inWord,
- inputIndex, inputSize, newWord, &inputIndex, &pos);
- if (isAlike && (!(BinaryFormat::FLAG_IS_NOT_A_WORD & flags))
- && (BinaryFormat::FLAG_IS_TERMINAL & flags) && (inputIndex == inputSize)) {
- const int probability =
- BinaryFormat::readProbabilityWithoutMovingPointer(root, pos);
- onTerminalWordLike(probability, newWord, inputIndex, outWord, &maxFreq);
- }
- pos = BinaryFormat::skipProbability(flags, pos);
- const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(root, flags, pos);
- const int childrenNodePos = BinaryFormat::readChildrenPosition(root, flags, pos);
- // If we had a match and the word has children, we want to traverse them. We don't have
- // to traverse words longer than the one we are searching for, since they will not match
- // anyway, so don't traverse unless inputIndex < inputSize.
- if (isAlike && (-1 != childrenNodePos) && (inputIndex < inputSize)) {
- // Save position for this depth, to get back to this once children are done
- stackChildCount[depth] = charGroupIndex;
- stackSiblingPos[depth] = siblingPos;
- // Prepare stack values for next depth
- ++depth;
- int childrenPos = childrenNodePos;
- stackChildCount[depth] =
- BinaryFormat::getGroupCountAndForwardPointer(root, &childrenPos);
- stackSiblingPos[depth] = childrenPos;
- stackInputIndex[depth] = inputIndex;
- pos = childrenPos;
- // Go to the next depth level.
- ++depth;
- break;
- } else {
- // No match, or no children, or word too long to ever match: go the next sibling.
- pos = siblingPos;
- }
- }
- --depth;
- }
- return maxFreq;
-}
-
-int UnigramDictionary::getProbability(const int *const inWord, const int length) const {
- const uint8_t *const root = DICT_ROOT;
- int pos = BinaryFormat::getTerminalPosition(root, inWord, length,
- false /* forceLowerCaseSearch */);
- if (NOT_VALID_WORD == pos) {
- return NOT_A_PROBABILITY;
- }
- const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
- if (flags & (BinaryFormat::FLAG_IS_BLACKLISTED | BinaryFormat::FLAG_IS_NOT_A_WORD)) {
- // If this is not a word, or if it's a blacklisted entry, it should behave as
- // having no probability outside of the suggestion process (where it should be used
- // for shortcuts).
- return NOT_A_PROBABILITY;
- }
- const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags));
- if (hasMultipleChars) {
- pos = BinaryFormat::skipOtherCharacters(root, pos);
- } else {
- BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos);
- }
- const int unigramProbability = BinaryFormat::readProbabilityWithoutMovingPointer(root, pos);
- return unigramProbability;
-}
-
-// TODO: remove this function.
-int UnigramDictionary::getBigramPosition(int pos, int *word, int offset, int length) const {
- return -1;
-}
-
-// ProcessCurrentNode returns a boolean telling whether to traverse children nodes or not.
-// If the return value is false, then the caller should read in the output "nextSiblingPosition"
-// to find out the address of the next sibling node and pass it to a new call of processCurrentNode.
-// It is worthy to note that when false is returned, the output values other than
-// nextSiblingPosition are undefined.
-// If the return value is true, then the caller must proceed to traverse the children of this
-// node. processCurrentNode will output the information about the children: their count in
-// newCount, their position in newChildrenPosition, the traverseAllNodes flag in
-// newTraverseAllNodes, the match weight into newMatchRate, the input index into newInputIndex, the
-// diffs into newDiffs, the sibling position in nextSiblingPosition, and the output index into
-// newOutputIndex. Please also note the following caveat: processCurrentNode does not know when
-// there aren't any more nodes at this level, it merely returns the address of the first byte after
-// the current node in nextSiblingPosition. Thus, the caller must keep count of the nodes at any
-// given level, as output into newCount when traversing this level's parent.
-bool UnigramDictionary::processCurrentNode(const int initialPos,
- const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, Correction *correction,
- int *newCount, int *newChildrenPosition, int *nextSiblingPosition,
- WordsPriorityQueuePool *queuePool, const int currentWordIndex) const {
- if (DEBUG_DICT) {
- correction->checkState();
- }
- int pos = initialPos;
-
- // Flags contain the following information:
- // - Address type (MASK_GROUP_ADDRESS_TYPE) on two bits:
- // - FLAG_GROUP_ADDRESS_TYPE_{ONE,TWO,THREE}_BYTES means there are children and their address
- // is on the specified number of bytes.
- // - FLAG_GROUP_ADDRESS_TYPE_NOADDRESS means there are no children, and therefore no address.
- // - FLAG_HAS_MULTIPLE_CHARS: whether this node has multiple char or not.
- // - FLAG_IS_TERMINAL: whether this node is a terminal or not (it may still have children)
- // - FLAG_HAS_BIGRAMS: whether this node has bigrams or not
- const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(DICT_ROOT, &pos);
- const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags));
- const bool isTerminalNode = (0 != (BinaryFormat::FLAG_IS_TERMINAL & flags));
-
- bool needsToInvokeOnTerminal = false;
-
- // This gets only ONE character from the stream. Next there will be:
- // if FLAG_HAS_MULTIPLE CHARS: the other characters of the same node
- // else if FLAG_IS_TERMINAL: the probability
- // else if MASK_GROUP_ADDRESS_TYPE is not NONE: the children address
- // Note that you can't have a node that both is not a terminal and has no children.
- int c = BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos);
- ASSERT(NOT_A_CODE_POINT != c);
-
- // We are going to loop through each character and make it look like it's a different
- // node each time. To do that, we will process characters in this node in order until
- // we find the character terminator. This is signalled by getCodePoint* returning
- // NOT_A_CODE_POINT.
- // As a special case, if there is only one character in this node, we must not read the
- // next bytes so we will simulate the NOT_A_CODE_POINT return by testing the flags.
- // This way, each loop run will look like a "virtual node".
- do {
- // We prefetch the next char. If 'c' is the last char of this node, we will have
- // NOT_A_CODE_POINT in the next char. From this we can decide whether this virtual node
- // should behave as a terminal or not and whether we have children.
- const int nextc = hasMultipleChars
- ? BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos) : NOT_A_CODE_POINT;
- const bool isLastChar = (NOT_A_CODE_POINT == nextc);
- // If there are more chars in this nodes, then this virtual node is not a terminal.
- // If we are on the last char, this virtual node is a terminal if this node is.
- const bool isTerminal = isLastChar && isTerminalNode;
-
- Correction::CorrectionType stateType = correction->processCharAndCalcState(
- c, isTerminal);
- if (stateType == Correction::TRAVERSE_ALL_ON_TERMINAL
- || stateType == Correction::ON_TERMINAL) {
- needsToInvokeOnTerminal = true;
- } else if (stateType == Correction::UNRELATED || correction->needsToPrune()) {
- // We found that this is an unrelated character, so we should give up traversing
- // this node and its children entirely.
- // However we may not be on the last virtual node yet so we skip the remaining
- // characters in this node, the probability if it's there, read the next sibling
- // position to output it, then return false.
- // We don't have to output other values because we return false, as in
- // "don't traverse children".
- if (!isLastChar) {
- pos = BinaryFormat::skipOtherCharacters(DICT_ROOT, pos);
- }
- pos = BinaryFormat::skipProbability(flags, pos);
- *nextSiblingPosition =
- BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
- return false;
- }
-
- // Prepare for the next character. Promote the prefetched char to current char - the loop
- // will take care of prefetching the next. If we finally found our last char, nextc will
- // contain NOT_A_CODE_POINT.
- c = nextc;
- } while (NOT_A_CODE_POINT != c);
-
- if (isTerminalNode) {
- // The probability should be here, because we come here only if this is actually
- // a terminal node, and we are on its last char.
- const int unigramProbability =
- BinaryFormat::readProbabilityWithoutMovingPointer(DICT_ROOT, pos);
- const int childrenAddressPos = BinaryFormat::skipProbability(flags, pos);
- const int attributesPos = BinaryFormat::skipChildrenPosition(flags, childrenAddressPos);
- TerminalAttributes terminalAttributes(DICT_ROOT, flags, attributesPos);
- // bigramMap contains the bigram frequencies indexed by addresses for fast lookup.
- // bigramFilter is a bloom filter of said frequencies for even faster rejection.
- const int probability = BinaryFormat::getProbability(initialPos, bigramMap, bigramFilter,
- unigramProbability);
- onTerminal(probability, terminalAttributes, correction, queuePool, needsToInvokeOnTerminal,
- currentWordIndex);
-
- // If there are more chars in this node, then this virtual node has children.
- // If we are on the last char, this virtual node has children if this node has.
- const bool hasChildren = BinaryFormat::hasChildrenInFlags(flags);
-
- // This character matched the typed character (enough to traverse the node at least)
- // so we just evaluated it. Now we should evaluate this virtual node's children - that
- // is, if it has any. If it has no children, we're done here - so we skip the end of
- // the node, output the siblings position, and return false "don't traverse children".
- // Note that !hasChildren implies isLastChar, so we know we don't have to skip any
- // remaining char in this group for there can't be any.
- if (!hasChildren) {
- pos = BinaryFormat::skipProbability(flags, pos);
- *nextSiblingPosition =
- BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
- return false;
- }
-
- // Optimization: Prune out words that are too long compared to how much was typed.
- if (correction->needsToPrune()) {
- pos = BinaryFormat::skipProbability(flags, pos);
- *nextSiblingPosition =
- BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
- if (DEBUG_DICT_FULL) {
- AKLOGI("Traversing was pruned.");
- }
- return false;
- }
- }
-
- // Now we finished processing this node, and we want to traverse children. If there are no
- // children, we can't come here.
- ASSERT(BinaryFormat::hasChildrenInFlags(flags));
-
- // If this node was a terminal it still has the probability under the pointer (it may have been
- // read, but not skipped - see readProbabilityWithoutMovingPointer).
- // Next come the children position, then possibly attributes (attributes are bigrams only for
- // now, maybe something related to shortcuts in the future).
- // Once this is read, we still need to output the number of nodes in the immediate children of
- // this node, so we read and output it before returning true, as in "please traverse children".
- pos = BinaryFormat::skipProbability(flags, pos);
- int childrenPos = BinaryFormat::readChildrenPosition(DICT_ROOT, flags, pos);
- *nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
- *newCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &childrenPos);
- *newChildrenPosition = childrenPos;
- return true;
-}
-} // namespace latinime
diff --git a/native/jni/src/unigram_dictionary.h b/native/jni/src/unigram_dictionary.h
deleted file mode 100644
index a505032..0000000
--- a/native/jni/src/unigram_dictionary.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright (C) 2010 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_UNIGRAM_DICTIONARY_H
-#define LATINIME_UNIGRAM_DICTIONARY_H
-
-#include <map>
-#include <stdint.h>
-
-#include "defines.h"
-#include "suggest/core/dictionary/digraph_utils.h"
-
-namespace latinime {
-
-class Correction;
-class ProximityInfo;
-class TerminalAttributes;
-class WordsPriorityQueuePool;
-
-class UnigramDictionary {
- public:
- // Error tolerances
- static const int DEFAULT_MAX_ERRORS = 2;
- static const int MAX_ERRORS_FOR_TWO_WORDS = 1;
-
- static const int FLAG_MULTIPLE_SUGGEST_ABORT = 0;
- static const int FLAG_MULTIPLE_SUGGEST_SKIP = 1;
- static const int FLAG_MULTIPLE_SUGGEST_CONTINUE = 2;
- UnigramDictionary(const uint8_t *const streamStart, const unsigned int dictFlags);
- int getProbability(const int *const inWord, const int length) const;
- int getBigramPosition(int pos, int *word, int offset, int length) const;
- int getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *inputCodePoints, const int inputSize,
- const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
- const bool useFullEditDistance, int *outWords, int *frequencies,
- int *outputTypes) const;
- int getDictFlags() const { return DICT_FLAGS; }
- virtual ~UnigramDictionary();
-
- private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(UnigramDictionary);
- void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *inputCodePoints, const int inputSize,
- const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
- const bool useFullEditDistance, Correction *correction,
- WordsPriorityQueuePool *queuePool) const;
- int getDigraphReplacement(const int *codes, const int i, const int inputSize,
- const DigraphUtils::digraph_t *const digraphs, const unsigned int digraphsSize) const;
- void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *codesBuffer, int *xCoordinatesBuffer,
- int *yCoordinatesBuffer, const int codesBufferSize, const std::map<int, int> *bigramMap,
- const uint8_t *bigramFilter, const bool useFullEditDistance, const int *codesSrc,
- const int codesRemain, const int currentDepth, int *codesDest, Correction *correction,
- WordsPriorityQueuePool *queuePool, const DigraphUtils::digraph_t *const digraphs,
- const unsigned int digraphsSize) const;
- void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *codes, const int inputSize,
- Correction *correction) const;
- void getOneWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *codes, const std::map<int, int> *bigramMap,
- const uint8_t *bigramFilter, const bool useFullEditDistance, const int inputSize,
- Correction *correction, WordsPriorityQueuePool *queuePool) const;
- void getSuggestionCandidates(
- const bool useFullEditDistance, const int inputSize,
- const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
- Correction *correction, WordsPriorityQueuePool *queuePool, const bool doAutoCompletion,
- const int maxErrors, const int currentWordIndex) const;
- void getSplitMultipleWordsSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *codes, const bool useFullEditDistance,
- const int inputSize, Correction *correction, WordsPriorityQueuePool *queuePool,
- const bool hasAutoCorrectionCandidate) const;
- void onTerminal(const int freq, const TerminalAttributes &terminalAttributes,
- Correction *correction, WordsPriorityQueuePool *queuePool, const bool addToMasterQueue,
- const int currentWordIndex) const;
- // Process a node by considering proximity, missing and excessive character
- bool processCurrentNode(const int initialPos, const std::map<int, int> *bigramMap,
- const uint8_t *bigramFilter, Correction *correction, int *newCount,
- int *newChildPosition, int *nextSiblingPosition, WordsPriorityQueuePool *queuePool,
- const int currentWordIndex) const;
- int getMostProbableWordLike(const int startInputIndex, const int inputSize,
- Correction *correction, int *word) const;
- int getMostProbableWordLikeInner(const int *const inWord, const int inputSize,
- int *outWord) const;
- int getSubStringSuggestion(ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *codes, const bool useFullEditDistance,
- Correction *correction, WordsPriorityQueuePool *queuePool, const int inputSize,
- const bool hasAutoCorrectionCandidate, const int currentWordIndex,
- const int inputWordStartPos, const int inputWordLength, const int outputWordStartPos,
- const bool isSpaceProximity, int *freqArray, int *wordLengthArray, int *outputWord,
- int *outputWordLength) const;
- void getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *codes, const bool useFullEditDistance,
- const int inputSize, Correction *correction, WordsPriorityQueuePool *queuePool,
- const bool hasAutoCorrectionCandidate, const int startPos, const int startWordIndex,
- const int outputWordLength, int *freqArray, int *wordLengthArray,
- int *outputWord) const;
-
- const uint8_t *const DICT_ROOT;
- const int ROOT_POS;
- const int MAX_DIGRAPH_SEARCH_DEPTH;
- const int DICT_FLAGS;
-};
-} // namespace latinime
-#endif // LATINIME_UNIGRAM_DICTIONARY_H
diff --git a/native/jni/src/char_utils.cpp b/native/jni/src/utils/char_utils.cpp
similarity index 99%
rename from native/jni/src/char_utils.cpp
rename to native/jni/src/utils/char_utils.cpp
index e219beb..0e70396 100644
--- a/native/jni/src/char_utils.cpp
+++ b/native/jni/src/utils/char_utils.cpp
@@ -14,9 +14,10 @@
* limitations under the License.
*/
+#include "utils/char_utils.h"
+
#include <cstdlib>
-#include "char_utils.h"
#include "defines.h"
namespace latinime {
@@ -36,8 +37,7 @@
* $ apt-get install libicu-dev
*
* 3. Build the following code
- * (You need this file, char_utils.h, and defines.h)
- * $ g++ -o char_utils -DUPDATING_CHAR_UTILS char_utils.cpp -licuuc
+ * $ g++ -o char_utils -I.. -DUPDATING_CHAR_UTILS char_utils.cpp -licuuc
*/
#ifdef UPDATING_CHAR_UTILS
#include <stdio.h>
@@ -47,7 +47,7 @@
for (unsigned short c = 0; c < 0xFFFF; c++) {
if (c <= 0x7F) continue;
const unsigned short icu4cLowerC = u_tolower(c);
- const unsigned short myLowerC = latin_tolower(c);
+ const unsigned short myLowerC = CharUtils::latin_tolower(c);
if (c != icu4cLowerC) {
#ifdef CONFIRMING_CHAR_UTILS
if (icu4cLowerC != myLowerC) {
@@ -70,7 +70,7 @@
*
* 5. Update the SORTED_CHAR_MAP[] array below with the output above.
* Then, rebuild with -DCONFIRMING_CHAR_UTILS and confirm the program exits successfully.
- * $ g++ -o char_utils -DUPDATING_CHAR_UTILS -DCONFIRMING_CHAR_UTILS char_utils.cpp -licuuc
+ * $ g++ -o char_utils -I.. -DUPDATING_CHAR_UTILS -DCONFIRMING_CHAR_UTILS char_utils.cpp -licuuc
* $ ./char_utils
* $
*/
@@ -1054,7 +1054,7 @@
- static_cast<int>((static_cast<const struct LatinCapitalSmallPair *>(b))->capital);
}
-unsigned short latin_tolower(const unsigned short c) {
+/* static */ unsigned short CharUtils::latin_tolower(const unsigned short c) {
struct LatinCapitalSmallPair *p =
static_cast<struct LatinCapitalSmallPair *>(bsearch(&c, SORTED_CHAR_MAP,
NELEMS(SORTED_CHAR_MAP), sizeof(SORTED_CHAR_MAP[0]), compare_pair_capital));
@@ -1063,7 +1063,7 @@
/*
* Table mapping most combined Latin, Greek, and Cyrillic characters
- * to their base characters. If c is in range, BASE_CHARS[c] == c
+ * to their base characters. If c is in range, CharUtils::BASE_CHARS[c] == c
* if c is not a combined character, or the base character if it
* is combined.
*
@@ -1074,7 +1074,7 @@
* for ($j = $i; $j < $i + 8; $j++) { \
* printf("0x%04X, ", $base[$j] ? $base[$j] : $j)}; print "\n"; }'
*/
-const unsigned short BASE_CHARS[BASE_CHARS_SIZE] = {
+/* static */ const unsigned short CharUtils::BASE_CHARS[CharUtils::BASE_CHARS_SIZE] = {
/* U+0000 */ 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
/* U+0008 */ 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,
/* U+0010 */ 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,
diff --git a/native/jni/src/utils/char_utils.h b/native/jni/src/utils/char_utils.h
new file mode 100644
index 0000000..2e735a8
--- /dev/null
+++ b/native/jni/src/utils/char_utils.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_CHAR_UTILS_H
+#define LATINIME_CHAR_UTILS_H
+
+#include <cctype>
+
+#include "defines.h"
+
+namespace latinime {
+
+class CharUtils {
+ public:
+ static AK_FORCE_INLINE bool isAsciiUpper(int c) {
+ // Note: isupper(...) reports false positives for some Cyrillic characters, causing them to
+ // be incorrectly lower-cased using toAsciiLower(...) rather than latin_tolower(...).
+ return (c >= 'A' && c <= 'Z');
+ }
+
+ static AK_FORCE_INLINE int toAsciiLower(int c) {
+ return c - 'A' + 'a';
+ }
+
+ static AK_FORCE_INLINE bool isAscii(int c) {
+ return isascii(c) != 0;
+ }
+
+ static AK_FORCE_INLINE int toLowerCase(const int c) {
+ if (isAsciiUpper(c)) {
+ return toAsciiLower(c);
+ }
+ if (isAscii(c)) {
+ return c;
+ }
+ return static_cast<int>(latin_tolower(static_cast<unsigned short>(c)));
+ }
+
+ static AK_FORCE_INLINE int toBaseLowerCase(const int c) {
+ return toLowerCase(toBaseCodePoint(c));
+ }
+
+ static AK_FORCE_INLINE bool isIntentionalOmissionCodePoint(const int codePoint) {
+ // TODO: Do not hardcode here
+ return codePoint == KEYCODE_SINGLE_QUOTE || codePoint == KEYCODE_HYPHEN_MINUS;
+ }
+
+ static AK_FORCE_INLINE int getCodePointCount(const int arraySize, const int *const codePoints) {
+ int size = 0;
+ for (; size < arraySize; ++size) {
+ if (codePoints[size] == '\0') {
+ break;
+ }
+ }
+ return size;
+ }
+
+ static AK_FORCE_INLINE int toBaseCodePoint(int c) {
+ if (c < BASE_CHARS_SIZE) {
+ return static_cast<int>(BASE_CHARS[c]);
+ }
+ return c;
+ }
+
+ static unsigned short latin_tolower(const unsigned short c);
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(CharUtils);
+
+ /**
+ * Table mapping most combined Latin, Greek, and Cyrillic characters
+ * to their base characters. If c is in range, BASE_CHARS[c] == c
+ * if c is not a combined character, or the base character if it
+ * is combined.
+ */
+ static const int BASE_CHARS_SIZE = 0x0500;
+ static const unsigned short BASE_CHARS[BASE_CHARS_SIZE];
+};
+} // namespace latinime
+#endif // LATINIME_CHAR_UTILS_H
diff --git a/native/jni/src/hash_map_compat.h b/native/jni/src/utils/hash_map_compat.h
similarity index 100%
rename from native/jni/src/hash_map_compat.h
rename to native/jni/src/utils/hash_map_compat.h
diff --git a/native/jni/src/words_priority_queue.cpp b/native/jni/src/words_priority_queue.cpp
deleted file mode 100644
index 7e18d0f..0000000
--- a/native/jni/src/words_priority_queue.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (C) 2012, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "words_priority_queue.h"
-
-namespace latinime {
-
-int WordsPriorityQueue::outputSuggestions(const int *before, const int beforeLength,
- int *frequencies, int *outputCodePoints, int* outputTypes) {
- mHighestSuggestedWord = 0;
- const int size = min(MAX_WORDS, static_cast<int>(mSuggestions.size()));
- SuggestedWord *swBuffer[size];
- int index = size - 1;
- while (!mSuggestions.empty() && index >= 0) {
- SuggestedWord *sw = mSuggestions.top();
- if (DEBUG_WORDS_PRIORITY_QUEUE) {
- AKLOGI("dump word. %d", sw->mScore);
- DUMP_WORD(sw->mWord, sw->mWordLength);
- }
- swBuffer[index] = sw;
- mSuggestions.pop();
- --index;
- }
- if (size >= 2) {
- SuggestedWord *nsMaxSw = 0;
- int maxIndex = 0;
- float maxNs = 0;
- for (int i = 0; i < size; ++i) {
- SuggestedWord *tempSw = swBuffer[i];
- if (!tempSw) {
- continue;
- }
- const float tempNs = getNormalizedScore(tempSw, before, beforeLength, 0, 0, 0);
- if (tempNs >= maxNs) {
- maxNs = tempNs;
- maxIndex = i;
- nsMaxSw = tempSw;
- }
- }
- if (maxIndex > 0 && nsMaxSw) {
- memmove(&swBuffer[1], &swBuffer[0], maxIndex * sizeof(swBuffer[0]));
- swBuffer[0] = nsMaxSw;
- }
- }
- for (int i = 0; i < size; ++i) {
- SuggestedWord *sw = swBuffer[i];
- if (!sw) {
- AKLOGE("SuggestedWord is null %d", i);
- continue;
- }
- const int wordLength = sw->mWordLength;
- int *targetAddress = outputCodePoints + i * MAX_WORD_LENGTH;
- frequencies[i] = sw->mScore;
- outputTypes[i] = sw->mType;
- memcpy(targetAddress, sw->mWord, wordLength * sizeof(targetAddress[0]));
- if (wordLength < MAX_WORD_LENGTH) {
- targetAddress[wordLength] = 0;
- }
- sw->mUsed = false;
- }
- return size;
-}
-} // namespace latinime
diff --git a/native/jni/src/words_priority_queue.h b/native/jni/src/words_priority_queue.h
deleted file mode 100644
index 54e8007..0000000
--- a/native/jni/src/words_priority_queue.h
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_WORDS_PRIORITY_QUEUE_H
-#define LATINIME_WORDS_PRIORITY_QUEUE_H
-
-#include <cstring> // for memcpy()
-#include <queue>
-
-#include "correction.h"
-#include "defines.h"
-
-namespace latinime {
-
-class WordsPriorityQueue {
- public:
- struct SuggestedWord {
- int mScore;
- int mWord[MAX_WORD_LENGTH];
- int mWordLength;
- bool mUsed;
- int mType;
-
- void setParams(int score, int *word, int wordLength, int type) {
- mScore = score;
- mWordLength = wordLength;
- memcpy(mWord, word, sizeof(mWord[0]) * wordLength);
- mUsed = true;
- mType = type;
- }
- };
-
- WordsPriorityQueue(int maxWords)
- : mSuggestions(), MAX_WORDS(maxWords),
- mSuggestedWords(new SuggestedWord[MAX_WORD_LENGTH]), mHighestSuggestedWord(0) {
- for (int i = 0; i < MAX_WORD_LENGTH; ++i) {
- mSuggestedWords[i].mUsed = false;
- }
- }
-
- // Non virtual inline destructor -- never inherit this class
- AK_FORCE_INLINE ~WordsPriorityQueue() {
- delete[] mSuggestedWords;
- }
-
- void push(int score, int *word, int wordLength, int type) {
- SuggestedWord *sw = 0;
- if (size() >= MAX_WORDS) {
- sw = mSuggestions.top();
- const int minScore = sw->mScore;
- if (minScore >= score) {
- return;
- }
- sw->mUsed = false;
- mSuggestions.pop();
- }
- if (sw == 0) {
- sw = getFreeSuggestedWord(score, word, wordLength, type);
- } else {
- sw->setParams(score, word, wordLength, type);
- }
- if (sw == 0) {
- AKLOGE("SuggestedWord is accidentally null.");
- return;
- }
- if (DEBUG_WORDS_PRIORITY_QUEUE) {
- AKLOGI("Push word. %d, %d", score, wordLength);
- DUMP_WORD(word, wordLength);
- }
- mSuggestions.push(sw);
- if (!mHighestSuggestedWord || mHighestSuggestedWord->mScore < sw->mScore) {
- mHighestSuggestedWord = sw;
- }
- }
-
- SuggestedWord *top() const {
- if (mSuggestions.empty()) return 0;
- SuggestedWord *sw = mSuggestions.top();
- return sw;
- }
-
- int size() const {
- return static_cast<int>(mSuggestions.size());
- }
-
- AK_FORCE_INLINE void clear() {
- mHighestSuggestedWord = 0;
- while (!mSuggestions.empty()) {
- SuggestedWord *sw = mSuggestions.top();
- if (DEBUG_WORDS_PRIORITY_QUEUE) {
- AKLOGI("Clear word. %d", sw->mScore);
- DUMP_WORD(sw->mWord, sw->mWordLength);
- }
- sw->mUsed = false;
- mSuggestions.pop();
- }
- }
-
- AK_FORCE_INLINE void dumpTopWord() const {
- if (size() <= 0) {
- return;
- }
- DUMP_WORD(mHighestSuggestedWord->mWord, mHighestSuggestedWord->mWordLength);
- }
-
- AK_FORCE_INLINE float getHighestNormalizedScore(const int *before, const int beforeLength,
- int **outWord, int *outScore, int *outLength) const {
- if (!mHighestSuggestedWord) {
- return 0.0f;
- }
- return getNormalizedScore(mHighestSuggestedWord, before, beforeLength, outWord, outScore,
- outLength);
- }
-
- int outputSuggestions(const int *before, const int beforeLength, int *frequencies,
- int *outputCodePoints, int* outputTypes);
-
- private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(WordsPriorityQueue);
- struct wordComparator {
- bool operator ()(SuggestedWord * left, SuggestedWord * right) {
- return left->mScore > right->mScore;
- }
- };
-
- SuggestedWord *getFreeSuggestedWord(int score, int *word, int wordLength, int type) const {
- for (int i = 0; i < MAX_WORD_LENGTH; ++i) {
- if (!mSuggestedWords[i].mUsed) {
- mSuggestedWords[i].setParams(score, word, wordLength, type);
- return &mSuggestedWords[i];
- }
- }
- return 0;
- }
-
- static float getNormalizedScore(SuggestedWord *sw, const int *before, const int beforeLength,
- int **outWord, int *outScore, int *outLength) {
- const int score = sw->mScore;
- int *word = sw->mWord;
- const int wordLength = sw->mWordLength;
- if (outScore) {
- *outScore = score;
- }
- if (outWord) {
- *outWord = word;
- }
- if (outLength) {
- *outLength = wordLength;
- }
- return Correction::RankingAlgorithm::calcNormalizedScore(before, beforeLength, word,
- wordLength, score);
- }
-
- typedef std::priority_queue<SuggestedWord *, std::vector<SuggestedWord *>,
- wordComparator> Suggestions;
- Suggestions mSuggestions;
- const int MAX_WORDS;
- SuggestedWord *mSuggestedWords;
- SuggestedWord *mHighestSuggestedWord;
-};
-} // namespace latinime
-#endif // LATINIME_WORDS_PRIORITY_QUEUE_H
diff --git a/native/jni/src/words_priority_queue_pool.h b/native/jni/src/words_priority_queue_pool.h
deleted file mode 100644
index 2cd210a..0000000
--- a/native/jni/src/words_priority_queue_pool.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_WORDS_PRIORITY_QUEUE_POOL_H
-#define LATINIME_WORDS_PRIORITY_QUEUE_POOL_H
-
-#include "defines.h"
-#include "words_priority_queue.h"
-
-namespace latinime {
-
-class WordsPriorityQueuePool {
- public:
- WordsPriorityQueuePool(int mainQueueMaxWords, int subQueueMaxWords)
- // Note: using placement new() requires the caller to call the destructor explicitly.
- : mMasterQueue(new(mMasterQueueBuf) WordsPriorityQueue(mainQueueMaxWords)) {
- for (int i = 0, subQueueBufOffset = 0;
- i < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS * SUB_QUEUE_MAX_COUNT;
- ++i, subQueueBufOffset += static_cast<int>(sizeof(WordsPriorityQueue))) {
- mSubQueues[i] = new(mSubQueueBuf + subQueueBufOffset)
- WordsPriorityQueue(subQueueMaxWords);
- }
- }
-
- // Non virtual inline destructor -- never inherit this class
- ~WordsPriorityQueuePool() {
- // Note: these explicit calls to the destructor match the calls to placement new() above.
- if (mMasterQueue) mMasterQueue->~WordsPriorityQueue();
- for (int i = 0; i < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS * SUB_QUEUE_MAX_COUNT; ++i) {
- if (mSubQueues[i]) mSubQueues[i]->~WordsPriorityQueue();
- }
- }
-
- WordsPriorityQueue *getMasterQueue() const {
- return mMasterQueue;
- }
-
- WordsPriorityQueue *getSubQueue(const int wordIndex, const int inputWordLength) const {
- if (wordIndex >= MULTIPLE_WORDS_SUGGESTION_MAX_WORDS) {
- return 0;
- }
- if (inputWordLength < 0 || inputWordLength >= SUB_QUEUE_MAX_COUNT) {
- if (DEBUG_WORDS_PRIORITY_QUEUE) {
- ASSERT(false);
- }
- return 0;
- }
- return mSubQueues[wordIndex * SUB_QUEUE_MAX_COUNT + inputWordLength];
- }
-
- inline void clearAll() {
- mMasterQueue->clear();
- for (int i = 0; i < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS; ++i) {
- clearSubQueue(i);
- }
- }
-
- AK_FORCE_INLINE void clearSubQueue(const int wordIndex) {
- for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) {
- WordsPriorityQueue *queue = getSubQueue(wordIndex, i);
- if (queue) {
- queue->clear();
- }
- }
- }
-
- void dumpSubQueue1TopSuggestions() const {
- AKLOGI("DUMP SUBQUEUE1 TOP SUGGESTIONS");
- for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) {
- getSubQueue(0, i)->dumpTopWord();
- }
- }
-
- private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(WordsPriorityQueuePool);
- char mMasterQueueBuf[sizeof(WordsPriorityQueue)];
- char mSubQueueBuf[SUB_QUEUE_MAX_COUNT * MULTIPLE_WORDS_SUGGESTION_MAX_WORDS
- * sizeof(WordsPriorityQueue)];
- WordsPriorityQueue *mMasterQueue;
- WordsPriorityQueue *mSubQueues[SUB_QUEUE_MAX_COUNT * MULTIPLE_WORDS_SUGGESTION_MAX_WORDS];
-};
-} // namespace latinime
-#endif // LATINIME_WORDS_PRIORITY_QUEUE_POOL_H
diff --git a/tests/src/com/android/inputmethod/keyboard/internal/MockKeyboardSwitcher.java b/tests/src/com/android/inputmethod/keyboard/internal/MockKeyboardSwitcher.java
index 2cb9648..90dbaab 100644
--- a/tests/src/com/android/inputmethod/keyboard/internal/MockKeyboardSwitcher.java
+++ b/tests/src/com/android/inputmethod/keyboard/internal/MockKeyboardSwitcher.java
@@ -53,7 +53,7 @@
// Following InputConnection's behavior. Simulating InputType.TYPE_TEXT_FLAG_CAP_WORDS.
private int mAutoCapsState = MockConstants.CAP_MODE_OFF;
- private boolean mIsInDoubleTapTimeout;
+ private boolean mIsInDoubleTapShiftKeyTimeout;
private int mLongPressTimeoutCode;
private final KeyboardState mState = new KeyboardState(this);
@@ -81,7 +81,7 @@
}
public void expireDoubleTapTimeout() {
- mIsInDoubleTapTimeout = false;
+ mIsInDoubleTapShiftKeyTimeout = false;
}
@Override
@@ -125,18 +125,18 @@
}
@Override
- public void startDoubleTapTimer() {
- mIsInDoubleTapTimeout = true;
+ public void startDoubleTapShiftKeyTimer() {
+ mIsInDoubleTapShiftKeyTimeout = true;
}
@Override
- public void cancelDoubleTapTimer() {
- mIsInDoubleTapTimeout = false;
+ public void cancelDoubleTapShiftKeyTimer() {
+ mIsInDoubleTapShiftKeyTimeout = false;
}
@Override
- public boolean isInDoubleTapTimeout() {
- return mIsInDoubleTapTimeout;
+ public boolean isInDoubleTapShiftKeyTimeout() {
+ return mIsInDoubleTapShiftKeyTimeout;
}
@Override