Merge "Make popup hint letter theme-aware" into lmp-dev
diff --git a/java/src/com/android/inputmethod/event/InputTransaction.java b/java/src/com/android/inputmethod/event/InputTransaction.java
index cdff265..b18bf56 100644
--- a/java/src/com/android/inputmethod/event/InputTransaction.java
+++ b/java/src/com/android/inputmethod/event/InputTransaction.java
@@ -33,7 +33,7 @@
// Initial conditions
public final SettingsValues mSettingsValues;
- public final Event mEvent;
+ private final Event mEvent;
public final long mTimestamp;
public final int mSpaceState;
public final int mShiftState;
@@ -42,6 +42,7 @@
private int mRequiredShiftUpdate = SHIFT_NO_UPDATE;
private boolean mRequiresUpdateSuggestions = false;
private boolean mDidAffectContents = false;
+ private boolean mDidAutoCorrect = false;
public InputTransaction(final SettingsValues settingsValues, final Event event,
final long timestamp, final int spaceState, final int shiftState) {
@@ -97,4 +98,19 @@
public boolean didAffectContents() {
return mDidAffectContents;
}
+
+ /**
+ * Indicate that this transaction performed an auto-correction.
+ */
+ public void setDidAutoCorrect() {
+ mDidAutoCorrect = true;
+ }
+
+ /**
+ * Find out whether this transaction performed an auto-correction.
+ * @return Whether this transaction performed an auto-correction.
+ */
+ public boolean didAutoCorrect() {
+ return mDidAutoCorrect;
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index 719aeac..4adc28d 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -248,7 +248,7 @@
break;
case MSG_RESET_CACHES:
final SettingsValues settingsValues = latinIme.mSettings.getCurrent();
- if (latinIme.mInputLogic.retryResetCachesAndReturnSuccess(settingsValues,
+ if (latinIme.mInputLogic.retryResetCachesAndReturnSuccess(
msg.arg1 == 1 /* tryResumeSuggestions */,
msg.arg2 /* remainingTries */, this /* handler */)) {
// If we were able to reset the caches, then we can reload the keyboard.
@@ -752,8 +752,30 @@
loadKeyboard();
}
+ /**
+ * A class that holds information to pass from onStartInputInternal to onStartInputViewInternal
+ *
+ * OnStartInput needs to reload the settings and that will prevent onStartInputViewInternal
+ * from comparing the old settings with the new ones, so we use this memory to pass the
+ * necessary information along.
+ */
+ private static class EditorChangeInfo {
+ public final boolean mIsSameInputType;
+ public final boolean mHasSameOrientation;
+ public EditorChangeInfo(final boolean isSameInputType, final boolean hasSameOrientation) {
+ mIsSameInputType = isSameInputType;
+ mHasSameOrientation = hasSameOrientation;
+ }
+ }
+
+ private EditorChangeInfo mLastEditorChangeInfo;
+
private void onStartInputInternal(final EditorInfo editorInfo, final boolean restarting) {
super.onStartInput(editorInfo, restarting);
+ SettingsValues currentSettingsValues = mSettings.getCurrent();
+ mLastEditorChangeInfo = new EditorChangeInfo(
+ currentSettingsValues.isSameInputType(editorInfo),
+ currentSettingsValues.hasSameOrientation(getResources().getConfiguration()));
}
@SuppressWarnings("deprecation")
@@ -763,9 +785,6 @@
final KeyboardSwitcher switcher = mKeyboardSwitcher;
switcher.updateKeyboardTheme();
final MainKeyboardView mainKeyboardView = switcher.getMainKeyboardView();
- // If we are starting input in a different text field from before, we'll have to reload
- // settings, so currentSettingsValues can't be final.
- SettingsValues currentSettingsValues = mSettings.getCurrent();
if (editorInfo == null) {
Log.e(TAG, "Null EditorInfo in onStartInputView()");
@@ -808,7 +827,7 @@
accessUtils.onStartInputViewInternal(mainKeyboardView, editorInfo, restarting);
}
- final boolean inputTypeChanged = !currentSettingsValues.isSameInputType(editorInfo);
+ final boolean inputTypeChanged = !mLastEditorChangeInfo.mIsSameInputType;
final boolean isDifferentTextField = !restarting || inputTypeChanged;
if (isDifferentTextField) {
mSubtypeSwitcher.updateParametersOnStartInputView();
@@ -853,13 +872,12 @@
canReachInputConnection = true;
}
- if (isDifferentTextField ||
- !currentSettingsValues.hasSameOrientation(getResources().getConfiguration())) {
+ if (isDifferentTextField || !mLastEditorChangeInfo.mHasSameOrientation) {
loadSettings();
}
+ final SettingsValues currentSettingsValues = mSettings.getCurrent();
if (isDifferentTextField) {
mainKeyboardView.closing();
- currentSettingsValues = mSettings.getCurrent();
if (currentSettingsValues.mAutoCorrectionEnabledPerUserSettings) {
suggest.setAutoCorrectionThreshold(
diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
index dee7cd4..5c719ed 100644
--- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
+++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
@@ -426,121 +426,15 @@
cancelDoubleSpacePeriodCountdown();
}
- boolean didAutoCorrect = false;
if (processedEvent.isConsumed()) {
- // A consumed event may have text to commit and an update to the composing state, so
- // we evaluate both. With some combiners, it's possible than an event contains both
- // and we enter both of the following if clauses.
- final CharSequence textToCommit = processedEvent.getTextToCommit();
- if (!TextUtils.isEmpty(textToCommit)) {
- mConnection.commitText(textToCommit, 1);
- inputTransaction.setDidAffectContents();
- }
- if (mWordComposer.isComposingWord()) {
- mConnection.setComposingText(mWordComposer.getTypedWord(), 1);
- inputTransaction.setDidAffectContents();
- inputTransaction.setRequiresUpdateSuggestions();
- }
+ handleConsumedEvent(processedEvent, inputTransaction);
} else if (processedEvent.isFunctionalKeyEvent()) {
- // A special key, like delete, shift, emoji, or the settings key.
- switch (processedEvent.mKeyCode) {
- case Constants.CODE_DELETE:
- handleBackspace(inputTransaction, currentKeyboardScriptId);
- // Backspace is a functional key, but it affects the contents of the editor.
- inputTransaction.setDidAffectContents();
- break;
- case Constants.CODE_SHIFT:
- performRecapitalization(inputTransaction.mSettingsValues);
- inputTransaction.requireShiftUpdate(InputTransaction.SHIFT_UPDATE_NOW);
- if (mSuggestedWords.mIsPrediction) {
- inputTransaction.setRequiresUpdateSuggestions();
- }
- break;
- case Constants.CODE_CAPSLOCK:
- // Note: Changing keyboard to shift lock state is handled in
- // {@link KeyboardSwitcher#onCodeInput(int)}.
- break;
- case Constants.CODE_SYMBOL_SHIFT:
- // Note: Calling back to the keyboard on the symbol Shift key is handled in
- // {@link #onPressKey(int,int,boolean)} and {@link #onReleaseKey(int,boolean)}.
- break;
- case Constants.CODE_SWITCH_ALPHA_SYMBOL:
- // Note: Calling back to the keyboard on symbol key is handled in
- // {@link #onPressKey(int,int,boolean)} and {@link #onReleaseKey(int,boolean)}.
- break;
- case Constants.CODE_SETTINGS:
- onSettingsKeyPressed();
- break;
- case Constants.CODE_SHORTCUT:
- // We need to switch to the shortcut IME. This is handled by LatinIME since the
- // input logic has no business with IME switching.
- break;
- case Constants.CODE_ACTION_NEXT:
- performEditorAction(EditorInfo.IME_ACTION_NEXT);
- break;
- case Constants.CODE_ACTION_PREVIOUS:
- performEditorAction(EditorInfo.IME_ACTION_PREVIOUS);
- break;
- case Constants.CODE_LANGUAGE_SWITCH:
- handleLanguageSwitchKey();
- break;
- case Constants.CODE_EMOJI:
- // Note: Switching emoji keyboard is being handled in
- // {@link KeyboardState#onCodeInput(int,int)}.
- break;
- case Constants.CODE_ALPHA_FROM_EMOJI:
- // Note: Switching back from Emoji keyboard to the main keyboard is being
- // handled in {@link KeyboardState#onCodeInput(int,int)}.
- break;
- case Constants.CODE_SHIFT_ENTER:
- // TODO: remove this object
- final Event tmpEvent = Event.createSoftwareKeypressEvent(Constants.CODE_ENTER,
- processedEvent.mKeyCode, processedEvent.mX, processedEvent.mY,
- processedEvent.isKeyRepeat());
- final InputTransaction tmpTransaction = new InputTransaction(
- inputTransaction.mSettingsValues, tmpEvent,
- inputTransaction.mTimestamp, inputTransaction.mSpaceState,
- inputTransaction.mShiftState);
- didAutoCorrect = handleNonSpecialCharacter(tmpTransaction, handler);
- // Shift + Enter is treated as a functional key but it results in adding a new
- // line, so that does affect the contents of the editor.
- inputTransaction.setDidAffectContents();
- break;
- default:
- throw new RuntimeException("Unknown key code : " + processedEvent.mKeyCode);
- }
+ handleFunctionalEvent(processedEvent, inputTransaction, currentKeyboardScriptId,
+ handler);
} else {
- inputTransaction.setDidAffectContents();
- switch (processedEvent.mCodePoint) {
- case Constants.CODE_ENTER:
- final EditorInfo editorInfo = getCurrentInputEditorInfo();
- final int imeOptionsActionId =
- InputTypeUtils.getImeOptionsActionIdFromEditorInfo(editorInfo);
- if (InputTypeUtils.IME_ACTION_CUSTOM_LABEL == imeOptionsActionId) {
- // Either we have an actionLabel and we should performEditorAction with
- // actionId regardless of its value.
- performEditorAction(editorInfo.actionId);
- } else if (EditorInfo.IME_ACTION_NONE != imeOptionsActionId) {
- // We didn't have an actionLabel, but we had another action to execute.
- // EditorInfo.IME_ACTION_NONE explicitly means no action. In contrast,
- // EditorInfo.IME_ACTION_UNSPECIFIED is the default value for an action, so it
- // means there should be an action and the app didn't bother to set a specific
- // code for it - presumably it only handles one. It does not have to be treated
- // in any specific way: anything that is not IME_ACTION_NONE should be sent to
- // performEditorAction.
- performEditorAction(imeOptionsActionId);
- } else {
- // No action label, and the action from imeOptions is NONE: this is a regular
- // enter key that should input a carriage return.
- didAutoCorrect = handleNonSpecialCharacter(inputTransaction, handler);
- }
- break;
- default:
- didAutoCorrect = handleNonSpecialCharacter(inputTransaction, handler);
- break;
- }
+ handleNonFunctionalEvent(processedEvent, inputTransaction, handler);
}
- if (!didAutoCorrect && processedEvent.mKeyCode != Constants.CODE_SHIFT
+ if (!inputTransaction.didAutoCorrect() && processedEvent.mKeyCode != Constants.CODE_SHIFT
&& processedEvent.mKeyCode != Constants.CODE_CAPSLOCK
&& processedEvent.mKeyCode != Constants.CODE_SWITCH_ALPHA_SYMBOL)
mLastComposedWord.deactivate();
@@ -686,6 +580,153 @@
}
/**
+ * Handle a consumed event.
+ *
+ * Consumed events represent events that have already been consumed, typically by the
+ * combining chain.
+ *
+ * @param event The event to handle.
+ * @param inputTransaction The transaction in progress.
+ */
+ private void handleConsumedEvent(final Event event, final InputTransaction inputTransaction) {
+ // A consumed event may have text to commit and an update to the composing state, so
+ // we evaluate both. With some combiners, it's possible than an event contains both
+ // and we enter both of the following if clauses.
+ final CharSequence textToCommit = event.getTextToCommit();
+ if (!TextUtils.isEmpty(textToCommit)) {
+ mConnection.commitText(textToCommit, 1);
+ inputTransaction.setDidAffectContents();
+ }
+ if (mWordComposer.isComposingWord()) {
+ mConnection.setComposingText(mWordComposer.getTypedWord(), 1);
+ inputTransaction.setDidAffectContents();
+ inputTransaction.setRequiresUpdateSuggestions();
+ }
+ }
+
+ /**
+ * Handle a functional key event.
+ *
+ * A functional event is a special key, like delete, shift, emoji, or the settings key.
+ * Non-special keys are those that generate a single code point.
+ * This includes all letters, digits, punctuation, separators, emoji. It excludes keys that
+ * manage keyboard-related stuff like shift, language switch, settings, layout switch, or
+ * any key that results in multiple code points like the ".com" key.
+ *
+ * @param event The event to handle.
+ * @param inputTransaction The transaction in progress.
+ */
+ private void handleFunctionalEvent(final Event event, final InputTransaction inputTransaction,
+ // TODO: remove these arguments
+ final int currentKeyboardScriptId, final LatinIME.UIHandler handler) {
+ switch (event.mKeyCode) {
+ case Constants.CODE_DELETE:
+ handleBackspaceEvent(event, inputTransaction, currentKeyboardScriptId);
+ // Backspace is a functional key, but it affects the contents of the editor.
+ inputTransaction.setDidAffectContents();
+ break;
+ case Constants.CODE_SHIFT:
+ performRecapitalization(inputTransaction.mSettingsValues);
+ inputTransaction.requireShiftUpdate(InputTransaction.SHIFT_UPDATE_NOW);
+ if (mSuggestedWords.mIsPrediction) {
+ inputTransaction.setRequiresUpdateSuggestions();
+ }
+ break;
+ case Constants.CODE_CAPSLOCK:
+ // Note: Changing keyboard to shift lock state is handled in
+ // {@link KeyboardSwitcher#onCodeInput(int)}.
+ break;
+ case Constants.CODE_SYMBOL_SHIFT:
+ // Note: Calling back to the keyboard on the symbol Shift key is handled in
+ // {@link #onPressKey(int,int,boolean)} and {@link #onReleaseKey(int,boolean)}.
+ break;
+ case Constants.CODE_SWITCH_ALPHA_SYMBOL:
+ // Note: Calling back to the keyboard on symbol key is handled in
+ // {@link #onPressKey(int,int,boolean)} and {@link #onReleaseKey(int,boolean)}.
+ break;
+ case Constants.CODE_SETTINGS:
+ onSettingsKeyPressed();
+ break;
+ case Constants.CODE_SHORTCUT:
+ // We need to switch to the shortcut IME. This is handled by LatinIME since the
+ // input logic has no business with IME switching.
+ break;
+ case Constants.CODE_ACTION_NEXT:
+ performEditorAction(EditorInfo.IME_ACTION_NEXT);
+ break;
+ case Constants.CODE_ACTION_PREVIOUS:
+ performEditorAction(EditorInfo.IME_ACTION_PREVIOUS);
+ break;
+ case Constants.CODE_LANGUAGE_SWITCH:
+ handleLanguageSwitchKey();
+ break;
+ case Constants.CODE_EMOJI:
+ // Note: Switching emoji keyboard is being handled in
+ // {@link KeyboardState#onCodeInput(int,int)}.
+ break;
+ case Constants.CODE_ALPHA_FROM_EMOJI:
+ // Note: Switching back from Emoji keyboard to the main keyboard is being
+ // handled in {@link KeyboardState#onCodeInput(int,int)}.
+ break;
+ case Constants.CODE_SHIFT_ENTER:
+ // TODO: remove this object
+ final Event tmpEvent = Event.createSoftwareKeypressEvent(Constants.CODE_ENTER,
+ event.mKeyCode, event.mX, event.mY, event.isKeyRepeat());
+ handleNonSpecialCharacterEvent(tmpEvent, inputTransaction, handler);
+ // Shift + Enter is treated as a functional key but it results in adding a new
+ // line, so that does affect the contents of the editor.
+ inputTransaction.setDidAffectContents();
+ break;
+ default:
+ throw new RuntimeException("Unknown key code : " + event.mKeyCode);
+ }
+ }
+
+ /**
+ * Handle an event that is not a functional event.
+ *
+ * These events are generally events that cause input, but in some cases they may do other
+ * things like trigger an editor action.
+ *
+ * @param event The event to handle.
+ * @param inputTransaction The transaction in progress.
+ */
+ private void handleNonFunctionalEvent(final Event event,
+ final InputTransaction inputTransaction,
+ // TODO: remove this argument
+ final LatinIME.UIHandler handler) {
+ inputTransaction.setDidAffectContents();
+ switch (event.mCodePoint) {
+ case Constants.CODE_ENTER:
+ final EditorInfo editorInfo = getCurrentInputEditorInfo();
+ final int imeOptionsActionId =
+ InputTypeUtils.getImeOptionsActionIdFromEditorInfo(editorInfo);
+ if (InputTypeUtils.IME_ACTION_CUSTOM_LABEL == imeOptionsActionId) {
+ // Either we have an actionLabel and we should performEditorAction with
+ // actionId regardless of its value.
+ performEditorAction(editorInfo.actionId);
+ } else if (EditorInfo.IME_ACTION_NONE != imeOptionsActionId) {
+ // We didn't have an actionLabel, but we had another action to execute.
+ // EditorInfo.IME_ACTION_NONE explicitly means no action. In contrast,
+ // EditorInfo.IME_ACTION_UNSPECIFIED is the default value for an action, so it
+ // means there should be an action and the app didn't bother to set a specific
+ // code for it - presumably it only handles one. It does not have to be treated
+ // in any specific way: anything that is not IME_ACTION_NONE should be sent to
+ // performEditorAction.
+ performEditorAction(imeOptionsActionId);
+ } else {
+ // No action label, and the action from imeOptions is NONE: this is a regular
+ // enter key that should input a carriage return.
+ handleNonSpecialCharacterEvent(event, inputTransaction, handler);
+ }
+ break;
+ default:
+ handleNonSpecialCharacterEvent(event, inputTransaction, handler);
+ break;
+ }
+ }
+
+ /**
* Handle inputting a code point to the editor.
*
* Non-special keys are those that generate a single code point.
@@ -693,21 +734,19 @@
* manage keyboard-related stuff like shift, language switch, settings, layout switch, or
* any key that results in multiple code points like the ".com" key.
*
+ * @param event The event to handle.
* @param inputTransaction The transaction in progress.
- * @return whether this caused an auto-correction to happen.
*/
- private boolean handleNonSpecialCharacter(final InputTransaction inputTransaction,
+ private void handleNonSpecialCharacterEvent(final Event event,
+ final InputTransaction inputTransaction,
// TODO: remove this argument
final LatinIME.UIHandler handler) {
- final int codePoint = inputTransaction.mEvent.mCodePoint;
+ final int codePoint = event.mCodePoint;
mSpaceState = SpaceState.NONE;
- final boolean didAutoCorrect;
if (inputTransaction.mSettingsValues.isWordSeparator(codePoint)
|| Character.getType(codePoint) == Character.OTHER_SYMBOL) {
- didAutoCorrect = handleSeparator(inputTransaction,
- inputTransaction.mEvent.isSuggestionStripPress(), handler);
+ handleSeparatorEvent(event, inputTransaction, handler);
} else {
- didAutoCorrect = false;
if (SpaceState.PHANTOM == inputTransaction.mSpaceState) {
if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) {
// If we are in the middle of a recorrection, we need to commit the recorrection
@@ -718,22 +757,23 @@
commitTyped(inputTransaction.mSettingsValues, LastComposedWord.NOT_A_SEPARATOR);
}
}
- handleNonSeparator(inputTransaction.mSettingsValues, inputTransaction);
+ handleNonSeparatorEvent(event, inputTransaction.mSettingsValues, inputTransaction);
}
- return didAutoCorrect;
}
/**
* Handle a non-separator.
+ * @param event The event to handle.
* @param settingsValues The current settings values.
* @param inputTransaction The transaction in progress.
*/
- private void handleNonSeparator(final SettingsValues settingsValues,
+ private void handleNonSeparatorEvent(final Event event, final SettingsValues settingsValues,
final InputTransaction inputTransaction) {
- final int codePoint = inputTransaction.mEvent.mCodePoint;
+ final int codePoint = event.mCodePoint;
// TODO: refactor this method to stop flipping isComposingWord around all the time, and
- // make it shorter (possibly cut into several pieces). Also factor handleNonSpecialCharacter
- // which has the same name as other handle* methods but is not the same.
+ // make it shorter (possibly cut into several pieces). Also factor
+ // handleNonSpecialCharacterEvent which has the same name as other handle* methods but is
+ // not the same.
boolean isComposingWord = mWordComposer.isComposingWord();
// TODO: remove isWordConnector() and use isUsuallyFollowedBySpace() instead.
@@ -780,7 +820,7 @@
resetComposingState(false /* alsoResetLastComposedWord */);
}
if (isComposingWord) {
- mWordComposer.applyProcessedEvent(inputTransaction.mEvent);
+ mWordComposer.applyProcessedEvent(event);
// If it's the first letter, make note of auto-caps state
if (mWordComposer.isSingleLetter()) {
mWordComposer.setCapitalizedModeAtStartComposingTime(inputTransaction.mShiftState);
@@ -788,10 +828,10 @@
mConnection.setComposingText(getTextWithUnderline(
mWordComposer.getTypedWord()), 1);
} else {
- final boolean swapWeakSpace = tryStripSpaceAndReturnWhetherShouldSwapInstead(
+ final boolean swapWeakSpace = tryStripSpaceAndReturnWhetherShouldSwapInstead(event,
inputTransaction);
- if (swapWeakSpace && trySwapSwapperAndSpace(inputTransaction)) {
+ if (swapWeakSpace && trySwapSwapperAndSpace(event, inputTransaction)) {
mSpaceState = SpaceState.WEAK;
} else {
sendKeyCodePoint(settingsValues, codePoint);
@@ -804,17 +844,14 @@
/**
* Handle input of a separator code point.
+ * @param event The event to handle.
* @param inputTransaction The transaction in progress.
- * @param isFromSuggestionStrip whether this code point comes from the suggestion strip.
- * @return whether this caused an auto-correction to happen.
*/
- private boolean handleSeparator(final InputTransaction inputTransaction,
- final boolean isFromSuggestionStrip,
+ private void handleSeparatorEvent(final Event event, final InputTransaction inputTransaction,
// TODO: remove this argument
final LatinIME.UIHandler handler) {
- final int codePoint = inputTransaction.mEvent.mCodePoint;
+ final int codePoint = event.mCodePoint;
final SettingsValues settingsValues = inputTransaction.mSettingsValues;
- boolean didAutoCorrect = false;
final boolean wasComposingWord = mWordComposer.isComposingWord();
// We avoid sending spaces in languages without spaces if we were composing.
final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == codePoint
@@ -832,14 +869,14 @@
final String separator = shouldAvoidSendingCode ? LastComposedWord.NOT_A_SEPARATOR
: StringUtils.newSingleCodePointString(codePoint);
commitCurrentAutoCorrection(settingsValues, separator, handler);
- didAutoCorrect = true;
+ inputTransaction.setDidAutoCorrect();
} else {
commitTyped(settingsValues,
StringUtils.newSingleCodePointString(codePoint));
}
}
- final boolean swapWeakSpace = tryStripSpaceAndReturnWhetherShouldSwapInstead(
+ final boolean swapWeakSpace = tryStripSpaceAndReturnWhetherShouldSwapInstead(event,
inputTransaction);
final boolean isInsideDoubleQuoteOrAfterDigit = Constants.CODE_DOUBLE_QUOTE == codePoint
@@ -864,10 +901,10 @@
promotePhantomSpace(settingsValues);
}
- if (tryPerformDoubleSpacePeriod(inputTransaction)) {
+ if (tryPerformDoubleSpacePeriod(event, inputTransaction)) {
mSpaceState = SpaceState.DOUBLE;
inputTransaction.setRequiresUpdateSuggestions();
- } else if (swapWeakSpace && trySwapSwapperAndSpace(inputTransaction)) {
+ } else if (swapWeakSpace && trySwapSwapperAndSpace(event, inputTransaction)) {
mSpaceState = SpaceState.SWAP_PUNCTUATION;
mSuggestionStripViewAccessor.setNeutralSuggestionStrip();
} else if (Constants.CODE_SPACE == codePoint) {
@@ -910,17 +947,16 @@
}
inputTransaction.requireShiftUpdate(InputTransaction.SHIFT_UPDATE_NOW);
- return didAutoCorrect;
}
/**
* Handle a press on the backspace key.
+ * @param event The event to handle.
* @param inputTransaction The transaction in progress.
*/
- private void handleBackspace(final InputTransaction inputTransaction,
+ private void handleBackspaceEvent(final Event event, final InputTransaction inputTransaction,
// TODO: remove this argument, put it into settingsValues
final int currentKeyboardScriptId) {
- final Event event = inputTransaction.mEvent;
mSpaceState = SpaceState.NONE;
mDeleteCount++;
@@ -1071,16 +1107,18 @@
*
* This method will check that there are two characters before the cursor and that the first
* one is a space before it does the actual swapping.
+ * @param event The event to handle.
* @param inputTransaction The transaction in progress.
* @return true if the swap has been performed, false if it was prevented by preliminary checks.
*/
- private boolean trySwapSwapperAndSpace(final InputTransaction inputTransaction) {
+ private boolean trySwapSwapperAndSpace(final Event event,
+ final InputTransaction inputTransaction) {
final int codePointBeforeCursor = mConnection.getCodePointBeforeCursor();
if (Constants.CODE_SPACE != codePointBeforeCursor) {
return false;
}
mConnection.deleteSurroundingText(1, 0);
- final String text = inputTransaction.mEvent.getTextToCommit() + " ";
+ final String text = event.getTextToCommit() + " ";
mConnection.commitText(text, 1);
inputTransaction.requireShiftUpdate(InputTransaction.SHIFT_UPDATE_NOW);
return true;
@@ -1088,13 +1126,14 @@
/*
* Strip a trailing space if necessary and returns whether it's a swap weak space situation.
+ * @param event The event to handle.
* @param inputTransaction The transaction in progress.
* @return whether we should swap the space instead of removing it.
*/
- private boolean tryStripSpaceAndReturnWhetherShouldSwapInstead(
+ private boolean tryStripSpaceAndReturnWhetherShouldSwapInstead(final Event event,
final InputTransaction inputTransaction) {
- final int codePoint = inputTransaction.mEvent.mCodePoint;
- final boolean isFromSuggestionStrip = inputTransaction.mEvent.isSuggestionStripPress();
+ final int codePoint = event.mCodePoint;
+ final boolean isFromSuggestionStrip = event.isSuggestionStripPress();
if (Constants.CODE_ENTER == codePoint &&
SpaceState.SWAP_PUNCTUATION == inputTransaction.mSpaceState) {
mConnection.removeTrailingSpace();
@@ -1139,14 +1178,16 @@
* these conditions are fulfilled, this method applies the transformation and returns true.
* Otherwise, it does nothing and returns false.
*
+ * @param event The event to handle.
* @param inputTransaction The transaction in progress.
* @return true if we applied the double-space-to-period transformation, false otherwise.
*/
- private boolean tryPerformDoubleSpacePeriod(final InputTransaction inputTransaction) {
+ private boolean tryPerformDoubleSpacePeriod(final Event event,
+ final InputTransaction inputTransaction) {
// Check the setting, the typed character and the countdown. If any of the conditions is
// not fulfilled, return false.
if (!inputTransaction.mSettingsValues.mUseDoubleSpacePeriod
- || Constants.CODE_SPACE != inputTransaction.mEvent.mCodePoint
+ || Constants.CODE_SPACE != event.mCodePoint
|| !isDoubleSpacePeriodCountdownActive(inputTransaction)) {
return false;
}
@@ -1974,14 +2015,13 @@
* This method handles the retry, and re-schedules a new retry if we still can't access.
* We only retry up to 5 times before giving up.
*
- * @param settingsValues the current values of the settings.
* @param tryResumeSuggestions Whether we should resume suggestions or not.
* @param remainingTries How many times we may try again before giving up.
* @return whether true if the caches were successfully reset, false otherwise.
*/
// TODO: make this private
- public boolean retryResetCachesAndReturnSuccess(final SettingsValues settingsValues,
- final boolean tryResumeSuggestions, final int remainingTries,
+ public boolean retryResetCachesAndReturnSuccess(final boolean tryResumeSuggestions,
+ final int remainingTries,
// TODO: remove these arguments
final LatinIME.UIHandler handler) {
final boolean shouldFinishComposition = mConnection.hasSelection()
diff --git a/native/jni/NativeFileList.mk b/native/jni/NativeFileList.mk
index 1e3775b..7a732a5 100644
--- a/native/jni/NativeFileList.mk
+++ b/native/jni/NativeFileList.mk
@@ -73,7 +73,6 @@
$(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \
bigram_dict_content.cpp \
language_model_dict_content.cpp \
- probability_dict_content.cpp \
shortcut_dict_content.cpp \
sparse_table_dict_content.cpp \
terminal_position_lookup_table.cpp) \
@@ -126,6 +125,8 @@
defines_test.cpp \
suggest/core/layout/normal_distribution_2d_test.cpp \
suggest/core/dictionary/bloom_filter_test.cpp \
+ suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp \
+ suggest/policyimpl/dictionary/structure/v4/content/probability_entry_test.cpp \
suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer_test.cpp \
suggest/policyimpl/dictionary/utils/trie_map_test.cpp \
utils/autocorrection_threshold_utils_test.cpp \
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp
index b165bf4..07e1051 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp
@@ -22,4 +22,63 @@
return mTrieMap.save(file);
}
+bool LanguageModelDictContent::runGC(
+ const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ const LanguageModelDictContent *const originalContent,
+ int *const outNgramCount) {
+ return runGCInner(terminalIdMap, originalContent->mTrieMap.getEntriesInRootLevel(),
+ 0 /* nextLevelBitmapEntryIndex */, outNgramCount);
+}
+
+ProbabilityEntry LanguageModelDictContent::getNgramProbabilityEntry(
+ const WordIdArrayView prevWordIds, const int wordId) const {
+ if (!prevWordIds.empty()) {
+ // TODO: Read n-gram entry.
+ return ProbabilityEntry();
+ }
+ const TrieMap::Result result = mTrieMap.getRoot(wordId);
+ if (!result.mIsValid) {
+ // Not found.
+ return ProbabilityEntry();
+ }
+ return ProbabilityEntry::decode(result.mValue, mHasHistoricalInfo);
+}
+
+bool LanguageModelDictContent::setNgramProbabilityEntry(const WordIdArrayView prevWordIds,
+ const int terminalId, const ProbabilityEntry *const probabilityEntry) {
+ if (!prevWordIds.empty()) {
+ // TODO: Add n-gram entry.
+ return false;
+ }
+ return mTrieMap.putRoot(terminalId, probabilityEntry->encode(mHasHistoricalInfo));
+}
+
+
+bool LanguageModelDictContent::runGCInner(
+ const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ const TrieMap::TrieMapRange trieMapRange,
+ const int nextLevelBitmapEntryIndex, int *const outNgramCount) {
+ for (auto &entry : trieMapRange) {
+ const auto it = terminalIdMap->find(entry.key());
+ if (it == terminalIdMap->end() || it->second == Ver4DictConstants::NOT_A_TERMINAL_ID) {
+ // The word has been removed.
+ continue;
+ }
+ if (!mTrieMap.put(it->second, entry.value(), nextLevelBitmapEntryIndex)) {
+ return false;
+ }
+ if (outNgramCount) {
+ *outNgramCount += 1;
+ }
+ if (entry.hasNextLevelMap()) {
+ if (!runGCInner(terminalIdMap, entry.getEntriesInNextLevel(),
+ mTrieMap.getNextLevelBitmapEntryIndex(it->second, nextLevelBitmapEntryIndex),
+ outNgramCount)) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h
index 2639113..f181dfe 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h
@@ -20,25 +20,62 @@
#include <cstdio>
#include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
+#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/trie_map.h"
#include "utils/byte_array_view.h"
+#include "utils/int_array_view.h"
namespace latinime {
+/**
+ * Class representing language model.
+ *
+ * This class provides methods to get and store unigram/n-gram probability information and flags.
+ */
class LanguageModelDictContent {
public:
LanguageModelDictContent(const ReadWriteByteArrayView trieMapBuffer,
const bool hasHistoricalInfo)
- : mTrieMap(trieMapBuffer) {}
+ : mTrieMap(trieMapBuffer), mHasHistoricalInfo(hasHistoricalInfo) {}
- explicit LanguageModelDictContent(const bool hasHistoricalInfo) : mTrieMap() {}
+ explicit LanguageModelDictContent(const bool hasHistoricalInfo)
+ : mTrieMap(), mHasHistoricalInfo(hasHistoricalInfo) {}
+
+ bool isNearSizeLimit() const {
+ return mTrieMap.isNearSizeLimit();
+ }
bool save(FILE *const file) const;
+ bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ const LanguageModelDictContent *const originalContent,
+ int *const outNgramCount);
+
+ ProbabilityEntry getProbabilityEntry(const int wordId) const {
+ return getNgramProbabilityEntry(WordIdArrayView(), wordId);
+ }
+
+ bool setProbabilityEntry(const int wordId, const ProbabilityEntry *const probabilityEntry) {
+ return setNgramProbabilityEntry(WordIdArrayView(), wordId, probabilityEntry);
+ }
+
+ ProbabilityEntry getNgramProbabilityEntry(const WordIdArrayView prevWordIds,
+ const int wordId) const;
+
+ bool setNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId,
+ const ProbabilityEntry *const probabilityEntry);
+
private:
DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent);
TrieMap mTrieMap;
+ const bool mHasHistoricalInfo;
+
+ bool runGCInner(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ const TrieMap::TrieMapRange trieMapRange, const int nextLevelBitmapEntryIndex,
+ int *const outNgramCount);
};
} // namespace latinime
#endif /* LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp
deleted file mode 100644
index 2425b3b..0000000
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h"
-
-#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
-#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
-#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
-
-namespace latinime {
-
-const ProbabilityEntry ProbabilityDictContent::getProbabilityEntry(const int terminalId) const {
- if (terminalId < 0 || terminalId >= mSize) {
- // This method can be called with invalid terminal id during GC.
- return ProbabilityEntry(0 /* flags */, NOT_A_PROBABILITY);
- }
- const BufferWithExtendableBuffer *const buffer = getBuffer();
- int entryPos = getEntryPos(terminalId);
- const int flags = buffer->readUintAndAdvancePosition(
- Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, &entryPos);
- const int probability = buffer->readUintAndAdvancePosition(
- Ver4DictConstants::PROBABILITY_SIZE, &entryPos);
- if (mHasHistoricalInfo) {
- const int timestamp = buffer->readUintAndAdvancePosition(
- Ver4DictConstants::TIME_STAMP_FIELD_SIZE, &entryPos);
- const int level = buffer->readUintAndAdvancePosition(
- Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, &entryPos);
- const int count = buffer->readUintAndAdvancePosition(
- Ver4DictConstants::WORD_COUNT_FIELD_SIZE, &entryPos);
- const HistoricalInfo historicalInfo(timestamp, level, count);
- return ProbabilityEntry(flags, probability, &historicalInfo);
- } else {
- return ProbabilityEntry(flags, probability);
- }
-}
-
-bool ProbabilityDictContent::setProbabilityEntry(const int terminalId,
- const ProbabilityEntry *const probabilityEntry) {
- if (terminalId < 0) {
- return false;
- }
- const int entryPos = getEntryPos(terminalId);
- if (terminalId >= mSize) {
- ProbabilityEntry dummyEntry;
- // Write new entry.
- int writingPos = getBuffer()->getTailPosition();
- while (writingPos <= entryPos) {
- // Fulfilling with dummy entries until writingPos.
- if (!writeEntry(&dummyEntry, writingPos)) {
- AKLOGE("Cannot write dummy entry. pos: %d, mSize: %d", writingPos, mSize);
- return false;
- }
- writingPos += getEntrySize();
- mSize++;
- }
- }
- return writeEntry(probabilityEntry, entryPos);
-}
-
-bool ProbabilityDictContent::flushToFile(FILE *const file) const {
- if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
- ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo);
- for (int i = 0; i < mSize; ++i) {
- const ProbabilityEntry probabilityEntry = getProbabilityEntry(i);
- if (!probabilityDictContentToWrite.setProbabilityEntry(i, &probabilityEntry)) {
- AKLOGE("Cannot set probability entry in flushToFile. terminalId: %d", i);
- return false;
- }
- }
- return probabilityDictContentToWrite.flush(file);
- } else {
- return flush(file);
- }
-}
-
-bool ProbabilityDictContent::runGC(
- const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
- const ProbabilityDictContent *const originalProbabilityDictContent) {
- mSize = 0;
- for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
- it != terminalIdMap->end(); ++it) {
- const ProbabilityEntry probabilityEntry =
- originalProbabilityDictContent->getProbabilityEntry(it->first);
- if (!setProbabilityEntry(it->second, &probabilityEntry)) {
- AKLOGE("Cannot set probability entry in runGC. terminalId: %d", it->second);
- return false;
- }
- mSize++;
- }
- return true;
-}
-
-int ProbabilityDictContent::getEntrySize() const {
- if (mHasHistoricalInfo) {
- return Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE
- + Ver4DictConstants::PROBABILITY_SIZE
- + Ver4DictConstants::TIME_STAMP_FIELD_SIZE
- + Ver4DictConstants::WORD_LEVEL_FIELD_SIZE
- + Ver4DictConstants::WORD_COUNT_FIELD_SIZE;
- } else {
- return Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE
- + Ver4DictConstants::PROBABILITY_SIZE;
- }
-}
-
-int ProbabilityDictContent::getEntryPos(const int terminalId) const {
- return terminalId * getEntrySize();
-}
-
-bool ProbabilityDictContent::writeEntry(const ProbabilityEntry *const probabilityEntry,
- const int entryPos) {
- BufferWithExtendableBuffer *const bufferToWrite = getWritableBuffer();
- int writingPos = entryPos;
- if (!bufferToWrite->writeUintAndAdvancePosition(probabilityEntry->getFlags(),
- Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, &writingPos)) {
- AKLOGE("Cannot write flags in probability dict content. pos: %d", writingPos);
- return false;
- }
- if (!bufferToWrite->writeUintAndAdvancePosition(probabilityEntry->getProbability(),
- Ver4DictConstants::PROBABILITY_SIZE, &writingPos)) {
- AKLOGE("Cannot write probability in probability dict content. pos: %d", writingPos);
- return false;
- }
- if (mHasHistoricalInfo) {
- const HistoricalInfo *const historicalInfo = probabilityEntry->getHistoricalInfo();
- if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(),
- Ver4DictConstants::TIME_STAMP_FIELD_SIZE, &writingPos)) {
- AKLOGE("Cannot write timestamp in probability dict content. pos: %d", writingPos);
- return false;
- }
- if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getLevel(),
- Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, &writingPos)) {
- AKLOGE("Cannot write level in probability dict content. pos: %d", writingPos);
- return false;
- }
- if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getCount(),
- Ver4DictConstants::WORD_COUNT_FIELD_SIZE, &writingPos)) {
- AKLOGE("Cannot write count in probability dict content. pos: %d", writingPos);
- return false;
- }
- }
- return true;
-}
-
-} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
deleted file mode 100644
index 80e992c..0000000
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (C) 2013, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_PROBABILITY_DICT_CONTENT_H
-#define LATINIME_PROBABILITY_DICT_CONTENT_H
-
-#include <cstdint>
-#include <cstdio>
-
-#include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
-#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
-#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
-
-namespace latinime {
-
-class ProbabilityEntry;
-
-class ProbabilityDictContent : public SingleDictContent {
- public:
- ProbabilityDictContent(uint8_t *const buffer, const int bufferSize,
- const bool hasHistoricalInfo)
- : SingleDictContent(buffer, bufferSize),
- mHasHistoricalInfo(hasHistoricalInfo),
- mSize(getBuffer()->getTailPosition() / getEntrySize()) {}
-
- ProbabilityDictContent(const bool hasHistoricalInfo)
- : mHasHistoricalInfo(hasHistoricalInfo), mSize(0) {}
-
- const ProbabilityEntry getProbabilityEntry(const int terminalId) const;
-
- bool setProbabilityEntry(const int terminalId, const ProbabilityEntry *const probabilityEntry);
-
- bool flushToFile(FILE *const file) const;
-
- bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
- const ProbabilityDictContent *const originalProbabilityDictContent);
-
- private:
- DISALLOW_COPY_AND_ASSIGN(ProbabilityDictContent);
-
- int getEntrySize() const;
-
- int getEntryPos(const int terminalId) const;
-
- bool writeEntry(const ProbabilityEntry *const probabilityEntry, const int entryPos);
-
- bool mHasHistoricalInfo;
- int mSize;
-};
-} // namespace latinime
-#endif /* LATINIME_PROBABILITY_DICT_CONTENT_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h
index 36ba82b..feff6b5 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h
@@ -17,6 +17,9 @@
#ifndef LATINIME_PROBABILITY_ENTRY_H
#define LATINIME_PROBABILITY_ENTRY_H
+#include <climits>
+#include <cstdint>
+
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/historical_info.h"
@@ -67,6 +70,50 @@
return &mHistoricalInfo;
}
+ uint64_t encode(const bool hasHistoricalInfo) const {
+ uint64_t encodedEntry = static_cast<uint64_t>(mFlags);
+ if (hasHistoricalInfo) {
+ encodedEntry = (encodedEntry << (Ver4DictConstants::TIME_STAMP_FIELD_SIZE * CHAR_BIT))
+ ^ static_cast<uint64_t>(mHistoricalInfo.getTimeStamp());
+ encodedEntry = (encodedEntry << (Ver4DictConstants::WORD_LEVEL_FIELD_SIZE * CHAR_BIT))
+ ^ static_cast<uint64_t>(mHistoricalInfo.getLevel());
+ encodedEntry = (encodedEntry << (Ver4DictConstants::WORD_COUNT_FIELD_SIZE * CHAR_BIT))
+ ^ static_cast<uint64_t>(mHistoricalInfo.getCount());
+ } else {
+ encodedEntry = (encodedEntry << (Ver4DictConstants::PROBABILITY_SIZE * CHAR_BIT))
+ ^ static_cast<uint64_t>(mProbability);
+ }
+ return encodedEntry;
+ }
+
+ static ProbabilityEntry decode(const uint64_t encodedEntry, const bool hasHistoricalInfo) {
+ if (hasHistoricalInfo) {
+ const int flags = readFromEncodedEntry(encodedEntry,
+ Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE,
+ Ver4DictConstants::TIME_STAMP_FIELD_SIZE
+ + Ver4DictConstants::WORD_LEVEL_FIELD_SIZE
+ + Ver4DictConstants::WORD_COUNT_FIELD_SIZE);
+ const int timestamp = readFromEncodedEntry(encodedEntry,
+ Ver4DictConstants::TIME_STAMP_FIELD_SIZE,
+ Ver4DictConstants::WORD_LEVEL_FIELD_SIZE
+ + Ver4DictConstants::WORD_COUNT_FIELD_SIZE);
+ const int level = readFromEncodedEntry(encodedEntry,
+ Ver4DictConstants::WORD_LEVEL_FIELD_SIZE,
+ Ver4DictConstants::WORD_COUNT_FIELD_SIZE);
+ const int count = readFromEncodedEntry(encodedEntry,
+ Ver4DictConstants::WORD_COUNT_FIELD_SIZE, 0 /* pos */);
+ const HistoricalInfo historicalInfo(timestamp, level, count);
+ return ProbabilityEntry(flags, NOT_A_PROBABILITY, &historicalInfo);
+ } else {
+ const int flags = readFromEncodedEntry(encodedEntry,
+ Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE,
+ Ver4DictConstants::PROBABILITY_SIZE);
+ const int probability = readFromEncodedEntry(encodedEntry,
+ Ver4DictConstants::PROBABILITY_SIZE, 0 /* pos */);
+ return ProbabilityEntry(flags, probability);
+ }
+ }
+
private:
// Copy constructor is public to use this class as a type of return value.
DISALLOW_ASSIGNMENT_OPERATOR(ProbabilityEntry);
@@ -74,6 +121,11 @@
const int mFlags;
const int mProbability;
const HistoricalInfo mHistoricalInfo;
+
+ static int readFromEncodedEntry(const uint64_t encodedEntry, const int size, const int pos) {
+ return static_cast<int>(
+ (encodedEntry >> (pos * CHAR_BIT)) & ((1ull << (size * CHAR_BIT)) - 1));
+ }
};
} // namespace latinime
#endif /* LATINIME_PROBABILITY_ENTRY_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
index 125ae17..3c8008d 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
@@ -157,11 +157,6 @@
AKLOGE("Terminal position lookup table cannot be written.");
return false;
}
- // Write probability dict content.
- if (!mProbabilityDictContent.flushToFile(file)) {
- AKLOGE("Probability dict content cannot be written.");
- return false;
- }
// Write language model content.
if (!mLanguageModelDictContent.save(file)) {
AKLOGE("Language model dict content cannot be written.");
@@ -196,10 +191,6 @@
contentBuffers[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX],
contentBufferSizes[
Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX]),
- mProbabilityDictContent(
- contentBuffers[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],
- contentBufferSizes[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],
- mHeaderPolicy.hasHistoricalInfoOfWords()),
mLanguageModelDictContent(
ReadWriteByteArrayView(
contentBuffers[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX],
@@ -216,7 +207,6 @@
: mHeaderBuffer(nullptr), mDictBuffer(nullptr), mHeaderPolicy(headerPolicy),
mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableTrieBuffer(maxTrieSize), mTerminalPositionLookupTable(),
- mProbabilityDictContent(headerPolicy->hasHistoricalInfoOfWords()),
mLanguageModelDictContent(headerPolicy->hasHistoricalInfoOfWords()),
mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(),
mIsUpdatable(true) {}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
index 36671bc..68027dc 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
@@ -24,7 +24,6 @@
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
@@ -53,7 +52,7 @@
AK_FORCE_INLINE bool isNearSizeLimit() const {
return mExpandableTrieBuffer.isNearSizeLimit()
|| mTerminalPositionLookupTable.isNearSizeLimit()
- || mProbabilityDictContent.isNearSizeLimit()
+ || mLanguageModelDictContent.isNearSizeLimit()
|| mBigramDictContent.isNearSizeLimit()
|| mShortcutDictContent.isNearSizeLimit();
}
@@ -82,12 +81,12 @@
return &mTerminalPositionLookupTable;
}
- AK_FORCE_INLINE ProbabilityDictContent *getMutableProbabilityDictContent() {
- return &mProbabilityDictContent;
+ AK_FORCE_INLINE LanguageModelDictContent *getMutableLanguageModelDictContent() {
+ return &mLanguageModelDictContent;
}
- AK_FORCE_INLINE const ProbabilityDictContent *getProbabilityDictContent() const {
- return &mProbabilityDictContent;
+ AK_FORCE_INLINE const LanguageModelDictContent *getLanguageModelDictContent() const {
+ return &mLanguageModelDictContent;
}
AK_FORCE_INLINE BigramDictContent *getMutableBigramDictContent() {
@@ -136,7 +135,6 @@
BufferWithExtendableBuffer mExpandableHeaderBuffer;
BufferWithExtendableBuffer mExpandableTrieBuffer;
TerminalPositionLookupTable mTerminalPositionLookupTable;
- ProbabilityDictContent mProbabilityDictContent;
LanguageModelDictContent mLanguageModelDictContent;
BigramDictContent mBigramDictContent;
ShortcutDictContent mShortcutDictContent;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
index e7e31e9..93d4e56 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
@@ -27,19 +27,18 @@
// limited to 1MB to prevent from inefficient traversing.
const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;
-// NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT for Trie, TerminalAddressLookupTable and Probability.
+// NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT for Trie and TerminalAddressLookupTable.
+// NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT for language model.
// NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut.
const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE =
- NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 3
+ NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 2
+ NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT
+ NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2;
const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0;
const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX =
TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
-const int Ver4DictConstants::PROBABILITY_BUFFER_INDEX =
- TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX =
- PROBABILITY_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
+ TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX =
LANGUAGE_MODEL_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT;
const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX =
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
index e75db9f..6950ca7 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
@@ -35,7 +35,6 @@
static const size_t NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE;
static const int TRIE_BUFFER_INDEX;
static const int TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX;
- static const int PROBABILITY_BUFFER_INDEX;
static const int LANGUAGE_MODEL_BUFFER_INDEX;
static const int BIGRAM_BUFFERS_INDEX;
static const int SHORTCUT_BUFFERS_INDEX;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
index 0a435e9..731092e 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
@@ -18,7 +18,7 @@
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
@@ -61,8 +61,9 @@
terminalIdFieldPos += mBuffer->getOriginalBufferSize();
}
terminalId = Ver4PatriciaTrieReadingUtils::getTerminalIdAndAdvancePosition(dictBuf, &pos);
+ // TODO: Quit reading probability here.
const ProbabilityEntry probabilityEntry =
- mProbabilityDictContent->getProbabilityEntry(terminalId);
+ mLanguageModelDictContent->getProbabilityEntry(terminalId);
if (probabilityEntry.hasHistoricalInfo()) {
probability = ForgettingCurveUtils::decodeProbability(
probabilityEntry.getHistoricalInfo(), mHeaderPolicy);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h
index 22ed4a6..a91ad57 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h
@@ -25,18 +25,18 @@
class BufferWithExtendableBuffer;
class HeaderPolicy;
-class ProbabilityDictContent;
+class LanguageModelDictContent;
/*
* This class is used for helping to read nodes of ver4 patricia trie. This class handles moved
- * node and reads node attributes including probability form probabilityBuffer.
+ * node and reads node attributes including probability form language model.
*/
class Ver4PatriciaTrieNodeReader : public PtNodeReader {
public:
Ver4PatriciaTrieNodeReader(const BufferWithExtendableBuffer *const buffer,
- const ProbabilityDictContent *const probabilityDictContent,
+ const LanguageModelDictContent *const languageModelDictContent,
const HeaderPolicy *const headerPolicy)
- : mBuffer(buffer), mProbabilityDictContent(probabilityDictContent),
+ : mBuffer(buffer), mLanguageModelDictContent(languageModelDictContent),
mHeaderPolicy(headerPolicy) {}
~Ver4PatriciaTrieNodeReader() {}
@@ -50,7 +50,7 @@
DISALLOW_COPY_AND_ASSIGN(Ver4PatriciaTrieNodeReader);
const BufferWithExtendableBuffer *const mBuffer;
- const ProbabilityDictContent *const mProbabilityDictContent;
+ const LanguageModelDictContent *const mLanguageModelDictContent;
const HeaderPolicy *const mHeaderPolicy;
const PtNodeParams fetchPtNodeInfoFromBufferAndProcessMovedPtNode(const int ptNodePos,
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
index 3d8da91..1a311b1 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
@@ -143,11 +143,11 @@
return false;
}
const ProbabilityEntry originalProbabilityEntry =
- mBuffers->getProbabilityDictContent()->getProbabilityEntry(
+ mBuffers->getLanguageModelDictContent()->getProbabilityEntry(
toBeUpdatedPtNodeParams->getTerminalId());
const ProbabilityEntry probabilityEntry = createUpdatedEntryFrom(&originalProbabilityEntry,
unigramProperty);
- return mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry(
+ return mBuffers->getMutableLanguageModelDictContent()->setProbabilityEntry(
toBeUpdatedPtNodeParams->getTerminalId(), &probabilityEntry);
}
@@ -158,14 +158,14 @@
return false;
}
const ProbabilityEntry originalProbabilityEntry =
- mBuffers->getProbabilityDictContent()->getProbabilityEntry(
+ mBuffers->getLanguageModelDictContent()->getProbabilityEntry(
toBeUpdatedPtNodeParams->getTerminalId());
if (originalProbabilityEntry.hasHistoricalInfo()) {
const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave(
originalProbabilityEntry.getHistoricalInfo(), mHeaderPolicy);
const ProbabilityEntry probabilityEntry =
originalProbabilityEntry.createEntryWithUpdatedHistoricalInfo(&historicalInfo);
- if (!mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry(
+ if (!mBuffers->getMutableLanguageModelDictContent()->setProbabilityEntry(
toBeUpdatedPtNodeParams->getTerminalId(), &probabilityEntry)) {
AKLOGE("Cannot write updated probability entry. terminalId: %d",
toBeUpdatedPtNodeParams->getTerminalId());
@@ -218,8 +218,8 @@
ProbabilityEntry newProbabilityEntry;
const ProbabilityEntry probabilityEntryToWrite = createUpdatedEntryFrom(
&newProbabilityEntry, unigramProperty);
- return mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry(terminalId,
- &probabilityEntryToWrite);
+ return mBuffers->getMutableLanguageModelDictContent()->setProbabilityEntry(
+ terminalId, &probabilityEntryToWrite);
}
bool Ver4PatriciaTrieNodeWriter::addNewBigramEntry(
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
index 4bf8050..2b92d5b 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
@@ -452,7 +452,7 @@
std::vector<int> codePointVector(ptNodeParams.getCodePoints(),
ptNodeParams.getCodePoints() + ptNodeParams.getCodePointCount());
const ProbabilityEntry probabilityEntry =
- mBuffers->getProbabilityDictContent()->getProbabilityEntry(
+ mBuffers->getLanguageModelDictContent()->getProbabilityEntry(
ptNodeParams.getTerminalId());
const HistoricalInfo *const historicalInfo = probabilityEntry.getHistoricalInfo();
// Fetch bigram information.
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
index 76b3404..faad429 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
@@ -46,7 +46,7 @@
mBuffers->getTerminalPositionLookupTable(), mHeaderPolicy),
mShortcutPolicy(mBuffers->getMutableShortcutDictContent(),
mBuffers->getTerminalPositionLookupTable()),
- mNodeReader(mDictBuffer, mBuffers->getProbabilityDictContent(), mHeaderPolicy),
+ mNodeReader(mDictBuffer, mBuffers->getLanguageModelDictContent(), mHeaderPolicy),
mPtNodeArrayReader(mDictBuffer),
mNodeWriter(mDictBuffer, mBuffers.get(), mHeaderPolicy, &mNodeReader,
&mPtNodeArrayReader, &mBigramPolicy, &mShortcutPolicy),
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
index 0e658f8..4220312 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
@@ -75,7 +75,7 @@
const HeaderPolicy *const headerPolicy, Ver4DictBuffers *const buffersToWrite,
int *const outUnigramCount, int *const outBigramCount) {
Ver4PatriciaTrieNodeReader ptNodeReader(mBuffers->getTrieBuffer(),
- mBuffers->getProbabilityDictContent(), headerPolicy);
+ mBuffers->getLanguageModelDictContent(), headerPolicy);
Ver4PtNodeArrayReader ptNodeArrayReader(mBuffers->getTrieBuffer());
Ver4BigramListPolicy bigramPolicy(mBuffers->getMutableBigramDictContent(),
mBuffers->getTerminalPositionLookupTable(), headerPolicy);
@@ -138,7 +138,7 @@
// Create policy instances for the GCed dictionary.
Ver4PatriciaTrieNodeReader newPtNodeReader(buffersToWrite->getTrieBuffer(),
- buffersToWrite->getProbabilityDictContent(), headerPolicy);
+ buffersToWrite->getLanguageModelDictContent(), headerPolicy);
Ver4PtNodeArrayReader newPtNodeArrayreader(buffersToWrite->getTrieBuffer());
Ver4BigramListPolicy newBigramPolicy(buffersToWrite->getMutableBigramDictContent(),
buffersToWrite->getTerminalPositionLookupTable(), headerPolicy);
@@ -154,8 +154,8 @@
return false;
}
// Run GC for probability dict content.
- if (!buffersToWrite->getMutableProbabilityDictContent()->runGC(&terminalIdMap,
- mBuffers->getProbabilityDictContent())) {
+ if (!buffersToWrite->getMutableLanguageModelDictContent()->runGC(&terminalIdMap,
+ mBuffers->getLanguageModelDictContent(), nullptr /* outNgramCount */)) {
return false;
}
// Run GC for bigram dict content.
@@ -201,7 +201,7 @@
continue;
}
const ProbabilityEntry probabilityEntry =
- mBuffers->getProbabilityDictContent()->getProbabilityEntry(i);
+ mBuffers->getLanguageModelDictContent()->getProbabilityEntry(i);
const int probability = probabilityEntry.hasHistoricalInfo() ?
ForgettingCurveUtils::decodeProbability(
probabilityEntry.getHistoricalInfo(), mBuffers->getHeaderPolicy()) :
diff --git a/native/jni/src/utils/int_array_view.h b/native/jni/src/utils/int_array_view.h
index 3ff01f5..4bc2487 100644
--- a/native/jni/src/utils/int_array_view.h
+++ b/native/jni/src/utils/int_array_view.h
@@ -61,6 +61,10 @@
return mPtr[index];
}
+ AK_FORCE_INLINE bool empty() const {
+ return size() == 0;
+ }
+
AK_FORCE_INLINE size_t size() const {
return mSize;
}
@@ -76,5 +80,7 @@
const size_t mSize;
};
+using WordIdArrayView = IntArrayView;
+
} // namespace latinime
#endif // LATINIME_MEMORY_VIEW_H
diff --git a/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp
new file mode 100644
index 0000000..6eef204
--- /dev/null
+++ b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h"
+
+#include <gtest/gtest.h>
+
+#include "utils/int_array_view.h"
+
+namespace latinime {
+namespace {
+
+TEST(LanguageModelDictContentTest, TestUnigramProbability) {
+ LanguageModelDictContent LanguageModelDictContent(false /* useHistoricalInfo */);
+
+ const int flag = 0xFF;
+ const int probability = 10;
+ const int wordId = 100;
+ const ProbabilityEntry probabilityEntry(flag, probability);
+ LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry);
+ const ProbabilityEntry entry =
+ LanguageModelDictContent.getProbabilityEntry(wordId);
+ EXPECT_EQ(flag, entry.getFlags());
+ EXPECT_EQ(probability, entry.getProbability());
+}
+
+TEST(LanguageModelDictContentTest, TestUnigramProbabilityWithHistoricalInfo) {
+ LanguageModelDictContent LanguageModelDictContent(true /* useHistoricalInfo */);
+
+ const int flag = 0xF0;
+ const int timestamp = 0x3FFFFFFF;
+ const int level = 3;
+ const int count = 10;
+ const int wordId = 100;
+ const HistoricalInfo historicalInfo(timestamp, level, count);
+ const ProbabilityEntry probabilityEntry(flag, NOT_A_PROBABILITY, &historicalInfo);
+ LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry);
+ const ProbabilityEntry entry = LanguageModelDictContent.getProbabilityEntry(wordId);
+ EXPECT_EQ(flag, entry.getFlags());
+ EXPECT_EQ(timestamp, entry.getHistoricalInfo()->getTimeStamp());
+ EXPECT_EQ(level, entry.getHistoricalInfo()->getLevel());
+ EXPECT_EQ(count, entry.getHistoricalInfo()->getCount());
+}
+
+} // namespace
+} // namespace latinime
diff --git a/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/probability_entry_test.cpp b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/probability_entry_test.cpp
new file mode 100644
index 0000000..db94550
--- /dev/null
+++ b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/probability_entry_test.cpp
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
+
+#include <gtest/gtest.h>
+
+#include "defines.h"
+
+namespace latinime {
+namespace {
+
+TEST(ProbabilityEntryTest, TestEncodeDecode) {
+ const int flag = 0xFF;
+ const int probability = 10;
+
+ const ProbabilityEntry entry(flag, probability);
+ const uint64_t encodedEntry = entry.encode(false /* hasHistoricalInfo */);
+ const ProbabilityEntry decodedEntry =
+ ProbabilityEntry::decode(encodedEntry, false /* hasHistoricalInfo */);
+ EXPECT_EQ(0xFF0Aull, encodedEntry);
+ EXPECT_EQ(flag, decodedEntry.getFlags());
+ EXPECT_EQ(probability, decodedEntry.getProbability());
+}
+
+TEST(ProbabilityEntryTest, TestEncodeDecodeWithHistoricalInfo) {
+ const int flag = 0xF0;
+ const int timestamp = 0x3FFFFFFF;
+ const int level = 3;
+ const int count = 10;
+
+ const HistoricalInfo historicalInfo(timestamp, level, count);
+ const ProbabilityEntry entry(flag, NOT_A_PROBABILITY, &historicalInfo);
+
+ const uint64_t encodedEntry = entry.encode(true /* hasHistoricalInfo */);
+ EXPECT_EQ(0xF03FFFFFFF030Aull, encodedEntry);
+ const ProbabilityEntry decodedEntry =
+ ProbabilityEntry::decode(encodedEntry, true /* hasHistoricalInfo */);
+
+ EXPECT_EQ(flag, decodedEntry.getFlags());
+ EXPECT_EQ(timestamp, decodedEntry.getHistoricalInfo()->getTimeStamp());
+ EXPECT_EQ(level, decodedEntry.getHistoricalInfo()->getLevel());
+ EXPECT_EQ(count, decodedEntry.getHistoricalInfo()->getCount());
+}
+
+} // namespace
+} // namespace latinime