Merge "Prepare ExpandableBinaryDictionary to make it updatable"
diff --git a/java/res/values-th/donottranslate.xml b/java/res/values-th/donottranslate.xml
new file mode 100644
index 0000000..aeeebed
--- /dev/null
+++ b/java/res/values-th/donottranslate.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+/*
+**
+** Copyright 2013, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+-->
+<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <!-- Whether this language uses spaces -->
+ <bool name="current_language_has_spaces">false</bool>
+</resources>
diff --git a/java/res/values/donottranslate.xml b/java/res/values/donottranslate.xml
index e352f08..8983536 100644
--- a/java/res/values/donottranslate.xml
+++ b/java/res/values/donottranslate.xml
@@ -18,6 +18,8 @@
*/
-->
<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <!-- TODO: these settings depend on the language. They should be put either in the dictionary
+ header, or in the subtype maybe? -->
<!-- Symbols that are suggested between words -->
<string name="suggested_punctuations">!,?,\\,,:,;,\",(,),\',-,/,@,_</string>
<!-- Symbols that are normally preceded by a space (used to add an auto-space before these) -->
@@ -29,6 +31,8 @@
<string name="symbols_word_separators">"	 \n"()[]{}*&<>+=|.,;:!?/_\"</string>
<!-- Word connectors -->
<string name="symbols_word_connectors">\'-</string>
+ <!-- Whether this language uses spaces -->
+ <bool name="current_language_has_spaces">true</bool>
<!-- Always show the suggestion strip -->
<string name="prefs_suggestion_visibility_show_value">0</string>
diff --git a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java
index 77f3234..8ce1263 100644
--- a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java
+++ b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java
@@ -172,7 +172,7 @@
return null;
}
final String keyDescription = getKeyDescription(key);
- final Rect boundsInParent = key.mHitBox;
+ final Rect boundsInParent = key.getHitBox();
// Calculate the key's in-screen bounds.
mTempBoundsInScreen.set(boundsInParent);
@@ -208,8 +208,8 @@
* @param key The key to press.
*/
void simulateKeyPress(final Key key) {
- final int x = key.mHitBox.centerX();
- final int y = key.mHitBox.centerY();
+ final int x = key.getHitBox().centerX();
+ final int y = key.getHitBox().centerY();
final long downTime = SystemClock.uptimeMillis();
final MotionEvent downEvent = MotionEvent.obtain(
downTime, downTime, MotionEvent.ACTION_DOWN, x, y, 0);
@@ -325,6 +325,6 @@
// The key x- and y-coordinates are stable between layout changes.
// Generate an identifier by bit-shifting the x-coordinate to the
// left-half of the integer and OR'ing with the y-coordinate.
- return ((0xFFFF & key.mX) << (Integer.SIZE / 2)) | (0xFFFF & key.mY);
+ return ((0xFFFF & key.getX()) << (Integer.SIZE / 2)) | (0xFFFF & key.getY());
}
}
diff --git a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java
index 41f5b9a..58624a2 100644
--- a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java
+++ b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java
@@ -97,7 +97,7 @@
*/
public String getDescriptionForKey(final Context context, final Keyboard keyboard,
final Key key, final boolean shouldObscure) {
- final int code = key.mCode;
+ final int code = key.getCode();
if (code == Constants.CODE_SWITCH_ALPHA_SYMBOL) {
final String description = getDescriptionForSwitchAlphaSymbol(context, keyboard);
@@ -116,8 +116,8 @@
return getDescriptionForActionKey(context, keyboard, key);
}
- if (!TextUtils.isEmpty(key.mLabel)) {
- final String label = key.mLabel.toString().trim();
+ if (!TextUtils.isEmpty(key.getLabel())) {
+ final String label = key.getLabel().trim();
// First, attempt to map the label to a pre-defined description.
if (mKeyLabelMap.containsKey(label)) {
@@ -126,7 +126,7 @@
}
// Just attempt to speak the description.
- if (key.mCode != Constants.CODE_UNSPECIFIED) {
+ if (key.getCode() != Constants.CODE_UNSPECIFIED) {
return getDescriptionForKeyCode(context, keyboard, key, shouldObscure);
}
return null;
@@ -215,8 +215,8 @@
final int resId;
// Always use the label, if available.
- if (!TextUtils.isEmpty(key.mLabel)) {
- return key.mLabel.toString().trim();
+ if (!TextUtils.isEmpty(key.getLabel())) {
+ return key.getLabel().trim();
}
// Otherwise, use the action ID.
@@ -267,7 +267,7 @@
*/
private String getDescriptionForKeyCode(final Context context, final Keyboard keyboard,
final Key key, final boolean shouldObscure) {
- final int code = key.mCode;
+ final int code = key.getCode();
// If the key description should be obscured, now is the time to do it.
final boolean isDefinedNonCtrl = Character.isDefined(code) && !Character.isISOControl(code);
@@ -280,8 +280,8 @@
if (isDefinedNonCtrl) {
return Character.toString((char) code);
}
- if (!TextUtils.isEmpty(key.mLabel)) {
- return key.mLabel;
+ if (!TextUtils.isEmpty(key.getLabel())) {
+ return key.getLabel();
}
return context.getString(R.string.spoken_description_unknown, code);
}
diff --git a/java/src/com/android/inputmethod/keyboard/Key.java b/java/src/com/android/inputmethod/keyboard/Key.java
index 16c79eb..143c6e8 100644
--- a/java/src/com/android/inputmethod/keyboard/Key.java
+++ b/java/src/com/android/inputmethod/keyboard/Key.java
@@ -58,12 +58,12 @@
/**
* The key code (unicode or custom code) that this key generates.
*/
- public final int mCode;
+ private final int mCode;
/** Label to display */
- public final String mLabel;
+ private final String mLabel;
/** Hint label to display on the key in conjunction with the label */
- public final String mHintLabel;
+ private final String mHintLabel;
/** Flags of the label */
private final int mLabelFlags;
private static final int LABEL_FLAGS_ALIGN_LEFT = 0x01;
@@ -95,18 +95,18 @@
private final int mIconId;
/** Width of the key, not including the gap */
- public final int mWidth;
+ private final int mWidth;
/** Height of the key, not including the gap */
- public final int mHeight;
+ private final int mHeight;
/** X coordinate of the key in the keyboard layout */
- public final int mX;
+ private final int mX;
/** Y coordinate of the key in the keyboard layout */
- public final int mY;
+ private final int mY;
/** Hit bounding box of the key */
- public final Rect mHitBox = new Rect();
+ private final Rect mHitBox = new Rect();
/** More keys. It is guaranteed that this is null or an array of one or more elements */
- public final MoreKeySpec[] mMoreKeys;
+ private final MoreKeySpec[] mMoreKeys;
/** More keys column number and flags */
private final int mMoreKeysColumnAndFlags;
private static final int MORE_KEYS_COLUMN_MASK = 0x000000ff;
@@ -121,7 +121,7 @@
private static final String MORE_KEYS_NO_PANEL_AUTO_MORE_KEY = "!noPanelAutoMoreKey!";
/** Background type that represents different key background visual than normal one. */
- public final int mBackgroundType;
+ private final int mBackgroundType;
public static final int BACKGROUND_TYPE_EMPTY = 0;
public static final int BACKGROUND_TYPE_NORMAL = 1;
public static final int BACKGROUND_TYPE_FUNCTIONAL = 2;
@@ -135,7 +135,7 @@
private static final int ACTION_FLAGS_ALT_CODE_WHILE_TYPING = 0x04;
private static final int ACTION_FLAGS_ENABLE_LONG_PRESS = 0x08;
- public final KeyVisualAttributes mKeyVisualAttributes;
+ private final KeyVisualAttributes mKeyVisualAttributes;
private final OptionalAttributes mOptionalAttributes;
@@ -151,7 +151,7 @@
public final int mVisualInsetsLeft;
public final int mVisualInsetsRight;
- public OptionalAttributes(final String outputText, final int altCode,
+ private OptionalAttributes(final String outputText, final int altCode,
final int disabledIconId, final int previewIconId,
final int visualInsetsLeft, final int visualInsetsRight) {
mOutputText = outputText;
@@ -161,6 +161,18 @@
mVisualInsetsLeft = visualInsetsLeft;
mVisualInsetsRight = visualInsetsRight;
}
+
+ public static OptionalAttributes newInstance(final String outputText, final int altCode,
+ final int disabledIconId, final int previewIconId,
+ final int visualInsetsLeft, final int visualInsetsRight) {
+ if (outputText == null && altCode == CODE_UNSPECIFIED
+ && disabledIconId == ICON_UNDEFINED && previewIconId == ICON_UNDEFINED
+ && visualInsetsLeft == 0 && visualInsetsRight == 0) {
+ return null;
+ }
+ return new OptionalAttributes(outputText, altCode, disabledIconId, previewIconId,
+ visualInsetsLeft, visualInsetsRight);
+ }
}
private final int mHashCode;
@@ -194,12 +206,9 @@
mMoreKeys = null;
mMoreKeysColumnAndFlags = 0;
mLabel = label;
- if (outputText == null) {
- mOptionalAttributes = null;
- } else {
- mOptionalAttributes = new OptionalAttributes(outputText, CODE_UNSPECIFIED,
- ICON_UNDEFINED, ICON_UNDEFINED, 0, 0);
- }
+ mOptionalAttributes = OptionalAttributes.newInstance(outputText, CODE_UNSPECIFIED,
+ ICON_UNDEFINED, ICON_UNDEFINED,
+ 0 /* visualInsetsLeft */, 0 /* visualInsetsRight */);
mCode = code;
mEnabled = (code != CODE_UNSPECIFIED);
mIconId = iconId;
@@ -225,7 +234,7 @@
public Key(final Resources res, final KeyboardParams params, final KeyboardRow row,
final XmlPullParser parser) throws XmlPullParserException {
final float horizontalGap = isSpacer() ? 0 : params.mHorizontalGap;
- final int rowHeight = row.mRowHeight;
+ final int rowHeight = row.getRowHeight();
mHeight = rowHeight - params.mVerticalGap;
final TypedArray keyAttr = res.obtainAttributes(Xml.asAttributeSet(parser),
@@ -260,11 +269,11 @@
final int previewIconId = KeySpecParser.getIconId(style.getString(keyAttr,
R.styleable.Keyboard_Key_keyIconPreview));
- mLabelFlags = style.getFlag(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags)
+ mLabelFlags = style.getFlags(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags)
| row.getDefaultKeyLabelFlags();
final boolean needsToUpperCase = needsToUpperCase(mLabelFlags, params.mId.mElementId);
final Locale locale = params.mId.mLocale;
- int actionFlags = style.getFlag(keyAttr, R.styleable.Keyboard_Key_keyActionFlags);
+ int actionFlags = style.getFlags(keyAttr, R.styleable.Keyboard_Key_keyActionFlags);
String[] moreKeys = style.getStringArray(keyAttr, R.styleable.Keyboard_Key_moreKeys);
int moreKeysColumn = style.getInt(keyAttr,
@@ -360,15 +369,8 @@
KeySpecParser.parseCode(style.getString(keyAttr,
R.styleable.Keyboard_Key_altCode), params.mCodesSet, CODE_UNSPECIFIED),
needsToUpperCase, locale);
- if (outputText == null && altCode == CODE_UNSPECIFIED
- && disabledIconId == ICON_UNDEFINED && previewIconId == ICON_UNDEFINED
- && visualInsetsLeft == 0 && visualInsetsRight == 0) {
- mOptionalAttributes = null;
- } else {
- mOptionalAttributes = new OptionalAttributes(outputText, altCode,
- disabledIconId, previewIconId,
- visualInsetsLeft, visualInsetsRight);
- }
+ mOptionalAttributes = OptionalAttributes.newInstance(outputText, altCode,
+ disabledIconId, previewIconId, visualInsetsLeft, visualInsetsRight);
mKeyVisualAttributes = KeyVisualAttributes.newInstance(keyAttr);
keyAttr.recycle();
mHashCode = computeHashCode(this);
@@ -377,6 +379,35 @@
}
}
+ /**
+ * Copy constructor.
+ *
+ * @param key the original key.
+ */
+ protected Key(final Key key) {
+ // Final attributes.
+ mCode = key.mCode;
+ mLabel = key.mLabel;
+ mHintLabel = key.mHintLabel;
+ mLabelFlags = key.mLabelFlags;
+ mIconId = key.mIconId;
+ mWidth = key.mWidth;
+ mHeight = key.mHeight;
+ mX = key.mX;
+ mY = key.mY;
+ mHitBox.set(key.mHitBox);
+ mMoreKeys = key.mMoreKeys;
+ mMoreKeysColumnAndFlags = key.mMoreKeysColumnAndFlags;
+ mBackgroundType = key.mBackgroundType;
+ mActionFlags = key.mActionFlags;
+ mKeyVisualAttributes = key.mKeyVisualAttributes;
+ mOptionalAttributes = key.mOptionalAttributes;
+ mHashCode = key.mHashCode;
+ // Key state.
+ mPressed = key.mPressed;
+ mEnabled = key.mEnabled;
+ }
+
private static boolean needsToUpperCase(final int labelFlags, final int keyboardElementId) {
if ((labelFlags & LABEL_FLAGS_PRESERVE_CASE) != 0) return false;
switch (keyboardElementId) {
@@ -476,6 +507,22 @@
}
}
+ public int getCode() {
+ return mCode;
+ }
+
+ public String getLabel() {
+ return mLabel;
+ }
+
+ public String getHintLabel() {
+ return mHintLabel;
+ }
+
+ public MoreKeySpec[] getMoreKeys() {
+ return mMoreKeys;
+ }
+
public void markAsLeftEdge(final KeyboardParams params) {
mHitBox.left = params.mLeftPadding;
}
@@ -522,6 +569,10 @@
&& (mLabelFlags & LABEL_FLAGS_SHIFTED_LETTER_ACTIVATED) == 0;
}
+ public KeyVisualAttributes getVisualAttributes() {
+ return mKeyVisualAttributes;
+ }
+
public final Typeface selectTypeface(final KeyDrawParams params) {
// TODO: Handle "bold" here too?
if ((mLabelFlags & LABEL_FLAGS_FONT_NORMAL) != 0) {
@@ -696,6 +747,22 @@
? iconSet.getIconDrawable(previewIconId) : iconSet.getIconDrawable(mIconId);
}
+ public int getWidth() {
+ return mWidth;
+ }
+
+ public int getHeight() {
+ return mHeight;
+ }
+
+ public int getX() {
+ return mX;
+ }
+
+ public int getY() {
+ return mY;
+ }
+
public final int getDrawX() {
final OptionalAttributes attrs = mOptionalAttributes;
return (attrs == null) ? mX : mX + attrs.mVisualInsetsLeft;
@@ -733,6 +800,10 @@
mEnabled = enabled;
}
+ public Rect getHitBox() {
+ return mHitBox;
+ }
+
/**
* Detects if a point falls on this key.
* @param x the x-coordinate of the point
diff --git a/java/src/com/android/inputmethod/keyboard/KeyDetector.java b/java/src/com/android/inputmethod/keyboard/KeyDetector.java
index 17e707f..befb6fa 100644
--- a/java/src/com/android/inputmethod/keyboard/KeyDetector.java
+++ b/java/src/com/android/inputmethod/keyboard/KeyDetector.java
@@ -108,8 +108,9 @@
if (distance > minDistance) {
continue;
}
- // To take care of hitbox overlaps, we compare mCode here too.
- if (primaryKey == null || distance < minDistance || key.mCode > primaryKey.mCode) {
+ // To take care of hitbox overlaps, we compare key's code here too.
+ if (primaryKey == null || distance < minDistance
+ || key.getCode() > primaryKey.getCode()) {
minDistance = distance;
primaryKey = key;
}
@@ -118,7 +119,7 @@
}
public static String printableCode(Key key) {
- return key != null ? Constants.printableCode(key.mCode) : "none";
+ return key != null ? Constants.printableCode(key.getCode()) : "none";
}
public static String printableCodes(int[] codes) {
diff --git a/java/src/com/android/inputmethod/keyboard/Keyboard.java b/java/src/com/android/inputmethod/keyboard/Keyboard.java
index fefac96..c40ddea 100644
--- a/java/src/com/android/inputmethod/keyboard/Keyboard.java
+++ b/java/src/com/android/inputmethod/keyboard/Keyboard.java
@@ -131,7 +131,7 @@
}
for (final Key key : mKeys) {
- if (key.mCode == code) {
+ if (key.getCode() == code) {
mKeyCache.put(code, key);
return key;
}
@@ -148,7 +148,7 @@
for (final Key key : mKeys) {
if (key == aKey) {
- mKeyCache.put(key.mCode, key);
+ mKeyCache.put(key.getCode(), key);
return true;
}
}
diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java b/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java
index bccf8fb..711de63 100644
--- a/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java
+++ b/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java
@@ -162,7 +162,8 @@
final KeyboardId id = new KeyboardId(keyboardLayoutSetElementId, mParams);
try {
return getKeyboard(elementParams, id);
- } catch (RuntimeException e) {
+ } catch (final RuntimeException e) {
+ Log.e(TAG, "Can't create keyboard: " + id, e);
throw new KeyboardLayoutSetException(e, id);
}
}
diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardView.java b/java/src/com/android/inputmethod/keyboard/KeyboardView.java
index 28eb585..d049a86 100644
--- a/java/src/com/android/inputmethod/keyboard/KeyboardView.java
+++ b/java/src/com/android/inputmethod/keyboard/KeyboardView.java
@@ -265,9 +265,9 @@
mClipRegion.setEmpty();
for (final Key key : mInvalidatedKeys) {
if (mKeyboard.hasKey(key)) {
- final int x = key.mX + getPaddingLeft();
- final int y = key.mY + getPaddingTop();
- mWorkingRect.set(x, y, x + key.mWidth, y + key.mHeight);
+ final int x = key.getX() + getPaddingLeft();
+ final int y = key.getY() + getPaddingTop();
+ mWorkingRect.set(x, y, x + key.getWidth(), y + key.getHeight());
mClipRegion.union(mWorkingRect);
}
}
@@ -310,11 +310,11 @@
private void onDrawKey(final Key key, final Canvas canvas, final Paint paint) {
final int keyDrawX = key.getDrawX() + getPaddingLeft();
- final int keyDrawY = key.mY + getPaddingTop();
+ final int keyDrawY = key.getY() + getPaddingTop();
canvas.translate(keyDrawX, keyDrawY);
final int keyHeight = mKeyboard.mMostCommonKeyHeight - mKeyboard.mVerticalGap;
- final KeyVisualAttributes attr = key.mKeyVisualAttributes;
+ final KeyVisualAttributes attr = key.getVisualAttributes();
final KeyDrawParams params = mKeyDrawParams.mayCloneAndUpdateParams(keyHeight, attr);
params.mAnimAlpha = Constants.Color.ALPHA_OPAQUE;
@@ -330,7 +330,7 @@
protected void onDrawKeyBackground(final Key key, final Canvas canvas) {
final Rect padding = mKeyBackgroundPadding;
final int bgWidth = key.getDrawWidth() + padding.left + padding.right;
- final int bgHeight = key.mHeight + padding.top + padding.bottom;
+ final int bgHeight = key.getHeight() + padding.top + padding.bottom;
final int bgX = -padding.left;
final int bgY = -padding.top;
final int[] drawableState = key.getCurrentDrawableState();
@@ -352,7 +352,7 @@
protected void onDrawKeyTopVisuals(final Key key, final Canvas canvas, final Paint paint,
final KeyDrawParams params) {
final int keyWidth = key.getDrawWidth();
- final int keyHeight = key.mHeight;
+ final int keyHeight = key.getHeight();
final float centerX = keyWidth * 0.5f;
final float centerY = keyHeight * 0.5f;
@@ -363,8 +363,8 @@
// Draw key label.
final Drawable icon = key.getIcon(mKeyboard.mIconsSet, params.mAnimAlpha);
float positionX = centerX;
- if (key.mLabel != null) {
- final String label = key.mLabel;
+ final String label = key.getLabel();
+ if (label != null) {
paint.setTypeface(key.selectTypeface(params));
paint.setTextSize(key.selectTextSize(params));
final float labelCharHeight = TypefaceUtils.getCharHeight(
@@ -441,8 +441,8 @@
}
// Draw hint label.
- if (key.mHintLabel != null) {
- final String hintLabel = key.mHintLabel;
+ final String hintLabel = key.getHintLabel();
+ if (hintLabel != null) {
paint.setTextSize(key.selectHintTextSize(params));
paint.setColor(key.selectHintTextColor(params));
blendAlpha(paint, params.mAnimAlpha);
@@ -481,7 +481,7 @@
}
// Draw key icon.
- if (key.mLabel == null && icon != null) {
+ if (label == null && icon != null) {
final int iconWidth = Math.min(icon.getIntrinsicWidth(), keyWidth);
final int iconHeight = icon.getIntrinsicHeight();
final int iconX, alignX;
@@ -505,7 +505,7 @@
}
}
- if (key.hasPopupHint() && key.mMoreKeys != null) {
+ if (key.hasPopupHint() && key.getMoreKeys() != null) {
drawKeyPopupHint(key, canvas, paint, params);
}
}
@@ -514,7 +514,7 @@
protected void drawKeyPopupHint(final Key key, final Canvas canvas, final Paint paint,
final KeyDrawParams params) {
final int keyWidth = key.getDrawWidth();
- final int keyHeight = key.mHeight;
+ final int keyHeight = key.getHeight();
paint.setTypeface(params.mTypeface);
paint.setTextSize(params.mHintLetterSize);
@@ -602,9 +602,9 @@
if (mInvalidateAllKeys) return;
if (key == null) return;
mInvalidatedKeys.add(key);
- final int x = key.mX + getPaddingLeft();
- final int y = key.mY + getPaddingTop();
- invalidate(x, y, x + key.mWidth, y + key.mHeight);
+ final int x = key.getX() + getPaddingLeft();
+ final int y = key.getY() + getPaddingTop();
+ invalidate(x, y, x + key.getWidth(), y + key.getHeight());
}
@Override
diff --git a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java
index f3d0ead..e4a8f4c 100644
--- a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java
+++ b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java
@@ -237,7 +237,7 @@
return;
}
sendMessageDelayed(
- obtainMessage(MSG_REPEAT_KEY, key.mCode, repeatCount, tracker), delay);
+ obtainMessage(MSG_REPEAT_KEY, key.getCode(), repeatCount, tracker), delay);
}
public void cancelKeyRepeatTimer() {
@@ -298,7 +298,7 @@
final MainKeyboardView keyboardView = getOuterInstance();
// When user hits the space or the enter key, just cancel the while-typing timer.
- final int typedCode = typedKey.mCode;
+ final int typedCode = typedKey.getCode();
if (typedCode == Constants.CODE_SPACE || typedCode == Constants.CODE_ENTER) {
if (isTyping) {
startWhileTypingFadeinAnimation(keyboardView);
@@ -638,7 +638,6 @@
mKeyPreviewLingerTimeout = delay;
}
-
private void locatePreviewPlacerView() {
if (mPreviewPlacerView.getParent() != null) {
return;
@@ -807,11 +806,11 @@
}
// The key preview is placed vertically above the top edge of the parent key with an
// arbitrary offset.
- final int previewY = key.mY - previewHeight + mKeyPreviewOffset
+ final int previewY = key.getY() - previewHeight + mKeyPreviewOffset
+ CoordinateUtils.y(mOriginCoords);
if (background != null) {
- final int hasMoreKeys = (key.mMoreKeys != null) ? STATE_HAS_MOREKEYS : STATE_NORMAL;
+ final int hasMoreKeys = (key.getMoreKeys() != null) ? STATE_HAS_MOREKEYS : STATE_NORMAL;
background.setState(KEY_PREVIEW_BACKGROUND_STATE_TABLE[statePosition][hasMoreKeys]);
background.setAlpha(PREVIEW_ALPHA);
}
@@ -903,7 +902,7 @@
}
private MoreKeysPanel onCreateMoreKeysPanel(final Key key, final Context context) {
- if (key.mMoreKeys == null) {
+ if (key.getMoreKeys() == null) {
return null;
}
Keyboard moreKeysKeyboard = mMoreKeysKeyboardCache.get(key);
@@ -938,7 +937,7 @@
}
final KeyboardActionListener listener = mKeyboardActionListener;
if (key.hasNoPanelAutoMoreKey()) {
- final int moreKeyCode = key.mMoreKeys[0].mCode;
+ final int moreKeyCode = key.getMoreKeys()[0].mCode;
tracker.onLongPressed();
listener.onPressKey(moreKeyCode, 0 /* repeatCount */, true /* isSinglePointer */);
listener.onCodeInput(moreKeyCode,
@@ -946,7 +945,7 @@
listener.onReleaseKey(moreKeyCode, false /* withSliding */);
return;
}
- final int code = key.mCode;
+ final int code = key.getCode();
if (code == Constants.CODE_SPACE || code == Constants.CODE_LANGUAGE_SWITCH) {
// Long pressing the space key invokes IME switcher dialog.
if (listener.onCustomRequest(Constants.CUSTOM_CODE_SHOW_INPUT_METHOD_PICKER)) {
@@ -972,13 +971,13 @@
// keys keyboard is placed at the touch point of the parent key.
final int pointX = (mConfigShowMoreKeysKeyboardAtTouchedPoint && !keyPreviewEnabled)
? CoordinateUtils.x(lastCoords)
- : key.mX + key.mWidth / 2;
+ : key.getX() + key.getWidth() / 2;
// The more keys keyboard is usually vertically aligned with the top edge of the parent key
// (plus vertical gap). If the key preview is enabled, the more keys keyboard is vertically
// aligned with the bottom edge of the visible part of the key preview.
// {@code mPreviewVisibleOffset} has been set appropriately in
// {@link KeyboardView#showKeyPreview(PointerTracker)}.
- final int pointY = key.mY + mKeyPreviewDrawParams.mPreviewVisibleOffset;
+ final int pointY = key.getY() + mKeyPreviewDrawParams.mPreviewVisibleOffset;
moreKeysPanel.showMoreKeysPanel(this, this, pointX, pointY, mKeyboardActionListener);
tracker.onShowMoreKeysPanel(moreKeysPanel);
}
@@ -1174,13 +1173,14 @@
if (key.altCodeWhileTyping() && key.isEnabled()) {
params.mAnimAlpha = mAltCodeKeyWhileTypingAnimAlpha;
}
- if (key.mCode == Constants.CODE_SPACE) {
+ final int code = key.getCode();
+ if (code == Constants.CODE_SPACE) {
drawSpacebar(key, canvas, paint);
// Whether space key needs to show the "..." popup hint for special purposes
if (key.isLongPressEnabled() && mHasMultipleEnabledIMEsOrSubtypes) {
drawKeyPopupHint(key, canvas, paint, params);
}
- } else if (key.mCode == Constants.CODE_LANGUAGE_SWITCH) {
+ } else if (code == Constants.CODE_LANGUAGE_SWITCH) {
super.onDrawKeyTopVisuals(key, canvas, paint, params);
drawKeyPopupHint(key, canvas, paint, params);
} else {
@@ -1228,8 +1228,8 @@
}
private void drawSpacebar(final Key key, final Canvas canvas, final Paint paint) {
- final int width = key.mWidth;
- final int height = key.mHeight;
+ final int width = key.getWidth();
+ final int height = key.getHeight();
// If input language are explicitly selected.
if (mNeedsToDisplayLanguage) {
diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java
index 3fd29dc..8256d46 100644
--- a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java
+++ b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java
@@ -276,6 +276,7 @@
mParams.mVerticalGap = parentKeyboard.mVerticalGap / 2;
mParentKey = parentKey;
+ final MoreKeySpec[] moreKeys = parentKey.getMoreKeys();
final int width, height;
// {@link KeyPreviewDrawParams#mPreviewVisibleWidth} should have been set at
// {@link MainKeyboardView#showKeyPreview(PointerTracker}, though there may be
@@ -283,7 +284,7 @@
// zero-division error at
// {@link MoreKeysKeyboardParams#setParameters(int,int,int,int,int,int,boolean,int)}.
final boolean singleMoreKeyWithPreview = parentKeyboardView.isKeyPreviewPopupEnabled()
- && !parentKey.noKeyPreview() && parentKey.mMoreKeys.length == 1
+ && !parentKey.noKeyPreview() && moreKeys.length == 1
&& keyPreviewDrawParams.mPreviewVisibleWidth > 0;
if (singleMoreKeyWithPreview) {
// Use pre-computed width and height if this more keys keyboard has only one key to
@@ -312,8 +313,8 @@
mDivider = null;
dividerWidth = 0;
}
- mParams.setParameters(parentKey.mMoreKeys.length, parentKey.getMoreKeysColumn(),
- width, height, parentKey.mX + parentKey.mWidth / 2,
+ mParams.setParameters(moreKeys.length, parentKey.getMoreKeysColumn(),
+ width, height, parentKey.getX() + parentKey.getWidth() / 2,
parentKeyboard.mId.mWidth, parentKey.isFixedColumnOrderMoreKeys(),
dividerWidth);
}
@@ -321,7 +322,7 @@
private static int getMaxKeyWidth(final Key parentKey, final int minKeyWidth,
final float padding, final Paint paint) {
int maxWidth = minKeyWidth;
- for (final MoreKeySpec spec : parentKey.mMoreKeys) {
+ for (final MoreKeySpec spec : parentKey.getMoreKeys()) {
final String label = spec.mLabel;
// If the label is single letter, minKeyWidth is enough to hold the label.
if (label != null && StringUtils.codePointCount(label) > 1) {
@@ -336,7 +337,7 @@
public MoreKeysKeyboard build() {
final MoreKeysKeyboardParams params = mParams;
final int moreKeyFlags = mParentKey.getMoreKeyLabelFlags();
- final MoreKeySpec[] moreKeys = mParentKey.mMoreKeys;
+ final MoreKeySpec[] moreKeys = mParentKey.getMoreKeys();
for (int n = 0; n < moreKeys.length; n++) {
final MoreKeySpec moreKeySpec = moreKeys[n];
final int row = n / params.mNumColumns;
diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java
index f00f5a9..973128d 100644
--- a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java
+++ b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java
@@ -127,7 +127,7 @@
public void onUpEvent(final int x, final int y, final int pointerId, final long eventTime) {
if (mCurrentKey != null && mActivePointerId == pointerId) {
updateReleaseKeyGraphics(mCurrentKey);
- onCodeInput(mCurrentKey.mCode, x, y);
+ onCodeInput(mCurrentKey.getCode(), x, y);
mCurrentKey = null;
}
}
diff --git a/java/src/com/android/inputmethod/keyboard/PointerTracker.java b/java/src/com/android/inputmethod/keyboard/PointerTracker.java
index 5387ddb..d4d0d87 100644
--- a/java/src/com/android/inputmethod/keyboard/PointerTracker.java
+++ b/java/src/com/android/inputmethod/keyboard/PointerTracker.java
@@ -510,7 +510,7 @@
return false;
}
if (key.isEnabled()) {
- mListener.onPressKey(key.mCode, repeatCount, getActivePointerTrackerCount() == 1);
+ mListener.onPressKey(key.getCode(), repeatCount, getActivePointerTrackerCount() == 1);
final boolean keyboardLayoutHasBeenChanged = mKeyboardLayoutHasBeenChanged;
mKeyboardLayoutHasBeenChanged = false;
mTimerProxy.startTypingStateTimer(key);
@@ -776,7 +776,7 @@
if (sInGesture || !mGestureStrokeWithPreviewPoints.isStartOfAGesture()) {
return;
}
- if (key == null || !Character.isLetter(key.mCode)) {
+ if (key == null || !Character.isLetter(key.getCode())) {
return;
}
if (DEBUG_LISTENER) {
@@ -1075,8 +1075,8 @@
+ " phantom sudden move event (distance=%d) is translated to "
+ "up[%d,%d,%s]/down[%d,%d,%s] events", mPointerId,
getDistance(x, y, lastX, lastY),
- lastX, lastY, Constants.printableCode(oldKey.mCode),
- x, y, Constants.printableCode(key.mCode)));
+ lastX, lastY, Constants.printableCode(oldKey.getCode()),
+ x, y, Constants.printableCode(key.getCode())));
}
// TODO: This should be moved to outside of this nested if-clause?
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
@@ -1098,8 +1098,8 @@
+ " bogus down-move-up event (raidus=%.2f key diagonal) is "
+ " translated to up[%d,%d,%s]/down[%d,%d,%s] events",
mPointerId, radiusRatio,
- lastX, lastY, Constants.printableCode(oldKey.mCode),
- x, y, Constants.printableCode(key.mCode)));
+ lastX, lastY, Constants.printableCode(oldKey.getCode()),
+ x, y, Constants.printableCode(key.getCode())));
}
onUpEventInternal(x, y, eventTime);
onDownEventInternal(x, y, eventTime);
@@ -1107,7 +1107,7 @@
private void processSildeOutFromOldKey(final Key oldKey) {
setReleasedKeyGraphics(oldKey);
- callListenerOnRelease(oldKey, oldKey.mCode, true /* withSliding */);
+ callListenerOnRelease(oldKey, oldKey.getCode(), true /* withSliding */);
startSlidingKeyInput(oldKey);
mTimerProxy.cancelKeyTimers();
}
@@ -1263,7 +1263,7 @@
if (sInGesture) {
if (currentKey != null) {
- callListenerOnRelease(currentKey, currentKey.mCode, true /* withSliding */);
+ callListenerOnRelease(currentKey, currentKey.getCode(), true /* withSliding */);
}
mayEndBatchInput(eventTime);
return;
@@ -1376,9 +1376,9 @@
// doesn't have its more keys. (e.g. spacebar, globe key)
// We always need to start the long press timer if the key has its more keys regardless of
// whether or not we are in the sliding input mode.
- if (mIsInSlidingKeyInput && key.mMoreKeys == null) return;
+ if (mIsInSlidingKeyInput && key.getMoreKeys() == null) return;
final int delay;
- switch (key.mCode) {
+ switch (key.getCode()) {
case Constants.CODE_SHIFT:
delay = sParams.mLongPressShiftLockTimeout;
break;
@@ -1401,7 +1401,7 @@
return;
}
- final int code = key.mCode;
+ final int code = key.getCode();
callListenerOnCodeInput(key, code, x, y, eventTime);
callListenerOnRelease(key, code, false /* withSliding */);
}
@@ -1412,14 +1412,14 @@
if (!key.isRepeatable()) return;
// Don't start key repeat when we are in sliding input mode.
if (mIsInSlidingKeyInput) return;
- detectAndSendKey(key, key.mX, key.mY, SystemClock.uptimeMillis());
+ detectAndSendKey(key, key.getX(), key.getY(), SystemClock.uptimeMillis());
final int startRepeatCount = 1;
mTimerProxy.startKeyRepeatTimer(this, startRepeatCount, sParams.mKeyRepeatStartTimeout);
}
public void onKeyRepeat(final int code, final int repeatCount) {
final Key key = getKey();
- if (key == null || key.mCode != code) {
+ if (key == null || key.getCode() != code) {
return;
}
final int nextRepeatCount = repeatCount + 1;
diff --git a/java/src/com/android/inputmethod/keyboard/ProximityInfo.java b/java/src/com/android/inputmethod/keyboard/ProximityInfo.java
index 9b0a33c..c0c02f1 100644
--- a/java/src/com/android/inputmethod/keyboard/ProximityInfo.java
+++ b/java/src/com/android/inputmethod/keyboard/ProximityInfo.java
@@ -96,7 +96,7 @@
private static boolean needsProximityInfo(final Key key) {
// Don't include special keys into ProximityInfo.
- return key.mCode >= Constants.CODE_SPACE;
+ return key.getCode() >= Constants.CODE_SPACE;
}
private static int getProximityInfoKeysCount(final Key[] keys) {
@@ -122,7 +122,7 @@
if (!needsProximityInfo(neighborKey)) {
continue;
}
- proximityCharsArray[infoIndex] = neighborKey.mCode;
+ proximityCharsArray[infoIndex] = neighborKey.getCode();
infoIndex++;
}
}
@@ -159,11 +159,11 @@
if (!needsProximityInfo(key)) {
continue;
}
- keyXCoordinates[infoIndex] = key.mX;
- keyYCoordinates[infoIndex] = key.mY;
- keyWidths[infoIndex] = key.mWidth;
- keyHeights[infoIndex] = key.mHeight;
- keyCharCodes[infoIndex] = key.mCode;
+ keyXCoordinates[infoIndex] = key.getX();
+ keyYCoordinates[infoIndex] = key.getY();
+ keyWidths[infoIndex] = key.getWidth();
+ keyHeights[infoIndex] = key.getHeight();
+ keyCharCodes[infoIndex] = key.getCode();
infoIndex++;
}
@@ -183,7 +183,7 @@
if (!needsProximityInfo(key)) {
continue;
}
- final Rect hitBox = key.mHitBox;
+ final Rect hitBox = key.getHitBox();
sweetSpotCenterXs[infoIndex] = hitBox.exactCenterX();
sweetSpotCenterYs[infoIndex] = hitBox.exactCenterY();
sweetSpotRadii[infoIndex] = defaultRadius;
@@ -204,7 +204,7 @@
" [%2d] row=%d x/y/r=%7.2f/%7.2f/%5.2f %s code=%s", infoIndex, row,
sweetSpotCenterXs[infoIndex], sweetSpotCenterYs[infoIndex],
sweetSpotRadii[infoIndex], (row < rows ? "correct" : "default"),
- Constants.printableCode(key.mCode)));
+ Constants.printableCode(key.getCode())));
}
infoIndex++;
}
@@ -322,19 +322,21 @@
have to align this on the center of the key. Hence, we don't need a separate value for
bottomPixelWithinThreshold and call this yEnd right away.
*/
- final int topPixelWithinThreshold = key.mY - threshold;
+ final int keyX = key.getX();
+ final int keyY = key.getY();
+ final int topPixelWithinThreshold = keyY - threshold;
final int yDeltaToGrid = topPixelWithinThreshold % mCellHeight;
final int yMiddleOfTopCell = topPixelWithinThreshold - yDeltaToGrid + halfCellHeight;
final int yStart = Math.max(halfCellHeight,
yMiddleOfTopCell + (yDeltaToGrid <= halfCellHeight ? 0 : mCellHeight));
- final int yEnd = Math.min(fullGridHeight, key.mY + key.mHeight + threshold);
+ final int yEnd = Math.min(fullGridHeight, keyY + key.getHeight() + threshold);
- final int leftPixelWithinThreshold = key.mX - threshold;
+ final int leftPixelWithinThreshold = keyX - threshold;
final int xDeltaToGrid = leftPixelWithinThreshold % mCellWidth;
final int xMiddleOfLeftCell = leftPixelWithinThreshold - xDeltaToGrid + halfCellWidth;
final int xStart = Math.max(halfCellWidth,
xMiddleOfLeftCell + (xDeltaToGrid <= halfCellWidth ? 0 : mCellWidth));
- final int xEnd = Math.min(fullGridWidth, key.mX + key.mWidth + threshold);
+ final int xEnd = Math.min(fullGridWidth, keyX + key.getWidth() + threshold);
int baseIndexOfCurrentRow = (yStart / mCellHeight) * mGridWidth + (xStart / mCellWidth);
for (int centerY = yStart; centerY <= yEnd; centerY += mCellHeight) {
@@ -372,7 +374,7 @@
if (index >= destLength) {
break;
}
- final int code = key.mCode;
+ final int code = key.getCode();
if (code <= Constants.CODE_SPACE) {
break;
}
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java b/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java
index f650569..e6a6743 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java
@@ -24,7 +24,7 @@
public abstract String[] getStringArray(TypedArray a, int index);
public abstract String getString(TypedArray a, int index);
public abstract int getInt(TypedArray a, int index, int defaultValue);
- public abstract int getFlag(TypedArray a, int index);
+ public abstract int getFlags(TypedArray a, int index);
protected KeyStyle(final KeyboardTextsSet textsSet) {
mTextsSet = textsSet;
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java
index 6aab3e7..b21ea3f 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java
@@ -66,7 +66,7 @@
}
@Override
- public int getFlag(final TypedArray a, final int index) {
+ public int getFlags(final TypedArray a, final int index) {
return a.getInt(index, 0);
}
}
@@ -123,14 +123,11 @@
}
@Override
- public int getFlag(final TypedArray a, final int index) {
- int flags = a.getInt(index, 0);
- final Object value = mStyleAttributes.get(index);
- if (value != null) {
- flags |= (Integer)value;
- }
+ public int getFlags(final TypedArray a, final int index) {
+ final Integer value = (Integer)mStyleAttributes.get(index);
+ final int flags = a.getInt(index, (value != null) ? value : 0);
final KeyStyle parentStyle = mStyles.get(mParentStyleName);
- return flags | parentStyle.getFlag(a, index);
+ return flags | parentStyle.getFlags(a, index);
}
public void readKeyAttributes(final TypedArray keyAttr) {
@@ -142,13 +139,13 @@
readString(keyAttr, R.styleable.Keyboard_Key_keyHintLabel);
readStringArray(keyAttr, R.styleable.Keyboard_Key_moreKeys);
readStringArray(keyAttr, R.styleable.Keyboard_Key_additionalMoreKeys);
- readFlag(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags);
+ readFlags(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags);
readString(keyAttr, R.styleable.Keyboard_Key_keyIcon);
readString(keyAttr, R.styleable.Keyboard_Key_keyIconDisabled);
readString(keyAttr, R.styleable.Keyboard_Key_keyIconPreview);
readInt(keyAttr, R.styleable.Keyboard_Key_maxMoreKeysColumn);
readInt(keyAttr, R.styleable.Keyboard_Key_backgroundType);
- readFlag(keyAttr, R.styleable.Keyboard_Key_keyActionFlags);
+ readFlags(keyAttr, R.styleable.Keyboard_Key_keyActionFlags);
}
private void readString(final TypedArray a, final int index) {
@@ -163,7 +160,7 @@
}
}
- private void readFlag(final TypedArray a, final int index) {
+ private void readFlags(final TypedArray a, final int index) {
if (a.hasValue(index)) {
final Integer value = (Integer)mStyleAttributes.get(index);
mStyleAttributes.put(index, a.getInt(index, 0) | (value != null ? value : 0));
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java
index b34d7c4..65eb171 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java
@@ -218,20 +218,18 @@
parseKeyboardAttributes(parser);
startKeyboard();
parseKeyboardContent(parser, false);
- break;
- } else {
- throw new XmlParseUtils.IllegalStartTag(parser, tag, TAG_KEYBOARD);
+ return;
}
+ throw new XmlParseUtils.IllegalStartTag(parser, tag, TAG_KEYBOARD);
}
}
}
private void parseKeyboardAttributes(final XmlPullParser parser) {
+ final AttributeSet attr = Xml.asAttributeSet(parser);
final TypedArray keyboardAttr = mContext.obtainStyledAttributes(
- Xml.asAttributeSet(parser), R.styleable.Keyboard, R.attr.keyboardStyle,
- R.style.Keyboard);
- final TypedArray keyAttr = mResources.obtainAttributes(Xml.asAttributeSet(parser),
- R.styleable.Keyboard_Key);
+ attr, R.styleable.Keyboard, R.attr.keyboardStyle, R.style.Keyboard);
+ final TypedArray keyAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard_Key);
try {
final KeyboardParams params = mParams;
final int height = params.mId.mHeight;
@@ -328,31 +326,30 @@
if (DEBUG) endTag("</%s>", tag);
if (TAG_KEYBOARD.equals(tag)) {
endKeyboard();
- break;
- } else if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag)
- || TAG_MERGE.equals(tag)) {
- break;
- } else {
- throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW);
+ return;
}
+ if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag) || TAG_MERGE.equals(tag)) {
+ return;
+ }
+ throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW);
}
}
}
private KeyboardRow parseRowAttributes(final XmlPullParser parser)
throws XmlPullParserException {
- final TypedArray a = mResources.obtainAttributes(Xml.asAttributeSet(parser),
- R.styleable.Keyboard);
+ final AttributeSet attr = Xml.asAttributeSet(parser);
+ final TypedArray keyboardAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard);
try {
- if (a.hasValue(R.styleable.Keyboard_horizontalGap)) {
+ if (keyboardAttr.hasValue(R.styleable.Keyboard_horizontalGap)) {
throw new XmlParseUtils.IllegalAttribute(parser, TAG_ROW, "horizontalGap");
}
- if (a.hasValue(R.styleable.Keyboard_verticalGap)) {
+ if (keyboardAttr.hasValue(R.styleable.Keyboard_verticalGap)) {
throw new XmlParseUtils.IllegalAttribute(parser, TAG_ROW, "verticalGap");
}
return new KeyboardRow(mResources, mParams, parser, mCurrentY);
} finally {
- a.recycle();
+ keyboardAttr.recycle();
}
}
@@ -382,13 +379,12 @@
if (!skip) {
endRow(row);
}
- break;
- } else if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag)
- || TAG_MERGE.equals(tag)) {
- break;
- } else {
- throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW);
+ return;
}
+ if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag) || TAG_MERGE.equals(tag)) {
+ return;
+ }
+ throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW);
}
}
}
@@ -397,19 +393,16 @@
throws XmlPullParserException, IOException {
if (skip) {
XmlParseUtils.checkEndTag(TAG_KEY, parser);
- if (DEBUG) {
- startEndTag("<%s /> skipped", TAG_KEY);
- }
- } else {
- final Key key = new Key(mResources, mParams, row, parser);
- if (DEBUG) {
- startEndTag("<%s%s %s moreKeys=%s />", TAG_KEY,
- (key.isEnabled() ? "" : " disabled"), key,
- Arrays.toString(key.mMoreKeys));
- }
- XmlParseUtils.checkEndTag(TAG_KEY, parser);
- endKey(key);
+ if (DEBUG) startEndTag("<%s /> skipped", TAG_KEY);
+ return;
}
+ final Key key = new Key(mResources, mParams, row, parser);
+ if (DEBUG) {
+ startEndTag("<%s%s %s moreKeys=%s />", TAG_KEY, (key.isEnabled() ? "" : " disabled"),
+ key, Arrays.toString(key.getMoreKeys()));
+ }
+ XmlParseUtils.checkEndTag(TAG_KEY, parser);
+ endKey(key);
}
private void parseSpacer(final XmlPullParser parser, final KeyboardRow row, final boolean skip)
@@ -417,12 +410,12 @@
if (skip) {
XmlParseUtils.checkEndTag(TAG_SPACER, parser);
if (DEBUG) startEndTag("<%s /> skipped", TAG_SPACER);
- } else {
- final Key.Spacer spacer = new Key.Spacer(mResources, mParams, row, parser);
- if (DEBUG) startEndTag("<%s />", TAG_SPACER);
- XmlParseUtils.checkEndTag(TAG_SPACER, parser);
- endKey(spacer);
+ return;
}
+ final Key.Spacer spacer = new Key.Spacer(mResources, mParams, row, parser);
+ if (DEBUG) startEndTag("<%s />", TAG_SPACER);
+ XmlParseUtils.checkEndTag(TAG_SPACER, parser);
+ endKey(spacer);
}
private void parseIncludeKeyboardContent(final XmlPullParser parser, final boolean skip)
@@ -440,66 +433,63 @@
if (skip) {
XmlParseUtils.checkEndTag(TAG_INCLUDE, parser);
if (DEBUG) startEndTag("</%s> skipped", TAG_INCLUDE);
- } else {
- final AttributeSet attr = Xml.asAttributeSet(parser);
- final TypedArray keyboardAttr = mResources.obtainAttributes(attr,
- R.styleable.Keyboard_Include);
- final TypedArray keyAttr = mResources.obtainAttributes(attr,
- R.styleable.Keyboard_Key);
- int keyboardLayout = 0;
- float savedDefaultKeyWidth = 0;
- int savedDefaultKeyLabelFlags = 0;
- int savedDefaultBackgroundType = Key.BACKGROUND_TYPE_NORMAL;
- try {
- XmlParseUtils.checkAttributeExists(keyboardAttr,
- R.styleable.Keyboard_Include_keyboardLayout, "keyboardLayout",
- TAG_INCLUDE, parser);
- keyboardLayout = keyboardAttr.getResourceId(
- R.styleable.Keyboard_Include_keyboardLayout, 0);
- if (row != null) {
- if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) {
- // Override current x coordinate.
- row.setXPos(row.getKeyX(keyAttr));
- }
- // TODO: Remove this if-clause and do the same as backgroundType below.
- savedDefaultKeyWidth = row.getDefaultKeyWidth();
- if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyWidth)) {
- // Override default key width.
- row.setDefaultKeyWidth(row.getKeyWidth(keyAttr));
- }
- savedDefaultKeyLabelFlags = row.getDefaultKeyLabelFlags();
- // Bitwise-or default keyLabelFlag if exists.
- row.setDefaultKeyLabelFlags(keyAttr.getInt(
- R.styleable.Keyboard_Key_keyLabelFlags, 0)
- | savedDefaultKeyLabelFlags);
- savedDefaultBackgroundType = row.getDefaultBackgroundType();
- // Override default backgroundType if exists.
- row.setDefaultBackgroundType(keyAttr.getInt(
- R.styleable.Keyboard_Key_backgroundType,
- savedDefaultBackgroundType));
+ return;
+ }
+ final AttributeSet attr = Xml.asAttributeSet(parser);
+ final TypedArray keyboardAttr = mResources.obtainAttributes(
+ attr, R.styleable.Keyboard_Include);
+ final TypedArray keyAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard_Key);
+ int keyboardLayout = 0;
+ float savedDefaultKeyWidth = 0;
+ int savedDefaultKeyLabelFlags = 0;
+ int savedDefaultBackgroundType = Key.BACKGROUND_TYPE_NORMAL;
+ try {
+ XmlParseUtils.checkAttributeExists(
+ keyboardAttr, R.styleable.Keyboard_Include_keyboardLayout, "keyboardLayout",
+ TAG_INCLUDE, parser);
+ keyboardLayout = keyboardAttr.getResourceId(
+ R.styleable.Keyboard_Include_keyboardLayout, 0);
+ if (row != null) {
+ if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) {
+ // Override current x coordinate.
+ row.setXPos(row.getKeyX(keyAttr));
}
- } finally {
- keyboardAttr.recycle();
- keyAttr.recycle();
+ // TODO: Remove this if-clause and do the same as backgroundType below.
+ savedDefaultKeyWidth = row.getDefaultKeyWidth();
+ if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyWidth)) {
+ // Override default key width.
+ row.setDefaultKeyWidth(row.getKeyWidth(keyAttr));
+ }
+ savedDefaultKeyLabelFlags = row.getDefaultKeyLabelFlags();
+ // Bitwise-or default keyLabelFlag if exists.
+ row.setDefaultKeyLabelFlags(keyAttr.getInt(
+ R.styleable.Keyboard_Key_keyLabelFlags, 0) | savedDefaultKeyLabelFlags);
+ savedDefaultBackgroundType = row.getDefaultBackgroundType();
+ // Override default backgroundType if exists.
+ row.setDefaultBackgroundType(keyAttr.getInt(
+ R.styleable.Keyboard_Key_backgroundType, savedDefaultBackgroundType));
}
+ } finally {
+ keyboardAttr.recycle();
+ keyAttr.recycle();
+ }
- XmlParseUtils.checkEndTag(TAG_INCLUDE, parser);
- if (DEBUG) {
- startEndTag("<%s keyboardLayout=%s />",TAG_INCLUDE,
- mResources.getResourceEntryName(keyboardLayout));
+ XmlParseUtils.checkEndTag(TAG_INCLUDE, parser);
+ if (DEBUG) {
+ startEndTag("<%s keyboardLayout=%s />",TAG_INCLUDE,
+ mResources.getResourceEntryName(keyboardLayout));
+ }
+ final XmlResourceParser parserForInclude = mResources.getXml(keyboardLayout);
+ try {
+ parseMerge(parserForInclude, row, skip);
+ } finally {
+ if (row != null) {
+ // Restore default keyWidth, keyLabelFlags, and backgroundType.
+ row.setDefaultKeyWidth(savedDefaultKeyWidth);
+ row.setDefaultKeyLabelFlags(savedDefaultKeyLabelFlags);
+ row.setDefaultBackgroundType(savedDefaultBackgroundType);
}
- final XmlResourceParser parserForInclude = mResources.getXml(keyboardLayout);
- try {
- parseMerge(parserForInclude, row, skip);
- } finally {
- if (row != null) {
- // Restore default keyWidth, keyLabelFlags, and backgroundType.
- row.setDefaultKeyWidth(savedDefaultKeyWidth);
- row.setDefaultKeyLabelFlags(savedDefaultKeyLabelFlags);
- row.setDefaultBackgroundType(savedDefaultBackgroundType);
- }
- parserForInclude.close();
- }
+ parserForInclude.close();
}
}
@@ -516,11 +506,10 @@
} else {
parseRowContent(parser, row, skip);
}
- break;
- } else {
- throw new XmlParseUtils.ParseException(
- "Included keyboard layout must have <merge> root element", parser);
+ return;
}
+ throw new XmlParseUtils.ParseException(
+ "Included keyboard layout must have <merge> root element", parser);
}
}
}
@@ -554,10 +543,9 @@
final String tag = parser.getName();
if (TAG_SWITCH.equals(tag)) {
if (DEBUG) endTag("</%s>", TAG_SWITCH);
- break;
- } else {
- throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_SWITCH);
+ return;
}
+ throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_SWITCH);
}
}
}
@@ -580,40 +568,40 @@
if (id == null) {
return true;
}
- final TypedArray a = mResources.obtainAttributes(Xml.asAttributeSet(parser),
- R.styleable.Keyboard_Case);
+ final AttributeSet attr = Xml.asAttributeSet(parser);
+ final TypedArray caseAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard_Case);
try {
- final boolean keyboardLayoutSetElementMatched = matchTypedValue(a,
+ final boolean keyboardLayoutSetElementMatched = matchTypedValue(caseAttr,
R.styleable.Keyboard_Case_keyboardLayoutSetElement, id.mElementId,
KeyboardId.elementIdToName(id.mElementId));
- final boolean modeMatched = matchTypedValue(a,
+ final boolean modeMatched = matchTypedValue(caseAttr,
R.styleable.Keyboard_Case_mode, id.mMode, KeyboardId.modeName(id.mMode));
- final boolean navigateNextMatched = matchBoolean(a,
+ final boolean navigateNextMatched = matchBoolean(caseAttr,
R.styleable.Keyboard_Case_navigateNext, id.navigateNext());
- final boolean navigatePreviousMatched = matchBoolean(a,
+ final boolean navigatePreviousMatched = matchBoolean(caseAttr,
R.styleable.Keyboard_Case_navigatePrevious, id.navigatePrevious());
- final boolean passwordInputMatched = matchBoolean(a,
+ final boolean passwordInputMatched = matchBoolean(caseAttr,
R.styleable.Keyboard_Case_passwordInput, id.passwordInput());
- final boolean clobberSettingsKeyMatched = matchBoolean(a,
+ final boolean clobberSettingsKeyMatched = matchBoolean(caseAttr,
R.styleable.Keyboard_Case_clobberSettingsKey, id.mClobberSettingsKey);
- final boolean shortcutKeyEnabledMatched = matchBoolean(a,
+ final boolean shortcutKeyEnabledMatched = matchBoolean(caseAttr,
R.styleable.Keyboard_Case_shortcutKeyEnabled, id.mShortcutKeyEnabled);
- final boolean shortcutKeyOnSymbolsMatched = matchBoolean(a,
+ final boolean shortcutKeyOnSymbolsMatched = matchBoolean(caseAttr,
R.styleable.Keyboard_Case_shortcutKeyOnSymbols, id.mShortcutKeyOnSymbols);
- final boolean hasShortcutKeyMatched = matchBoolean(a,
+ final boolean hasShortcutKeyMatched = matchBoolean(caseAttr,
R.styleable.Keyboard_Case_hasShortcutKey, id.mHasShortcutKey);
- final boolean languageSwitchKeyEnabledMatched = matchBoolean(a,
+ final boolean languageSwitchKeyEnabledMatched = matchBoolean(caseAttr,
R.styleable.Keyboard_Case_languageSwitchKeyEnabled,
id.mLanguageSwitchKeyEnabled);
- final boolean isMultiLineMatched = matchBoolean(a,
+ final boolean isMultiLineMatched = matchBoolean(caseAttr,
R.styleable.Keyboard_Case_isMultiLine, id.isMultiLine());
- final boolean imeActionMatched = matchInteger(a,
+ final boolean imeActionMatched = matchInteger(caseAttr,
R.styleable.Keyboard_Case_imeAction, id.imeAction());
- final boolean localeCodeMatched = matchString(a,
+ final boolean localeCodeMatched = matchString(caseAttr,
R.styleable.Keyboard_Case_localeCode, id.mLocale.toString());
- final boolean languageCodeMatched = matchString(a,
+ final boolean languageCodeMatched = matchString(caseAttr,
R.styleable.Keyboard_Case_languageCode, id.mLocale.getLanguage());
- final boolean countryCodeMatched = matchString(a,
+ final boolean countryCodeMatched = matchString(caseAttr,
R.styleable.Keyboard_Case_countryCode, id.mLocale.getCountry());
final boolean selected = keyboardLayoutSetElementMatched && modeMatched
&& navigateNextMatched && navigatePreviousMatched && passwordInputMatched
@@ -624,42 +612,42 @@
if (DEBUG) {
startTag("<%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s>%s", TAG_CASE,
- textAttr(a.getString(
+ textAttr(caseAttr.getString(
R.styleable.Keyboard_Case_keyboardLayoutSetElement),
"keyboardLayoutSetElement"),
- textAttr(a.getString(R.styleable.Keyboard_Case_mode), "mode"),
- textAttr(a.getString(R.styleable.Keyboard_Case_imeAction),
+ textAttr(caseAttr.getString(R.styleable.Keyboard_Case_mode), "mode"),
+ textAttr(caseAttr.getString(R.styleable.Keyboard_Case_imeAction),
"imeAction"),
- booleanAttr(a, R.styleable.Keyboard_Case_navigateNext,
+ booleanAttr(caseAttr, R.styleable.Keyboard_Case_navigateNext,
"navigateNext"),
- booleanAttr(a, R.styleable.Keyboard_Case_navigatePrevious,
+ booleanAttr(caseAttr, R.styleable.Keyboard_Case_navigatePrevious,
"navigatePrevious"),
- booleanAttr(a, R.styleable.Keyboard_Case_clobberSettingsKey,
+ booleanAttr(caseAttr, R.styleable.Keyboard_Case_clobberSettingsKey,
"clobberSettingsKey"),
- booleanAttr(a, R.styleable.Keyboard_Case_passwordInput,
+ booleanAttr(caseAttr, R.styleable.Keyboard_Case_passwordInput,
"passwordInput"),
- booleanAttr(a, R.styleable.Keyboard_Case_shortcutKeyEnabled,
+ booleanAttr(caseAttr, R.styleable.Keyboard_Case_shortcutKeyEnabled,
"shortcutKeyEnabled"),
- booleanAttr(a, R.styleable.Keyboard_Case_shortcutKeyOnSymbols,
+ booleanAttr(caseAttr, R.styleable.Keyboard_Case_shortcutKeyOnSymbols,
"shortcutKeyOnSymbols"),
- booleanAttr(a, R.styleable.Keyboard_Case_hasShortcutKey,
+ booleanAttr(caseAttr, R.styleable.Keyboard_Case_hasShortcutKey,
"hasShortcutKey"),
- booleanAttr(a, R.styleable.Keyboard_Case_languageSwitchKeyEnabled,
+ booleanAttr(caseAttr, R.styleable.Keyboard_Case_languageSwitchKeyEnabled,
"languageSwitchKeyEnabled"),
- booleanAttr(a, R.styleable.Keyboard_Case_isMultiLine,
+ booleanAttr(caseAttr, R.styleable.Keyboard_Case_isMultiLine,
"isMultiLine"),
- textAttr(a.getString(R.styleable.Keyboard_Case_localeCode),
+ textAttr(caseAttr.getString(R.styleable.Keyboard_Case_localeCode),
"localeCode"),
- textAttr(a.getString(R.styleable.Keyboard_Case_languageCode),
+ textAttr(caseAttr.getString(R.styleable.Keyboard_Case_languageCode),
"languageCode"),
- textAttr(a.getString(R.styleable.Keyboard_Case_countryCode),
+ textAttr(caseAttr.getString(R.styleable.Keyboard_Case_countryCode),
"countryCode"),
selected ? "" : " skipped");
}
return selected;
} finally {
- a.recycle();
+ caseAttr.recycle();
}
}
@@ -692,7 +680,8 @@
}
if (ResourceUtils.isIntegerValue(v)) {
return intValue == a.getInt(index, 0);
- } else if (ResourceUtils.isStringValue(v)) {
+ }
+ if (ResourceUtils.isStringValue(v)) {
return StringUtils.containsInArray(strValue, a.getString(index).split("\\|"));
}
return false;
@@ -711,10 +700,10 @@
private void parseKeyStyle(final XmlPullParser parser, final boolean skip)
throws XmlPullParserException, IOException {
- TypedArray keyStyleAttr = mResources.obtainAttributes(Xml.asAttributeSet(parser),
- R.styleable.Keyboard_KeyStyle);
- TypedArray keyAttrs = mResources.obtainAttributes(Xml.asAttributeSet(parser),
- R.styleable.Keyboard_Key);
+ final AttributeSet attr = Xml.asAttributeSet(parser);
+ final TypedArray keyStyleAttr = mResources.obtainAttributes(
+ attr, R.styleable.Keyboard_KeyStyle);
+ final TypedArray keyAttrs = mResources.obtainAttributes(attr, R.styleable.Keyboard_Key);
try {
if (!keyStyleAttr.hasValue(R.styleable.Keyboard_KeyStyle_styleName)) {
throw new XmlParseUtils.ParseException("<" + TAG_KEY_STYLE
@@ -756,7 +745,7 @@
mRightEdgeKey = null;
}
addEdgeSpace(mParams.mRightPadding, row);
- mCurrentY += row.mRowHeight;
+ mCurrentY += row.getRowHeight();
mCurrentRow = null;
mTopEdge = false;
}
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java
index a57b83a..d32bb75 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java
@@ -85,7 +85,7 @@
public void onAddKey(final Key newKey) {
final Key key = (mKeysCache != null) ? mKeysCache.get(newKey) : newKey;
final boolean isSpacer = key.isSpacer();
- if (isSpacer && key.mWidth == 0) {
+ if (isSpacer && key.getWidth() == 0) {
// Ignore zero width {@link Spacer}.
return;
}
@@ -94,7 +94,7 @@
return;
}
updateHistogram(key);
- if (key.mCode == Constants.CODE_SHIFT) {
+ if (key.getCode() == Constants.CODE_SHIFT) {
mShiftKeys.add(key);
}
if (key.altCodeWhileTyping()) {
@@ -125,14 +125,14 @@
}
private void updateHistogram(final Key key) {
- final int height = key.mHeight + mVerticalGap;
+ final int height = key.getHeight() + mVerticalGap;
final int heightCount = updateHistogramCounter(mHeightHistogram, height);
if (heightCount > mMaxHeightCount) {
mMaxHeightCount = heightCount;
mMostCommonKeyHeight = height;
}
- final int width = key.mWidth + mHorizontalGap;
+ final int width = key.getWidth() + mHorizontalGap;
final int widthCount = updateHistogramCounter(mWidthHistogram, width);
if (widthCount > mMaxWidthCount) {
mMaxWidthCount = widthCount;
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java
index 5fe84a7..23c0b91 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java
@@ -38,10 +38,10 @@
private static final int KEYWIDTH_FILL_RIGHT = -1;
private final KeyboardParams mParams;
+ /** The height of this row. */
+ private final int mRowHeight;
/** Default width of a key in this row. */
private float mDefaultKeyWidth;
- /** Default height of a key in this row. */
- public final int mRowHeight;
/** Default keyLabelFlags in this row. */
private int mDefaultKeyLabelFlags;
/** Default backgroundType for this row */
@@ -64,16 +64,19 @@
R.styleable.Keyboard_Key);
mDefaultKeyWidth = keyAttr.getFraction(R.styleable.Keyboard_Key_keyWidth,
params.mBaseWidth, params.mBaseWidth, params.mDefaultKeyWidth);
+ mDefaultKeyLabelFlags = keyAttr.getInt(R.styleable.Keyboard_Key_keyLabelFlags, 0);
mDefaultBackgroundType = keyAttr.getInt(R.styleable.Keyboard_Key_backgroundType,
Key.BACKGROUND_TYPE_NORMAL);
keyAttr.recycle();
- // TODO: Initialize this with <Row> attribute as backgroundType is done.
- mDefaultKeyLabelFlags = 0;
mCurrentY = y;
mCurrentX = 0.0f;
}
+ public int getRowHeight() {
+ return mRowHeight;
+ }
+
public float getDefaultKeyWidth() {
return mDefaultKeyWidth;
}
@@ -111,22 +114,21 @@
}
public float getKeyX(final TypedArray keyAttr) {
- if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) {
- final float keyXPos = keyAttr.getFraction(R.styleable.Keyboard_Key_keyXPos,
- mParams.mBaseWidth, mParams.mBaseWidth, 0);
- if (keyXPos < 0) {
- // If keyXPos is negative, the actual x-coordinate will be
- // keyboardWidth + keyXPos.
- // keyXPos shouldn't be less than mCurrentX because drawable area for this
- // key starts at mCurrentX. Or, this key will overlaps the adjacent key on
- // its left hand side.
- final int keyboardRightEdge = mParams.mOccupiedWidth - mParams.mRightPadding;
- return Math.max(keyXPos + keyboardRightEdge, mCurrentX);
- } else {
- return keyXPos + mParams.mLeftPadding;
- }
+ if (keyAttr == null || !keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) {
+ return mCurrentX;
}
- return mCurrentX;
+ final float keyXPos = keyAttr.getFraction(R.styleable.Keyboard_Key_keyXPos,
+ mParams.mBaseWidth, mParams.mBaseWidth, 0);
+ if (keyXPos >= 0) {
+ return keyXPos + mParams.mLeftPadding;
+ }
+ // If keyXPos is negative, the actual x-coordinate will be
+ // keyboardWidth + keyXPos.
+ // keyXPos shouldn't be less than mCurrentX because drawable area for this
+ // key starts at mCurrentX. Or, this key will overlaps the adjacent key on
+ // its left hand side.
+ final int keyboardRightEdge = mParams.mOccupiedWidth - mParams.mRightPadding;
+ return Math.max(keyXPos + keyboardRightEdge, mCurrentX);
}
public float getKeyWidth(final TypedArray keyAttr) {
@@ -134,6 +136,9 @@
}
public float getKeyWidth(final TypedArray keyAttr, final float keyXPos) {
+ if (keyAttr == null) {
+ return mDefaultKeyWidth;
+ }
final int widthType = ResourceUtils.getEnumValue(keyAttr,
R.styleable.Keyboard_Key_keyWidth, KEYWIDTH_NOT_ENUM);
switch (widthType) {
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
index d0a4afd..7e497e9 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
@@ -21,7 +21,7 @@
import android.content.res.AssetFileDescriptor;
import android.util.Log;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.DictionaryInfoUtils;
@@ -231,8 +231,8 @@
try {
// Read the version of the file
inStream = new FileInputStream(f);
- final BinaryDictInputOutput.ByteBufferWrapper buffer =
- new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map(
+ final BinaryDictDecoder.ByteBufferWrapper buffer =
+ new BinaryDictDecoder.ByteBufferWrapper(inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, f.length()));
final int magic = buffer.readInt();
if (magic != FormatSpec.MAGIC_NUMBER) {
@@ -241,7 +241,7 @@
final int formatVersion = buffer.readInt();
final int headerSize = buffer.readInt();
final HashMap<String, String> options = CollectionUtils.newHashMap();
- BinaryDictInputOutput.populateOptions(buffer, headerSize, options);
+ BinaryDictDecoder.populateOptions(buffer, headerSize, options);
final String version = options.get(VERSION_KEY);
if (null == version) {
diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
index a3c6071..1ececd5 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
@@ -20,10 +20,10 @@
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.CollectionUtils;
@@ -51,7 +51,7 @@
@Override
public void clear() {
final HashMap<String, String> attributes = CollectionUtils.newHashMap();
- mFusionDictionary = new FusionDictionary(new Node(),
+ mFusionDictionary = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(attributes, false, false));
}
@@ -87,7 +87,7 @@
@Override
protected void writeBinaryDictionary(final FileOutputStream out)
throws IOException, UnsupportedFormatException {
- BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
+ BinaryDictEncoder.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
}
@Override
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index b3e9301..3725677 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -88,6 +88,9 @@
/** Controls access to the local binary dictionary for this instance. */
private final DictionaryController mLocalDictionaryController = new DictionaryController();
+ /* A extension for a binary dictionary file. */
+ public static final String DICT_FILE_EXTENSION = ".dict";
+
/**
* Abstract method for loading the unigrams and bigrams of a given dictionary in a background
* thread.
@@ -146,7 +149,7 @@
}
protected static String getFilenameWithLocale(final String name, final String localeStr) {
- return name + "." + localeStr + ".dict";
+ return name + "." + localeStr + DICT_FILE_EXTENSION;
}
/**
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index ffe3171..ee7478c 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -1948,7 +1948,8 @@
}
}
}
- if (currentSettings.isSuggestionsRequested(mDisplayOrientation)) {
+ if (currentSettings.isSuggestionsRequested(mDisplayOrientation)
+ && currentSettings.mCurrentLanguageHasSpaces) {
restartSuggestionsOnWordBeforeCursorIfAtEndOfWord();
}
// We just removed a character. We need to update the auto-caps state.
@@ -1977,6 +1978,9 @@
private void handleCharacter(final int primaryCode, final int x,
final int y, final int spaceState) {
+ // TODO: refactor this method to stop flipping isComposingWord around all the time, and
+ // make it shorter (possibly cut into several pieces). Also factor handleNonSpecialCharacter
+ // which has the same name as other handle* methods but is not the same.
boolean isComposingWord = mWordComposer.isComposingWord();
// TODO: remove isWordConnector() and use isUsuallyFollowedBySpace() instead.
@@ -1996,12 +2000,20 @@
resetEntireInputState(mLastSelectionStart);
isComposingWord = false;
}
- // NOTE: isCursorTouchingWord() is a blocking IPC call, so it often takes several
- // dozen milliseconds. Avoid calling it as much as possible, since we are on the UI
- // thread here.
- if (!isComposingWord && currentSettings.isWordCodePoint(primaryCode)
+ // We want to find out whether to start composing a new word with this character. If so,
+ // we need to reset the composing state and switch isComposingWord. The order of the
+ // tests is important for good performance.
+ // We only start composing if we're not already composing.
+ if (!isComposingWord
+ // We only start composing if this is a word code point. Essentially that means it's a
+ // a letter or a word connector.
+ && currentSettings.isWordCodePoint(primaryCode)
+ // We never go into composing state if suggestions are not requested.
&& currentSettings.isSuggestionsRequested(mDisplayOrientation) &&
- !mConnection.isCursorTouchingWord(currentSettings)) {
+ // In languages with spaces, we only start composing a word when we are not already
+ // touching a word. In languages without spaces, the above conditions are sufficient.
+ (!mConnection.isCursorTouchingWord(currentSettings)
+ || !currentSettings.mCurrentLanguageHasSpaces)) {
// Reset entirely the composing state anyway, then start composing a new word unless
// the character is a single quote. The idea here is, single quote is not a
// separator and it should be treated as a normal character, except in the first
@@ -2089,16 +2101,20 @@
private boolean handleSeparator(final int primaryCode, final int x, final int y,
final int spaceState) {
boolean didAutoCorrect = false;
+ final SettingsValues currentSettings = mSettings.getCurrent();
+ // We avoid sending spaces in languages without spaces if we were composing.
+ final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == primaryCode
+ && !currentSettings.mCurrentLanguageHasSpaces && mWordComposer.isComposingWord();
if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) {
// If we are in the middle of a recorrection, we need to commit the recorrection
// first so that we can insert the separator at the current cursor position.
resetEntireInputState(mLastSelectionStart);
}
- final SettingsValues currentSettings = mSettings.getCurrent();
- if (mWordComposer.isComposingWord()) {
+ if (mWordComposer.isComposingWord()) { // May have changed since we stored wasComposing
if (currentSettings.mCorrectionEnabled) {
- // TODO: maybe cache Strings in an <String> sparse array or something
- commitCurrentAutoCorrection(new String(new int[]{primaryCode}, 0, 1));
+ final String separator = shouldAvoidSendingCode ? LastComposedWord.NOT_A_SEPARATOR
+ : new String(new int[] { primaryCode }, 0, 1);
+ commitCurrentAutoCorrection(separator);
didAutoCorrect = true;
} else {
commitTyped(new String(new int[]{primaryCode}, 0, 1));
@@ -2115,7 +2131,10 @@
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
ResearchLogger.latinIME_handleSeparator(primaryCode, mWordComposer.isComposingWord());
}
- sendKeyCodePoint(primaryCode);
+
+ if (!shouldAvoidSendingCode) {
+ sendKeyCodePoint(primaryCode);
+ }
if (Constants.CODE_SPACE == primaryCode) {
if (currentSettings.isSuggestionsRequested(mDisplayOrientation)) {
@@ -2260,11 +2279,17 @@
// Get the word on which we should search the bigrams. If we are composing a word, it's
// whatever is *before* the half-committed word in the buffer, hence 2; if we aren't, we
// should just skip whitespace if any, so 1.
- // TODO: this is slow (2-way IPC) - we should probably cache this instead.
final SettingsValues currentSettings = mSettings.getCurrent();
- final String prevWord =
- mConnection.getNthPreviousWord(currentSettings.mWordSeparators,
- mWordComposer.isComposingWord() ? 2 : 1);
+ final String prevWord;
+ if (currentSettings.mCurrentLanguageHasSpaces) {
+ // If we are typing in a language with spaces we can just look up the previous
+ // word from textview.
+ prevWord = mConnection.getNthPreviousWord(currentSettings.mWordSeparators,
+ mWordComposer.isComposingWord() ? 2 : 1);
+ } else {
+ prevWord = LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? null
+ : mLastComposedWord.mCommittedWord;
+ }
return suggest.getSuggestedWords(mWordComposer, prevWord, keyboard.getProximityInfo(),
currentSettings.mBlockPotentiallyOffensive,
currentSettings.mCorrectionEnabled, sessionId);
@@ -2534,6 +2559,9 @@
// recorrection. This is a temporary, stopgap measure that will be removed later.
// TODO: remove this.
if (mAppWorkAroundsUtils.isBrokenByRecorrection()) return;
+ // Recorrection is not supported in languages without spaces because we don't know
+ // how to segment them yet.
+ if (!mSettings.getCurrent().mCurrentLanguageHasSpaces) return;
// If the cursor is not touching a word, or if there is a selection, return right away.
if (mLastSelectionStart != mLastSelectionEnd) return;
// If we don't know the cursor location, return.
@@ -2656,7 +2684,18 @@
if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) {
mUserHistoryPredictionDictionary.cancelAddingUserHistory(previousWord, committedWord);
}
- mConnection.commitText(originallyTypedWord + mLastComposedWord.mSeparatorString, 1);
+ final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString;
+ if (mSettings.getCurrent().mCurrentLanguageHasSpaces) {
+ // For languages with spaces, we revert to the typed string, but the cursor is still
+ // after the separator so we don't resume suggestions. If the user wants to correct
+ // the word, they have to press backspace again.
+ mConnection.commitText(stringToCommit, 1);
+ } else {
+ // For languages without spaces, we revert the typed string but the cursor is flush
+ // with the typed word, so we need to resume suggestions right away.
+ mWordComposer.setComposingWord(stringToCommit, mKeyboardSwitcher.getKeyboard());
+ mConnection.setComposingText(stringToCommit, 1);
+ }
if (mSettings.isInternal()) {
LatinImeLoggerUtils.onSeparator(mLastComposedWord.mSeparatorString,
Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE);
@@ -2674,7 +2713,9 @@
// This essentially inserts a space, and that's it.
public void promotePhantomSpace() {
- if (mSettings.getCurrent().shouldInsertSpacesAutomatically()
+ final SettingsValues currentSettings = mSettings.getCurrent();
+ if (currentSettings.shouldInsertSpacesAutomatically()
+ && currentSettings.mCurrentLanguageHasSpaces
&& !mConnection.textBeforeCursorLooksLikeURL()) {
if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
ResearchLogger.latinIME_promotePhantomSpace();
@@ -2887,6 +2928,12 @@
return mSuggest.hasMainDictionary();
}
+ // DO NOT USE THIS for any other purpose than testing. This can break the keyboard badly.
+ @UsedForTesting
+ /* package for test */ void replaceMainDictionaryForTest(final Locale locale) {
+ mSuggest.resetMainDict(this, locale, null);
+ }
+
public void debugDumpStateAndCrashWithException(final String context) {
final StringBuilder s = new StringBuilder(mAppWorkAroundsUtils.toString());
s.append("\nAttributes : ").append(mSettings.getCurrent().mInputAttributes)
diff --git a/java/src/com/android/inputmethod/latin/WordComposer.java b/java/src/com/android/inputmethod/latin/WordComposer.java
index a09ca60..039dadc 100644
--- a/java/src/com/android/inputmethod/latin/WordComposer.java
+++ b/java/src/com/android/inputmethod/latin/WordComposer.java
@@ -272,8 +272,8 @@
final int x, y;
final Key key;
if (keyboard != null && (key = keyboard.getKey(codePoint)) != null) {
- x = key.mX + key.mWidth / 2;
- y = key.mY + key.mHeight / 2;
+ x = key.getX() + key.getWidth() / 2;
+ y = key.getY() + key.getHeight() / 2;
} else {
x = Constants.NOT_A_COORDINATE;
y = Constants.NOT_A_COORDINATE;
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java
new file mode 100644
index 0000000..5e3d6d2
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java
@@ -0,0 +1,795 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.makedict;
+
+import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
+import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
+import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
+import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+import com.android.inputmethod.latin.makedict.decoder.HeaderReaderInterface;
+import com.android.inputmethod.latin.utils.JniUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Decodes binary files for a FusionDictionary.
+ *
+ * All the methods in this class are static.
+ */
+public final class BinaryDictDecoder {
+
+ private static final boolean DBG = MakedictLog.DBG;
+
+ static {
+ JniUtils.loadNativeLibrary();
+ }
+
+ // TODO: implement something sensical instead of just a phony method
+ private static native int doNothing();
+
+ private BinaryDictDecoder() {
+ // This utility class is not publicly instantiable.
+ }
+
+ private static final int MAX_JUMPS = 12;
+
+ @UsedForTesting
+ public interface FusionDictionaryBufferInterface {
+ public int readUnsignedByte();
+ public int readUnsignedShort();
+ public int readUnsignedInt24();
+ public int readInt();
+ public int position();
+ public void position(int newPosition);
+ public void put(final byte b);
+ public int limit();
+ @UsedForTesting
+ public int capacity();
+ }
+
+ public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
+ private ByteBuffer mBuffer;
+
+ public ByteBufferWrapper(final ByteBuffer buffer) {
+ mBuffer = buffer;
+ }
+
+ @Override
+ public int readUnsignedByte() {
+ return mBuffer.get() & 0xFF;
+ }
+
+ @Override
+ public int readUnsignedShort() {
+ return mBuffer.getShort() & 0xFFFF;
+ }
+
+ @Override
+ public int readUnsignedInt24() {
+ final int retval = readUnsignedByte();
+ return (retval << 16) + readUnsignedShort();
+ }
+
+ @Override
+ public int readInt() {
+ return mBuffer.getInt();
+ }
+
+ @Override
+ public int position() {
+ return mBuffer.position();
+ }
+
+ @Override
+ public void position(int newPos) {
+ mBuffer.position(newPos);
+ }
+
+ @Override
+ public void put(final byte b) {
+ mBuffer.put(b);
+ }
+
+ @Override
+ public int limit() {
+ return mBuffer.limit();
+ }
+
+ @Override
+ public int capacity() {
+ return mBuffer.capacity();
+ }
+ }
+
+ /**
+ * A class grouping utility function for our specific character encoding.
+ */
+ static final class CharEncoding {
+ private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
+ private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF;
+
+ /**
+ * Helper method to find out whether this code fits on one byte
+ */
+ private static boolean fitsOnOneByte(final int character) {
+ return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE
+ && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE;
+ }
+
+ /**
+ * Compute the size of a character given its character code.
+ *
+ * Char format is:
+ * 1 byte = bbbbbbbb match
+ * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
+ * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
+ * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
+ * 00011111 would be outside unicode.
+ * else: iso-latin-1 code
+ * This allows for the whole unicode range to be encoded, including chars outside of
+ * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
+ * characters which should never happen anyway (and still work, but take 3 bytes).
+ *
+ * @param character the character code.
+ * @return the size in binary encoded-form, either 1 or 3 bytes.
+ */
+ static int getCharSize(final int character) {
+ // See char encoding in FusionDictionary.java
+ if (fitsOnOneByte(character)) return 1;
+ if (FormatSpec.INVALID_CHARACTER == character) return 1;
+ return 3;
+ }
+
+ /**
+ * Compute the byte size of a character array.
+ */
+ static int getCharArraySize(final int[] chars) {
+ int size = 0;
+ for (int character : chars) size += getCharSize(character);
+ return size;
+ }
+
+ /**
+ * Writes a char array to a byte buffer.
+ *
+ * @param codePoints the code point array to write.
+ * @param buffer the byte buffer to write to.
+ * @param index the index in buffer to write the character array to.
+ * @return the index after the last character.
+ */
+ static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) {
+ for (int codePoint : codePoints) {
+ if (1 == getCharSize(codePoint)) {
+ buffer[index++] = (byte)codePoint;
+ } else {
+ buffer[index++] = (byte)(0xFF & (codePoint >> 16));
+ buffer[index++] = (byte)(0xFF & (codePoint >> 8));
+ buffer[index++] = (byte)(0xFF & codePoint);
+ }
+ }
+ return index;
+ }
+
+ /**
+ * Writes a string with our character format to a byte buffer.
+ *
+ * This will also write the terminator byte.
+ *
+ * @param buffer the byte buffer to write to.
+ * @param origin the offset to write from.
+ * @param word the string to write.
+ * @return the size written, in bytes.
+ */
+ static int writeString(final byte[] buffer, final int origin,
+ final String word) {
+ final int length = word.length();
+ int index = origin;
+ for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
+ final int codePoint = word.codePointAt(i);
+ if (1 == getCharSize(codePoint)) {
+ buffer[index++] = (byte)codePoint;
+ } else {
+ buffer[index++] = (byte)(0xFF & (codePoint >> 16));
+ buffer[index++] = (byte)(0xFF & (codePoint >> 8));
+ buffer[index++] = (byte)(0xFF & codePoint);
+ }
+ }
+ buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
+ return index - origin;
+ }
+
+ /**
+ * Writes a string with our character format to a ByteArrayOutputStream.
+ *
+ * This will also write the terminator byte.
+ *
+ * @param buffer the ByteArrayOutputStream to write to.
+ * @param word the string to write.
+ */
+ static void writeString(final ByteArrayOutputStream buffer, final String word) {
+ final int length = word.length();
+ for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
+ final int codePoint = word.codePointAt(i);
+ if (1 == getCharSize(codePoint)) {
+ buffer.write((byte) codePoint);
+ } else {
+ buffer.write((byte) (0xFF & (codePoint >> 16)));
+ buffer.write((byte) (0xFF & (codePoint >> 8)));
+ buffer.write((byte) (0xFF & codePoint));
+ }
+ }
+ buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR);
+ }
+
+ /**
+ * Reads a string from a buffer. This is the converse of the above method.
+ */
+ static String readString(final FusionDictionaryBufferInterface buffer) {
+ final StringBuilder s = new StringBuilder();
+ int character = readChar(buffer);
+ while (character != FormatSpec.INVALID_CHARACTER) {
+ s.appendCodePoint(character);
+ character = readChar(buffer);
+ }
+ return s.toString();
+ }
+
+ /**
+ * Reads a character from the buffer.
+ *
+ * This follows the character format documented earlier in this source file.
+ *
+ * @param buffer the buffer, positioned over an encoded character.
+ * @return the character code.
+ */
+ static int readChar(final FusionDictionaryBufferInterface buffer) {
+ int character = buffer.readUnsignedByte();
+ if (!fitsOnOneByte(character)) {
+ if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) {
+ return FormatSpec.INVALID_CHARACTER;
+ }
+ character <<= 16;
+ character += buffer.readUnsignedShort();
+ }
+ return character;
+ }
+ }
+
+ // Input methods: Read a binary dictionary to memory.
+ // readDictionaryBinary is the public entry point for them.
+
+ static int readChildrenAddress(final FusionDictionaryBufferInterface buffer,
+ final int optionFlags, final FormatOptions options) {
+ if (options.mSupportsDynamicUpdate) {
+ final int address = buffer.readUnsignedInt24();
+ if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS;
+ if ((address & FormatSpec.MSB24) != 0) {
+ return -(address & FormatSpec.SINT24_MAX);
+ } else {
+ return address;
+ }
+ }
+ int address;
+ switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
+ return buffer.readUnsignedByte();
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
+ return buffer.readUnsignedShort();
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
+ return buffer.readUnsignedInt24();
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
+ default:
+ return FormatSpec.NO_CHILDREN_ADDRESS;
+ }
+ }
+
+ static int readParentAddress(final FusionDictionaryBufferInterface buffer,
+ final FormatOptions formatOptions) {
+ if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
+ final int parentAddress = buffer.readUnsignedInt24();
+ final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1;
+ return sign * (parentAddress & FormatSpec.SINT24_MAX);
+ } else {
+ return FormatSpec.NO_PARENT_ADDRESS;
+ }
+ }
+
+ private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH];
+ public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
+ final int originalGroupAddress, final FormatOptions options) {
+ int addressPointer = originalGroupAddress;
+ final int flags = buffer.readUnsignedByte();
+ ++addressPointer;
+
+ final int parentAddress = readParentAddress(buffer, options);
+ if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
+ addressPointer += 3;
+ }
+
+ final int characters[];
+ if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) {
+ int index = 0;
+ int character = CharEncoding.readChar(buffer);
+ addressPointer += CharEncoding.getCharSize(character);
+ while (-1 != character) {
+ // FusionDictionary is making sure that the length of the word is smaller than
+ // MAX_WORD_LENGTH.
+ // So we'll never write past the end of CHARACTER_BUFFER.
+ CHARACTER_BUFFER[index++] = character;
+ character = CharEncoding.readChar(buffer);
+ addressPointer += CharEncoding.getCharSize(character);
+ }
+ characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index);
+ } else {
+ final int character = CharEncoding.readChar(buffer);
+ addressPointer += CharEncoding.getCharSize(character);
+ characters = new int[] { character };
+ }
+ final int frequency;
+ if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) {
+ ++addressPointer;
+ frequency = buffer.readUnsignedByte();
+ } else {
+ frequency = CharGroup.NOT_A_TERMINAL;
+ }
+ int childrenAddress = readChildrenAddress(buffer, flags, options);
+ if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
+ childrenAddress += addressPointer;
+ }
+ addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options);
+ ArrayList<WeightedString> shortcutTargets = null;
+ if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) {
+ final int pointerBefore = buffer.position();
+ shortcutTargets = new ArrayList<WeightedString>();
+ buffer.readUnsignedShort(); // Skip the size
+ while (true) {
+ final int targetFlags = buffer.readUnsignedByte();
+ final String word = CharEncoding.readString(buffer);
+ shortcutTargets.add(new WeightedString(word,
+ targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY));
+ if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
+ }
+ addressPointer += buffer.position() - pointerBefore;
+ }
+ ArrayList<PendingAttribute> bigrams = null;
+ if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
+ bigrams = new ArrayList<PendingAttribute>();
+ int bigramCount = 0;
+ while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
+ final int bigramFlags = buffer.readUnsignedByte();
+ ++addressPointer;
+ final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE)
+ ? 1 : -1;
+ int bigramAddress = addressPointer;
+ switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) {
+ case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
+ bigramAddress += sign * buffer.readUnsignedByte();
+ addressPointer += 1;
+ break;
+ case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
+ bigramAddress += sign * buffer.readUnsignedShort();
+ addressPointer += 2;
+ break;
+ case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
+ final int offset = (buffer.readUnsignedByte() << 16)
+ + buffer.readUnsignedShort();
+ bigramAddress += sign * offset;
+ addressPointer += 3;
+ break;
+ default:
+ throw new RuntimeException("Has bigrams with no address");
+ }
+ bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY,
+ bigramAddress));
+ if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
+ }
+ if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
+ MakedictLog.d("too many bigrams in a group.");
+ }
+ }
+ return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency,
+ parentAddress, childrenAddress, shortcutTargets, bigrams);
+ }
+
+ /**
+ * Reads and returns the char group count out of a buffer and forwards the pointer.
+ */
+ public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
+ final int msb = buffer.readUnsignedByte();
+ if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) {
+ return msb;
+ } else {
+ return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8)
+ + buffer.readUnsignedByte();
+ }
+ }
+
+ /**
+ * Finds, as a string, the word at the address passed as an argument.
+ *
+ * @param buffer the buffer to read from.
+ * @param headerSize the size of the header.
+ * @param address the address to seek.
+ * @param formatOptions file format options.
+ * @return the word with its frequency, as a weighted string.
+ */
+ /* package for tests */ static WeightedString getWordAtAddress(
+ final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
+ final FormatOptions formatOptions) {
+ final WeightedString result;
+ final int originalPointer = buffer.position();
+ buffer.position(address);
+
+ if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
+ result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions);
+ } else {
+ result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address,
+ formatOptions);
+ }
+
+ buffer.position(originalPointer);
+ return result;
+ }
+
+ @SuppressWarnings("unused")
+ private static WeightedString getWordAtAddressWithParentAddress(
+ final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
+ final FormatOptions options) {
+ int currentAddress = address;
+ int frequency = Integer.MIN_VALUE;
+ final StringBuilder builder = new StringBuilder();
+ // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH
+ for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) {
+ CharGroupInfo currentInfo;
+ int loopCounter = 0;
+ do {
+ buffer.position(currentAddress + headerSize);
+ currentInfo = readCharGroup(buffer, currentAddress, options);
+ if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) {
+ currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
+ }
+ if (DBG && loopCounter++ > MAX_JUMPS) {
+ MakedictLog.d("Too many jumps - probably a bug");
+ }
+ } while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options));
+ if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency;
+ builder.insert(0,
+ new String(currentInfo.mCharacters, 0, currentInfo.mCharacters.length));
+ if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break;
+ currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
+ }
+ return new WeightedString(builder.toString(), frequency);
+ }
+
+ private static WeightedString getWordAtAddressWithoutParentAddress(
+ final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
+ final FormatOptions options) {
+ buffer.position(headerSize);
+ final int count = readCharGroupCount(buffer);
+ int groupOffset = BinaryDictIOUtils.getGroupCountSize(count);
+ final StringBuilder builder = new StringBuilder();
+ WeightedString result = null;
+
+ CharGroupInfo last = null;
+ for (int i = count - 1; i >= 0; --i) {
+ CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
+ groupOffset = info.mEndAddress;
+ if (info.mOriginalAddress == address) {
+ builder.append(new String(info.mCharacters, 0, info.mCharacters.length));
+ result = new WeightedString(builder.toString(), info.mFrequency);
+ break; // and return
+ }
+ if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
+ if (info.mChildrenAddress > address) {
+ if (null == last) continue;
+ builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
+ buffer.position(last.mChildrenAddress + headerSize);
+ i = readCharGroupCount(buffer);
+ groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
+ last = null;
+ continue;
+ }
+ last = info;
+ }
+ if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) {
+ builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
+ buffer.position(last.mChildrenAddress + headerSize);
+ i = readCharGroupCount(buffer);
+ groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
+ last = null;
+ continue;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Reads a single node array from a buffer.
+ *
+ * This methods reads the file at the current position. A node array is fully expected to start
+ * at the current position.
+ * This will recursively read other node arrays into the structure, populating the reverse
+ * maps on the fly and using them to keep track of already read nodes.
+ *
+ * @param buffer the buffer, correctly positioned at the start of a node array.
+ * @param headerSize the size, in bytes, of the file header.
+ * @param reverseNodeArrayMap a mapping from addresses to already read node arrays.
+ * @param reverseGroupMap a mapping from addresses to already read character groups.
+ * @param options file format options.
+ * @return the read node array with all his children already read.
+ */
+ private static PtNodeArray readNodeArray(final FusionDictionaryBufferInterface buffer,
+ final int headerSize, final Map<Integer, PtNodeArray> reverseNodeArrayMap,
+ final Map<Integer, CharGroup> reverseGroupMap, final FormatOptions options)
+ throws IOException {
+ final ArrayList<CharGroup> nodeArrayContents = new ArrayList<CharGroup>();
+ final int nodeArrayOrigin = buffer.position() - headerSize;
+
+ do { // Scan the linked-list node.
+ final int nodeArrayHeadPosition = buffer.position() - headerSize;
+ final int count = readCharGroupCount(buffer);
+ int groupOffset = nodeArrayHeadPosition + BinaryDictIOUtils.getGroupCountSize(count);
+ for (int i = count; i > 0; --i) { // Scan the array of CharGroup.
+ CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
+ if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue;
+ ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets;
+ ArrayList<WeightedString> bigrams = null;
+ if (null != info.mBigrams) {
+ bigrams = new ArrayList<WeightedString>();
+ for (PendingAttribute bigram : info.mBigrams) {
+ final WeightedString word = getWordAtAddress(
+ buffer, headerSize, bigram.mAddress, options);
+ final int reconstructedFrequency =
+ reconstructBigramFrequency(word.mFrequency, bigram.mFrequency);
+ bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
+ }
+ }
+ if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
+ PtNodeArray children = reverseNodeArrayMap.get(info.mChildrenAddress);
+ if (null == children) {
+ final int currentPosition = buffer.position();
+ buffer.position(info.mChildrenAddress + headerSize);
+ children = readNodeArray(
+ buffer, headerSize, reverseNodeArrayMap, reverseGroupMap, options);
+ buffer.position(currentPosition);
+ }
+ nodeArrayContents.add(
+ new CharGroup(info.mCharacters, shortcutTargets, bigrams,
+ info.mFrequency,
+ 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
+ 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children));
+ } else {
+ nodeArrayContents.add(
+ new CharGroup(info.mCharacters, shortcutTargets, bigrams,
+ info.mFrequency,
+ 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
+ 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED)));
+ }
+ groupOffset = info.mEndAddress;
+ }
+
+ // reach the end of the array.
+ if (options.mSupportsDynamicUpdate) {
+ final int nextAddress = buffer.readUnsignedInt24();
+ if (nextAddress >= 0 && nextAddress < buffer.limit()) {
+ buffer.position(nextAddress);
+ } else {
+ break;
+ }
+ }
+ } while (options.mSupportsDynamicUpdate &&
+ buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS);
+
+ final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents);
+ nodeArray.mCachedAddressBeforeUpdate = nodeArrayOrigin;
+ nodeArray.mCachedAddressAfterUpdate = nodeArrayOrigin;
+ reverseNodeArrayMap.put(nodeArray.mCachedAddressAfterUpdate, nodeArray);
+ return nodeArray;
+ }
+
+ /**
+ * Helper function to get the binary format version from the header.
+ * @throws IOException
+ */
+ private static int getFormatVersion(final FusionDictionaryBufferInterface buffer)
+ throws IOException {
+ final int magic = buffer.readInt();
+ if (FormatSpec.MAGIC_NUMBER == magic) return buffer.readUnsignedShort();
+ return FormatSpec.NOT_A_VERSION_NUMBER;
+ }
+
+ /**
+ * Helper function to get and validate the binary format version.
+ * @throws UnsupportedFormatException
+ * @throws IOException
+ */
+ static int checkFormatVersion(final FusionDictionaryBufferInterface buffer)
+ throws IOException, UnsupportedFormatException {
+ final int version = getFormatVersion(buffer);
+ if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
+ || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
+ throw new UnsupportedFormatException("This file has version " + version
+ + ", but this implementation does not support versions above "
+ + FormatSpec.MAXIMUM_SUPPORTED_VERSION);
+ }
+ return version;
+ }
+
+ /**
+ * Reads a header from a buffer.
+ * @param headerReader the header reader
+ * @throws IOException
+ * @throws UnsupportedFormatException
+ */
+ public static FileHeader readHeader(final HeaderReaderInterface headerReader)
+ throws IOException, UnsupportedFormatException {
+ final int version = headerReader.readVersion();
+ final int optionsFlags = headerReader.readOptionFlags();
+
+ final int headerSize = headerReader.readHeaderSize();
+
+ if (headerSize < 0) {
+ throw new UnsupportedFormatException("header size can't be negative.");
+ }
+
+ final HashMap<String, String> attributes = headerReader.readAttributes(headerSize);
+
+ final FileHeader header = new FileHeader(headerSize,
+ new FusionDictionary.DictionaryOptions(attributes,
+ 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG),
+ 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)),
+ new FormatOptions(version,
+ 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE)));
+ return header;
+ }
+
+ /**
+ * Reads options from a buffer and populate a map with their contents.
+ *
+ * The buffer is read at the current position, so the caller must take care the pointer
+ * is in the right place before calling this.
+ */
+ public static void populateOptions(final FusionDictionaryBufferInterface buffer,
+ final int headerSize, final HashMap<String, String> options) {
+ while (buffer.position() < headerSize) {
+ final String key = CharEncoding.readString(buffer);
+ final String value = CharEncoding.readString(buffer);
+ options.put(key, value);
+ }
+ }
+
+ /**
+ * Reads a buffer and returns the memory representation of the dictionary.
+ *
+ * This high-level method takes a buffer and reads its contents, populating a
+ * FusionDictionary structure. The optional dict argument is an existing dictionary to
+ * which words from the buffer should be added. If it is null, a new dictionary is created.
+ *
+ * @param reader the reader.
+ * @param dict an optional dictionary to add words to, or null.
+ * @return the created (or merged) dictionary.
+ */
+ @UsedForTesting
+ public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
+ final FusionDictionary dict) throws FileNotFoundException, IOException,
+ UnsupportedFormatException {
+
+ // if the buffer has not been opened, open the buffer with bytebuffer.
+ if (reader.getBuffer() == null) reader.openBuffer(
+ new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
+ if (reader.getBuffer() == null) {
+ MakedictLog.e("Cannot open the buffer");
+ }
+
+ // Read header
+ final FileHeader fileHeader = readHeader(reader);
+
+ Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>();
+ Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
+ final PtNodeArray root = readNodeArray(reader.getBuffer(), fileHeader.mHeaderSize,
+ reverseNodeArrayMapping, reverseGroupMapping, fileHeader.mFormatOptions);
+
+ FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions);
+ if (null != dict) {
+ for (final Word w : dict) {
+ if (w.mIsBlacklistEntry) {
+ newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord);
+ } else {
+ newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord);
+ }
+ }
+ for (final Word w : dict) {
+ // By construction a binary dictionary may not have bigrams pointing to
+ // words that are not also registered as unigrams so we don't have to avoid
+ // them explicitly here.
+ for (final WeightedString bigram : w.mBigrams) {
+ newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency);
+ }
+ }
+ }
+
+ return newDict;
+ }
+
+ /**
+ * Helper method to pass a file name instead of a File object to isBinaryDictionary.
+ */
+ public static boolean isBinaryDictionary(final String filename) {
+ final File file = new File(filename);
+ return isBinaryDictionary(file);
+ }
+
+ /**
+ * Basic test to find out whether the file is a binary dictionary or not.
+ *
+ * Concretely this only tests the magic number.
+ *
+ * @param file The file to test.
+ * @return true if it's a binary dictionary, false otherwise
+ */
+ public static boolean isBinaryDictionary(final File file) {
+ FileInputStream inStream = null;
+ try {
+ inStream = new FileInputStream(file);
+ final ByteBuffer buffer = inStream.getChannel().map(
+ FileChannel.MapMode.READ_ONLY, 0, file.length());
+ final int version = getFormatVersion(new ByteBufferWrapper(buffer));
+ return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION
+ && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION);
+ } catch (FileNotFoundException e) {
+ return false;
+ } catch (IOException e) {
+ return false;
+ } finally {
+ if (inStream != null) {
+ try {
+ inStream.close();
+ } catch (IOException e) {
+ // do nothing
+ }
+ }
+ }
+ }
+
+ /**
+ * Calculate bigram frequency from compressed value
+ *
+ * @param unigramFrequency
+ * @param bigramFrequency compressed frequency
+ * @return approximate bigram frequency
+ */
+ public static int reconstructBigramFrequency(final int unigramFrequency,
+ final int bigramFrequency) {
+ final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
+ / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
+ final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f);
+ return (int)resultFreqFloat;
+ }
+}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java
new file mode 100644
index 0000000..d9005b9
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java
@@ -0,0 +1,1006 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.makedict;
+
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.CharEncoding;
+import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
+import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
+import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
+import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+/**
+ * Encodes binary files for a FusionDictionary.
+ *
+ * All the methods in this class are static.
+ */
+public class BinaryDictEncoder {
+
+ private static final boolean DBG = MakedictLog.DBG;
+
+ private BinaryDictEncoder() {
+ // This utility class is not publicly instantiable.
+ }
+
+ // Arbitrary limit to how much passes we consider address size compression should
+ // terminate in. At the time of this writing, our largest dictionary completes
+ // compression in five passes.
+ // If the number of passes exceeds this number, makedict bails with an exception on
+ // suspicion that a bug might be causing an infinite loop.
+ private static final int MAX_PASSES = 24;
+
+ /**
+ * Compute the binary size of the character array.
+ *
+ * If only one character, this is the size of this character. If many, it's the sum of their
+ * sizes + 1 byte for the terminator.
+ *
+ * @param characters the character array
+ * @return the size of the char array, including the terminator if any
+ */
+ static int getGroupCharactersSize(final int[] characters) {
+ int size = CharEncoding.getCharArraySize(characters);
+ if (characters.length > 1) size += FormatSpec.GROUP_TERMINATOR_SIZE;
+ return size;
+ }
+
+ /**
+ * Compute the binary size of the character array in a group
+ *
+ * If only one character, this is the size of this character. If many, it's the sum of their
+ * sizes + 1 byte for the terminator.
+ *
+ * @param group the group
+ * @return the size of the char array, including the terminator if any
+ */
+ private static int getGroupCharactersSize(final CharGroup group) {
+ return getGroupCharactersSize(group.mChars);
+ }
+
+ /**
+ * Compute the binary size of the group count for a node array.
+ * @param nodeArray the nodeArray
+ * @return the size of the group count, either 1 or 2 bytes.
+ */
+ private static int getGroupCountSize(final PtNodeArray nodeArray) {
+ return BinaryDictIOUtils.getGroupCountSize(nodeArray.mData.size());
+ }
+
+ /**
+ * Compute the size of a shortcut in bytes.
+ */
+ private static int getShortcutSize(final WeightedString shortcut) {
+ int size = FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE;
+ final String word = shortcut.mWord;
+ final int length = word.length();
+ for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
+ final int codePoint = word.codePointAt(i);
+ size += CharEncoding.getCharSize(codePoint);
+ }
+ size += FormatSpec.GROUP_TERMINATOR_SIZE;
+ return size;
+ }
+
+ /**
+ * Compute the size of a shortcut list in bytes.
+ *
+ * This is known in advance and does not change according to position in the file
+ * like address lists do.
+ */
+ static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) {
+ if (null == shortcutList) return 0;
+ int size = FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
+ for (final WeightedString shortcut : shortcutList) {
+ size += getShortcutSize(shortcut);
+ }
+ return size;
+ }
+
+ /**
+ * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything.
+ *
+ * @param group the CharGroup to compute the size of.
+ * @param options file format options.
+ * @return the maximum size of the group.
+ */
+ private static int getCharGroupMaximumSize(final CharGroup group, final FormatOptions options) {
+ int size = getGroupHeaderSize(group, options);
+ // If terminal, one byte for the frequency
+ if (group.isTerminal()) size += FormatSpec.GROUP_FREQUENCY_SIZE;
+ size += FormatSpec.GROUP_MAX_ADDRESS_SIZE; // For children address
+ size += getShortcutListSize(group.mShortcutTargets);
+ if (null != group.mBigrams) {
+ size += (FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE
+ + FormatSpec.GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE)
+ * group.mBigrams.size();
+ }
+ return size;
+ }
+
+ /**
+ * Compute the maximum size of each node of a node array, assuming 3-byte addresses for
+ * everything, and caches it in the `mCachedSize' member of the nodes; deduce the size of
+ * the containing node array, and cache it it its 'mCachedSize' member.
+ *
+ * @param nodeArray the node array to compute the maximum size of.
+ * @param options file format options.
+ */
+ private static void calculateNodeArrayMaximumSize(final PtNodeArray nodeArray,
+ final FormatOptions options) {
+ int size = getGroupCountSize(nodeArray);
+ for (CharGroup g : nodeArray.mData) {
+ final int groupSize = getCharGroupMaximumSize(g, options);
+ g.mCachedSize = groupSize;
+ size += groupSize;
+ }
+ if (options.mSupportsDynamicUpdate) {
+ size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
+ }
+ nodeArray.mCachedSize = size;
+ }
+
+ /**
+ * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup.
+ *
+ * @param group the group of which to compute the size of the header
+ * @param options file format options.
+ */
+ private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) {
+ if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
+ return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
+ + getGroupCharactersSize(group);
+ } else {
+ return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group);
+ }
+ }
+
+ /**
+ * Compute the size, in bytes, that an address will occupy.
+ *
+ * This can be used either for children addresses (which are always positive) or for
+ * attribute, which may be positive or negative but
+ * store their sign bit separately.
+ *
+ * @param address the address
+ * @return the byte size.
+ */
+ static int getByteSize(final int address) {
+ assert(address <= FormatSpec.UINT24_MAX);
+ if (!BinaryDictIOUtils.hasChildrenAddress(address)) {
+ return 0;
+ } else if (Math.abs(address) <= FormatSpec.UINT8_MAX) {
+ return 1;
+ } else if (Math.abs(address) <= FormatSpec.UINT16_MAX) {
+ return 2;
+ } else {
+ return 3;
+ }
+ }
+
+ // End utility methods
+
+ // This method is responsible for finding a nice ordering of the nodes that favors run-time
+ // cache performance and dictionary size.
+ /* package for tests */ static ArrayList<PtNodeArray> flattenTree(
+ final PtNodeArray rootNodeArray) {
+ final int treeSize = FusionDictionary.countCharGroups(rootNodeArray);
+ MakedictLog.i("Counted nodes : " + treeSize);
+ final ArrayList<PtNodeArray> flatTree = new ArrayList<PtNodeArray>(treeSize);
+ return flattenTreeInner(flatTree, rootNodeArray);
+ }
+
+ private static ArrayList<PtNodeArray> flattenTreeInner(final ArrayList<PtNodeArray> list,
+ final PtNodeArray nodeArray) {
+ // Removing the node is necessary if the tails are merged, because we would then
+ // add the same node several times when we only want it once. A number of places in
+ // the code also depends on any node being only once in the list.
+ // Merging tails can only be done if there are no attributes. Searching for attributes
+ // in LatinIME code depends on a total breadth-first ordering, which merging tails
+ // breaks. If there are no attributes, it should be fine (and reduce the file size)
+ // to merge tails, and removing the node from the list would be necessary. However,
+ // we don't merge tails because breaking the breadth-first ordering would result in
+ // extreme overhead at bigram lookup time (it would make the search function O(n) instead
+ // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty
+ // high).
+ // If no nodes are ever merged, we can't have the same node twice in the list, hence
+ // searching for duplicates in unnecessary. It is also very performance consuming,
+ // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making
+ // this simple list.remove operation O(n*n) overall. On Android this overhead is very
+ // high.
+ // For future reference, the code to remove duplicate is a simple : list.remove(node);
+ list.add(nodeArray);
+ final ArrayList<CharGroup> branches = nodeArray.mData;
+ final int nodeSize = branches.size();
+ for (CharGroup group : branches) {
+ if (null != group.mChildren) flattenTreeInner(list, group.mChildren);
+ }
+ return list;
+ }
+
+ /**
+ * Get the offset from a position inside a current node array to a target node array, during
+ * update.
+ *
+ * If the current node array is before the target node array, the target node array has not
+ * been updated yet, so we should return the offset from the old position of the current node
+ * array to the old position of the target node array. If on the other hand the target is
+ * before the current node array, it already has been updated, so we should return the offset
+ * from the new position in the current node array to the new position in the target node
+ * array.
+ *
+ * @param currentNodeArray node array containing the CharGroup where the offset will be written
+ * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray
+ * @param targetNodeArray the target node array to get the offset to
+ * @return the offset to the target node array
+ */
+ private static int getOffsetToTargetNodeArrayDuringUpdate(final PtNodeArray currentNodeArray,
+ final int offsetFromStartOfCurrentNodeArray, final PtNodeArray targetNodeArray) {
+ final boolean isTargetBeforeCurrent = (targetNodeArray.mCachedAddressBeforeUpdate
+ < currentNodeArray.mCachedAddressBeforeUpdate);
+ if (isTargetBeforeCurrent) {
+ return targetNodeArray.mCachedAddressAfterUpdate
+ - (currentNodeArray.mCachedAddressAfterUpdate
+ + offsetFromStartOfCurrentNodeArray);
+ } else {
+ return targetNodeArray.mCachedAddressBeforeUpdate
+ - (currentNodeArray.mCachedAddressBeforeUpdate
+ + offsetFromStartOfCurrentNodeArray);
+ }
+ }
+
+ /**
+ * Get the offset from a position inside a current node array to a target CharGroup, during
+ * update.
+ *
+ * @param currentNodeArray node array containing the CharGroup where the offset will be written
+ * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray
+ * @param targetCharGroup the target CharGroup to get the offset to
+ * @return the offset to the target CharGroup
+ */
+ // TODO: is there any way to factorize this method with the one above?
+ private static int getOffsetToTargetCharGroupDuringUpdate(final PtNodeArray currentNodeArray,
+ final int offsetFromStartOfCurrentNodeArray, final CharGroup targetCharGroup) {
+ final int oldOffsetBasePoint = currentNodeArray.mCachedAddressBeforeUpdate
+ + offsetFromStartOfCurrentNodeArray;
+ final boolean isTargetBeforeCurrent = (targetCharGroup.mCachedAddressBeforeUpdate
+ < oldOffsetBasePoint);
+ // If the target is before the current node array, then its address has already been
+ // updated. We can use the AfterUpdate member, and compare it to our own member after
+ // update. Otherwise, the AfterUpdate member is not updated yet, so we need to use the
+ // BeforeUpdate member, and of course we have to compare this to our own address before
+ // update.
+ if (isTargetBeforeCurrent) {
+ final int newOffsetBasePoint = currentNodeArray.mCachedAddressAfterUpdate
+ + offsetFromStartOfCurrentNodeArray;
+ return targetCharGroup.mCachedAddressAfterUpdate - newOffsetBasePoint;
+ } else {
+ return targetCharGroup.mCachedAddressBeforeUpdate - oldOffsetBasePoint;
+ }
+ }
+
+ /**
+ * Computes the actual node array size, based on the cached addresses of the children nodes.
+ *
+ * Each node array stores its tentative address. During dictionary address computing, these
+ * are not final, but they can be used to compute the node array size (the node array size
+ * depends on the address of the children because the number of bytes necessary to store an
+ * address depends on its numeric value. The return value indicates whether the node array
+ * contents (as in, any of the addresses stored in the cache fields) have changed with
+ * respect to their previous value.
+ *
+ * @param nodeArray the node array to compute the size of.
+ * @param dict the dictionary in which the word/attributes are to be found.
+ * @param formatOptions file format options.
+ * @return false if none of the cached addresses inside the node array changed, true otherwise.
+ */
+ private static boolean computeActualNodeArraySize(final PtNodeArray nodeArray,
+ final FusionDictionary dict, final FormatOptions formatOptions) {
+ boolean changed = false;
+ int size = getGroupCountSize(nodeArray);
+ for (CharGroup group : nodeArray.mData) {
+ group.mCachedAddressAfterUpdate = nodeArray.mCachedAddressAfterUpdate + size;
+ if (group.mCachedAddressAfterUpdate != group.mCachedAddressBeforeUpdate) {
+ changed = true;
+ }
+ int groupSize = getGroupHeaderSize(group, formatOptions);
+ if (group.isTerminal()) groupSize += FormatSpec.GROUP_FREQUENCY_SIZE;
+ if (null == group.mChildren && formatOptions.mSupportsDynamicUpdate) {
+ groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
+ } else if (null != group.mChildren) {
+ if (formatOptions.mSupportsDynamicUpdate) {
+ groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
+ } else {
+ groupSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(nodeArray,
+ groupSize + size, group.mChildren));
+ }
+ }
+ groupSize += getShortcutListSize(group.mShortcutTargets);
+ if (null != group.mBigrams) {
+ for (WeightedString bigram : group.mBigrams) {
+ final int offset = getOffsetToTargetCharGroupDuringUpdate(nodeArray,
+ groupSize + size + FormatSpec.GROUP_FLAGS_SIZE,
+ FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord));
+ groupSize += getByteSize(offset) + FormatSpec.GROUP_FLAGS_SIZE;
+ }
+ }
+ group.mCachedSize = groupSize;
+ size += groupSize;
+ }
+ if (formatOptions.mSupportsDynamicUpdate) {
+ size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
+ }
+ if (nodeArray.mCachedSize != size) {
+ nodeArray.mCachedSize = size;
+ changed = true;
+ }
+ return changed;
+ }
+
+ /**
+ * Initializes the cached addresses of node arrays and their containing nodes from their size.
+ *
+ * @param flatNodes the list of node arrays.
+ * @param formatOptions file format options.
+ * @return the byte size of the entire stack.
+ */
+ private static int initializeNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes,
+ final FormatOptions formatOptions) {
+ int nodeArrayOffset = 0;
+ for (final PtNodeArray nodeArray : flatNodes) {
+ nodeArray.mCachedAddressBeforeUpdate = nodeArrayOffset;
+ int groupCountSize = getGroupCountSize(nodeArray);
+ int groupOffset = 0;
+ for (final CharGroup g : nodeArray.mData) {
+ g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate =
+ groupCountSize + nodeArrayOffset + groupOffset;
+ groupOffset += g.mCachedSize;
+ }
+ final int nodeSize = groupCountSize + groupOffset
+ + (formatOptions.mSupportsDynamicUpdate
+ ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0);
+ nodeArrayOffset += nodeArray.mCachedSize;
+ }
+ return nodeArrayOffset;
+ }
+
+ /**
+ * Updates the cached addresses of node arrays after recomputing their new positions.
+ *
+ * @param flatNodes the list of node arrays.
+ */
+ private static void updateNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes) {
+ for (final PtNodeArray nodeArray : flatNodes) {
+ nodeArray.mCachedAddressBeforeUpdate = nodeArray.mCachedAddressAfterUpdate;
+ for (final CharGroup g : nodeArray.mData) {
+ g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate;
+ }
+ }
+ }
+
+ /**
+ * Compute the cached parent addresses after all has been updated.
+ *
+ * The parent addresses are used by some binary formats at write-to-disk time. Not all formats
+ * need them. In particular, version 2 does not need them, and version 3 does.
+ *
+ * @param flatNodes the flat array of node arrays to fill in
+ */
+ private static void computeParentAddresses(final ArrayList<PtNodeArray> flatNodes) {
+ for (final PtNodeArray nodeArray : flatNodes) {
+ for (final CharGroup group : nodeArray.mData) {
+ if (null != group.mChildren) {
+ // Assign my address to children's parent address
+ // Here BeforeUpdate and AfterUpdate addresses have the same value, so it
+ // does not matter which we use.
+ group.mChildren.mCachedParentAddress = group.mCachedAddressAfterUpdate
+ - group.mChildren.mCachedAddressAfterUpdate;
+ }
+ }
+ }
+ }
+
+ /**
+ * Compute the addresses and sizes of an ordered list of node arrays.
+ *
+ * This method takes a list of node arrays and will update their cached address and size
+ * values so that they can be written into a file. It determines the smallest size each of the
+ * nodes arrays can be given the addresses of its children and attributes, and store that into
+ * each node.
+ * The order of the node is given by the order of the array. This method makes no effort
+ * to find a good order; it only mechanically computes the size this order results in.
+ *
+ * @param dict the dictionary
+ * @param flatNodes the ordered list of nodes arrays
+ * @param formatOptions file format options.
+ * @return the same array it was passed. The nodes have been updated for address and size.
+ */
+ private static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict,
+ final ArrayList<PtNodeArray> flatNodes, final FormatOptions formatOptions) {
+ // First get the worst possible sizes and offsets
+ for (final PtNodeArray n : flatNodes) calculateNodeArrayMaximumSize(n, formatOptions);
+ final int offset = initializeNodeArraysCachedAddresses(flatNodes, formatOptions);
+
+ MakedictLog.i("Compressing the array addresses. Original size : " + offset);
+ MakedictLog.i("(Recursively seen size : " + offset + ")");
+
+ int passes = 0;
+ boolean changesDone = false;
+ do {
+ changesDone = false;
+ int nodeArrayStartOffset = 0;
+ for (final PtNodeArray nodeArray : flatNodes) {
+ nodeArray.mCachedAddressAfterUpdate = nodeArrayStartOffset;
+ final int oldNodeArraySize = nodeArray.mCachedSize;
+ final boolean changed = computeActualNodeArraySize(nodeArray, dict, formatOptions);
+ final int newNodeArraySize = nodeArray.mCachedSize;
+ if (oldNodeArraySize < newNodeArraySize) {
+ throw new RuntimeException("Increased size ?!");
+ }
+ nodeArrayStartOffset += newNodeArraySize;
+ changesDone |= changed;
+ }
+ updateNodeArraysCachedAddresses(flatNodes);
+ ++passes;
+ if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug");
+ } while (changesDone);
+
+ if (formatOptions.mSupportsDynamicUpdate) {
+ computeParentAddresses(flatNodes);
+ }
+ final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
+ MakedictLog.i("Compression complete in " + passes + " passes.");
+ MakedictLog.i("After address compression : "
+ + (lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize));
+
+ return flatNodes;
+ }
+
+ /**
+ * Sanity-checking method.
+ *
+ * This method checks a list of node arrays for juxtaposition, that is, it will do
+ * nothing if each node array's cached address is actually the previous node array's address
+ * plus the previous node's size.
+ * If this is not the case, it will throw an exception.
+ *
+ * @param arrays the list of node arrays to check
+ */
+ private static void checkFlatNodeArrayList(final ArrayList<PtNodeArray> arrays) {
+ int offset = 0;
+ int index = 0;
+ for (final PtNodeArray nodeArray : arrays) {
+ // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter
+ // which we use.
+ if (nodeArray.mCachedAddressAfterUpdate != offset) {
+ throw new RuntimeException("Wrong address for node " + index
+ + " : expected " + offset + ", got " + nodeArray.mCachedAddressAfterUpdate);
+ }
+ ++index;
+ offset += nodeArray.mCachedSize;
+ }
+ }
+
+ /**
+ * Helper method to write a variable-size address to a file.
+ *
+ * @param buffer the buffer to write to.
+ * @param index the index in the buffer to write the address to.
+ * @param address the address to write.
+ * @return the size in bytes the address actually took.
+ */
+ private static int writeVariableAddress(final byte[] buffer, int index, final int address) {
+ switch (getByteSize(address)) {
+ case 1:
+ buffer[index++] = (byte)address;
+ return 1;
+ case 2:
+ buffer[index++] = (byte)(0xFF & (address >> 8));
+ buffer[index++] = (byte)(0xFF & address);
+ return 2;
+ case 3:
+ buffer[index++] = (byte)(0xFF & (address >> 16));
+ buffer[index++] = (byte)(0xFF & (address >> 8));
+ buffer[index++] = (byte)(0xFF & address);
+ return 3;
+ case 0:
+ return 0;
+ default:
+ throw new RuntimeException("Address " + address + " has a strange size");
+ }
+ }
+
+ /**
+ * Helper method to write a variable-size signed address to a file.
+ *
+ * @param buffer the buffer to write to.
+ * @param index the index in the buffer to write the address to.
+ * @param address the address to write.
+ * @return the size in bytes the address actually took.
+ */
+ private static int writeVariableSignedAddress(final byte[] buffer, int index,
+ final int address) {
+ if (!BinaryDictIOUtils.hasChildrenAddress(address)) {
+ buffer[index] = buffer[index + 1] = buffer[index + 2] = 0;
+ } else {
+ final int absAddress = Math.abs(address);
+ buffer[index++] =
+ (byte)((address < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absAddress >> 16)));
+ buffer[index++] = (byte)(0xFF & (absAddress >> 8));
+ buffer[index++] = (byte)(0xFF & absAddress);
+ }
+ return 3;
+ }
+
+ /**
+ * Makes the flag value for a char group.
+ *
+ * @param hasMultipleChars whether the group has multiple chars.
+ * @param isTerminal whether the group is terminal.
+ * @param childrenAddressSize the size of a children address.
+ * @param hasShortcuts whether the group has shortcuts.
+ * @param hasBigrams whether the group has bigrams.
+ * @param isNotAWord whether the group is not a word.
+ * @param isBlackListEntry whether the group is a blacklist entry.
+ * @param formatOptions file format options.
+ * @return the flags
+ */
+ static int makeCharGroupFlags(final boolean hasMultipleChars, final boolean isTerminal,
+ final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams,
+ final boolean isNotAWord, final boolean isBlackListEntry,
+ final FormatOptions formatOptions) {
+ byte flags = 0;
+ if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS;
+ if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL;
+ if (formatOptions.mSupportsDynamicUpdate) {
+ flags |= FormatSpec.FLAG_IS_NOT_MOVED;
+ } else if (true) {
+ switch (childrenAddressSize) {
+ case 1:
+ flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE;
+ break;
+ case 2:
+ flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES;
+ break;
+ case 3:
+ flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES;
+ break;
+ case 0:
+ flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS;
+ break;
+ default:
+ throw new RuntimeException("Node with a strange address");
+ }
+ }
+ if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS;
+ if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS;
+ if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD;
+ if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED;
+ return flags;
+ }
+
+ private static byte makeCharGroupFlags(final CharGroup group, final int groupAddress,
+ final int childrenOffset, final FormatOptions formatOptions) {
+ return (byte) makeCharGroupFlags(group.mChars.length > 1, group.mFrequency >= 0,
+ getByteSize(childrenOffset), group.mShortcutTargets != null, group.mBigrams != null,
+ group.mIsNotAWord, group.mIsBlacklistEntry, formatOptions);
+ }
+
+ /**
+ * Makes the flag value for a bigram.
+ *
+ * @param more whether there are more bigrams after this one.
+ * @param offset the offset of the bigram.
+ * @param bigramFrequency the frequency of the bigram, 0..255.
+ * @param unigramFrequency the unigram frequency of the same word, 0..255.
+ * @param word the second bigram, for debugging purposes
+ * @return the flags
+ */
+ private static final int makeBigramFlags(final boolean more, final int offset,
+ int bigramFrequency, final int unigramFrequency, final String word) {
+ int bigramFlags = (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0)
+ + (offset < 0 ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0);
+ switch (getByteSize(offset)) {
+ case 1:
+ bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
+ break;
+ case 2:
+ bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES;
+ break;
+ case 3:
+ bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES;
+ break;
+ default:
+ throw new RuntimeException("Strange offset size");
+ }
+ if (unigramFrequency > bigramFrequency) {
+ MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word
+ + "\". Bigram freq is " + bigramFrequency + ", unigram freq for "
+ + word + " is " + unigramFrequency);
+ bigramFrequency = unigramFrequency;
+ }
+ // We compute the difference between 255 (which means probability = 1) and the
+ // unigram score. We split this into a number of discrete steps.
+ // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15
+ // represents an increase of 16 steps: a value of 15 will be interpreted as the median
+ // value of the 16th step. In all justice, if the bigram frequency is low enough to be
+ // rounded below the first step (which means it is less than half a step higher than the
+ // unigram frequency) then the unigram frequency itself is the best approximation of the
+ // bigram freq that we could possibly supply, hence we should *not* include this bigram
+ // in the file at all.
+ // until this is done, we'll write 0 and slightly overestimate this case.
+ // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step
+ // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to
+ // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the
+ // step size. Then we compute the start of the first step (the one where value 0 starts)
+ // by adding half-a-step to the unigramFrequency. From there, we compute the integer
+ // number of steps to the bigramFrequency. One last thing: we want our steps to include
+ // their lower bound and exclude their higher bound so we need to have the first step
+ // start at exactly 1 unit higher than floor(unigramFreq + half a step).
+ // Note : to reconstruct the score, the dictionary reader will need to divide
+ // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step,
+ // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best
+ // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the
+ // step pointed by the discretized frequency.
+ final float stepSize =
+ (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
+ / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
+ final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f);
+ final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize);
+ // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1
+ // here. The best approximation would be the unigram freq itself, so we should not
+ // include this bigram in the dictionary. For now, register as 0, and live with the
+ // small over-estimation that we get in this case. TODO: actually remove this bigram
+ // if discretizedFrequency < 0.
+ final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0;
+ bigramFlags += finalBigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY;
+ return bigramFlags;
+ }
+
+ /**
+ * Makes the 2-byte value for options flags.
+ */
+ private static final int makeOptionsValue(final FusionDictionary dictionary,
+ final FormatOptions formatOptions) {
+ final DictionaryOptions options = dictionary.mOptions;
+ final boolean hasBigrams = dictionary.hasBigrams();
+ return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0)
+ + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0)
+ + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0)
+ + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0);
+ }
+
+ /**
+ * Makes the flag value for a shortcut.
+ *
+ * @param more whether there are more attributes after this one.
+ * @param frequency the frequency of the attribute, 0..15
+ * @return the flags
+ */
+ static final int makeShortcutFlags(final boolean more, final int frequency) {
+ return (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0)
+ + (frequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY);
+ }
+
+ private static final int writeParentAddress(final byte[] buffer, final int index,
+ final int address, final FormatOptions formatOptions) {
+ if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
+ if (address == FormatSpec.NO_PARENT_ADDRESS) {
+ buffer[index] = buffer[index + 1] = buffer[index + 2] = 0;
+ } else {
+ final int absAddress = Math.abs(address);
+ assert(absAddress <= FormatSpec.SINT24_MAX);
+ buffer[index] = (byte)((address < 0 ? FormatSpec.MSB8 : 0)
+ | ((absAddress >> 16) & 0xFF));
+ buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF);
+ buffer[index + 2] = (byte)(absAddress & 0xFF);
+ }
+ return index + 3;
+ } else {
+ return index;
+ }
+ }
+
+ /**
+ * Write a node array to memory. The node array is expected to have its final position cached.
+ *
+ * @param dict the dictionary the node array is a part of (for relative offsets).
+ * @param buffer the memory buffer to write to.
+ * @param nodeArray the node array to write.
+ * @param formatOptions file format options.
+ * @return the address of the END of the node.
+ */
+ @SuppressWarnings("unused")
+ private static int writePlacedNode(final FusionDictionary dict, byte[] buffer,
+ final PtNodeArray nodeArray, final FormatOptions formatOptions) {
+ // TODO: Make the code in common with BinaryDictIOUtils#writeCharGroup
+ int index = nodeArray.mCachedAddressAfterUpdate;
+
+ final int groupCount = nodeArray.mData.size();
+ final int countSize = getGroupCountSize(nodeArray);
+ final int parentAddress = nodeArray.mCachedParentAddress;
+ if (1 == countSize) {
+ buffer[index++] = (byte)groupCount;
+ } else if (2 == countSize) {
+ // We need to signal 2-byte size by setting the top bit of the MSB to 1, so
+ // we | 0x80 to do this.
+ buffer[index++] = (byte)((groupCount >> 8) | 0x80);
+ buffer[index++] = (byte)(groupCount & 0xFF);
+ } else {
+ throw new RuntimeException("Strange size from getGroupCountSize : " + countSize);
+ }
+ int groupAddress = index;
+ for (int i = 0; i < groupCount; ++i) {
+ final CharGroup group = nodeArray.mData.get(i);
+ if (index != group.mCachedAddressAfterUpdate) {
+ throw new RuntimeException("Bug: write index is not the same as the cached address "
+ + "of the group : " + index + " <> " + group.mCachedAddressAfterUpdate);
+ }
+ groupAddress += getGroupHeaderSize(group, formatOptions);
+ // Sanity checks.
+ if (DBG && group.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) {
+ throw new RuntimeException("A node has a frequency > "
+ + FormatSpec.MAX_TERMINAL_FREQUENCY
+ + " : " + group.mFrequency);
+ }
+ if (group.mFrequency >= 0) groupAddress += FormatSpec.GROUP_FREQUENCY_SIZE;
+ final int childrenOffset = null == group.mChildren
+ ? FormatSpec.NO_CHILDREN_ADDRESS
+ : group.mChildren.mCachedAddressAfterUpdate - groupAddress;
+ buffer[index++] =
+ makeCharGroupFlags(group, groupAddress, childrenOffset, formatOptions);
+
+ if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) {
+ index = writeParentAddress(buffer, index, parentAddress, formatOptions);
+ } else {
+ index = writeParentAddress(buffer, index, parentAddress
+ + (nodeArray.mCachedAddressAfterUpdate - group.mCachedAddressAfterUpdate),
+ formatOptions);
+ }
+
+ index = CharEncoding.writeCharArray(group.mChars, buffer, index);
+ if (group.hasSeveralChars()) {
+ buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
+ }
+ if (group.mFrequency >= 0) {
+ buffer[index++] = (byte) group.mFrequency;
+ }
+
+ final int shift;
+ if (formatOptions.mSupportsDynamicUpdate) {
+ shift = writeVariableSignedAddress(buffer, index, childrenOffset);
+ } else {
+ shift = writeVariableAddress(buffer, index, childrenOffset);
+ }
+ index += shift;
+ groupAddress += shift;
+
+ // Write shortcuts
+ if (null != group.mShortcutTargets) {
+ final int indexOfShortcutByteSize = index;
+ index += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
+ groupAddress += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
+ final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator();
+ while (shortcutIterator.hasNext()) {
+ final WeightedString target = shortcutIterator.next();
+ ++groupAddress;
+ int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(),
+ target.mFrequency);
+ buffer[index++] = (byte)shortcutFlags;
+ final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord);
+ index += shortcutShift;
+ groupAddress += shortcutShift;
+ }
+ final int shortcutByteSize = index - indexOfShortcutByteSize;
+ if (shortcutByteSize > 0xFFFF) {
+ throw new RuntimeException("Shortcut list too large");
+ }
+ buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8);
+ buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF);
+ }
+ // Write bigrams
+ if (null != group.mBigrams) {
+ final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator();
+ while (bigramIterator.hasNext()) {
+ final WeightedString bigram = bigramIterator.next();
+ final CharGroup target =
+ FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord);
+ final int addressOfBigram = target.mCachedAddressAfterUpdate;
+ final int unigramFrequencyForThisWord = target.mFrequency;
+ ++groupAddress;
+ final int offset = addressOfBigram - groupAddress;
+ int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset,
+ bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
+ buffer[index++] = (byte)bigramFlags;
+ final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset));
+ index += bigramShift;
+ groupAddress += bigramShift;
+ }
+ }
+
+ }
+ if (formatOptions.mSupportsDynamicUpdate) {
+ buffer[index] = buffer[index + 1] = buffer[index + 2]
+ = FormatSpec.NO_FORWARD_LINK_ADDRESS;
+ index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
+ }
+ if (index != nodeArray.mCachedAddressAfterUpdate + nodeArray.mCachedSize) {
+ throw new RuntimeException(
+ "Not the same size : written " + (index - nodeArray.mCachedAddressAfterUpdate)
+ + " bytes from a node that should have " + nodeArray.mCachedSize + " bytes");
+ }
+ return index;
+ }
+
+ /**
+ * Dumps a collection of useful statistics about a list of node arrays.
+ *
+ * This prints purely informative stuff, like the total estimated file size, the
+ * number of node arrays, of character groups, the repartition of each address size, etc
+ *
+ * @param nodeArrays the list of node arrays.
+ */
+ private static void showStatistics(ArrayList<PtNodeArray> nodeArrays) {
+ int firstTerminalAddress = Integer.MAX_VALUE;
+ int lastTerminalAddress = Integer.MIN_VALUE;
+ int size = 0;
+ int charGroups = 0;
+ int maxGroups = 0;
+ int maxRuns = 0;
+ for (final PtNodeArray nodeArray : nodeArrays) {
+ if (maxGroups < nodeArray.mData.size()) maxGroups = nodeArray.mData.size();
+ for (final CharGroup cg : nodeArray.mData) {
+ ++charGroups;
+ if (cg.mChars.length > maxRuns) maxRuns = cg.mChars.length;
+ if (cg.mFrequency >= 0) {
+ if (nodeArray.mCachedAddressAfterUpdate < firstTerminalAddress)
+ firstTerminalAddress = nodeArray.mCachedAddressAfterUpdate;
+ if (nodeArray.mCachedAddressAfterUpdate > lastTerminalAddress)
+ lastTerminalAddress = nodeArray.mCachedAddressAfterUpdate;
+ }
+ }
+ if (nodeArray.mCachedAddressAfterUpdate + nodeArray.mCachedSize > size) {
+ size = nodeArray.mCachedAddressAfterUpdate + nodeArray.mCachedSize;
+ }
+ }
+ final int[] groupCounts = new int[maxGroups + 1];
+ final int[] runCounts = new int[maxRuns + 1];
+ for (final PtNodeArray nodeArray : nodeArrays) {
+ ++groupCounts[nodeArray.mData.size()];
+ for (final CharGroup cg : nodeArray.mData) {
+ ++runCounts[cg.mChars.length];
+ }
+ }
+
+ MakedictLog.i("Statistics:\n"
+ + " total file size " + size + "\n"
+ + " " + nodeArrays.size() + " node arrays\n"
+ + " " + charGroups + " groups (" + ((float)charGroups / nodeArrays.size())
+ + " groups per node)\n"
+ + " first terminal at " + firstTerminalAddress + "\n"
+ + " last terminal at " + lastTerminalAddress + "\n"
+ + " Group stats : max = " + maxGroups);
+ for (int i = 0; i < groupCounts.length; ++i) {
+ MakedictLog.i(" " + i + " : " + groupCounts[i]);
+ }
+ MakedictLog.i(" Character run stats : max = " + maxRuns);
+ for (int i = 0; i < runCounts.length; ++i) {
+ MakedictLog.i(" " + i + " : " + runCounts[i]);
+ }
+ }
+
+ /**
+ * Dumps a FusionDictionary to a file.
+ *
+ * This is the public entry point to write a dictionary to a file.
+ *
+ * @param destination the stream to write the binary data to.
+ * @param dict the dictionary to write.
+ * @param formatOptions file format options.
+ */
+ public static void writeDictionaryBinary(final OutputStream destination,
+ final FusionDictionary dict, final FormatOptions formatOptions)
+ throws IOException, UnsupportedFormatException {
+
+ // Addresses are limited to 3 bytes, but since addresses can be relative to each node
+ // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding
+ // the order of the node arrays becomes a quite complicated problem, because though the
+ // dictionary itself does not have a size limit, each node array must still be within 16MB
+ // of all its children and parents. As long as this is ensured, the dictionary file may
+ // grow to any size.
+
+ final int version = formatOptions.mVersion;
+ if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
+ || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
+ throw new UnsupportedFormatException("Requested file format version " + version
+ + ", but this implementation only supports versions "
+ + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through "
+ + FormatSpec.MAXIMUM_SUPPORTED_VERSION);
+ }
+
+ ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256);
+
+ // The magic number in big-endian order.
+ // Magic number for all versions.
+ headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 24)));
+ headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 16)));
+ headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 8)));
+ headerBuffer.write((byte) (0xFF & FormatSpec.MAGIC_NUMBER));
+ // Dictionary version.
+ headerBuffer.write((byte) (0xFF & (version >> 8)));
+ headerBuffer.write((byte) (0xFF & version));
+
+ // Options flags
+ final int options = makeOptionsValue(dict, formatOptions);
+ headerBuffer.write((byte) (0xFF & (options >> 8)));
+ headerBuffer.write((byte) (0xFF & options));
+ final int headerSizeOffset = headerBuffer.size();
+ // Placeholder to be written later with header size.
+ for (int i = 0; i < 4; ++i) {
+ headerBuffer.write(0);
+ }
+ // Write out the options.
+ for (final String key : dict.mOptions.mAttributes.keySet()) {
+ final String value = dict.mOptions.mAttributes.get(key);
+ CharEncoding.writeString(headerBuffer, key);
+ CharEncoding.writeString(headerBuffer, value);
+ }
+ final int size = headerBuffer.size();
+ final byte[] bytes = headerBuffer.toByteArray();
+ // Write out the header size.
+ bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24));
+ bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16));
+ bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8));
+ bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0));
+ destination.write(bytes);
+
+ headerBuffer.close();
+
+ // Leave the choice of the optimal node order to the flattenTree function.
+ MakedictLog.i("Flattening the tree...");
+ ArrayList<PtNodeArray> flatNodes = flattenTree(dict.mRootNodeArray);
+
+ MakedictLog.i("Computing addresses...");
+ computeAddresses(dict, flatNodes, formatOptions);
+ MakedictLog.i("Checking array...");
+ if (DBG) checkFlatNodeArrayList(flatNodes);
+
+ // Create a buffer that matches the final dictionary size.
+ final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
+ final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize;
+ final byte[] buffer = new byte[bufferSize];
+ int index = 0;
+
+ MakedictLog.i("Writing file...");
+ int dataEndOffset = 0;
+ for (PtNodeArray nodeArray : flatNodes) {
+ dataEndOffset = writePlacedNode(dict, buffer, nodeArray, formatOptions);
+ }
+
+ if (DBG) showStatistics(flatNodes);
+
+ destination.write(buffer, 0, dataEndOffset);
+
+ destination.close();
+ MakedictLog.i("Done");
+ }
+}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
index 2e6c4b2..e5735aa 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
@@ -18,19 +18,19 @@
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.CharEncoding;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.CharEncoding;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
-import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
@@ -59,7 +59,7 @@
}
/**
- * Tours all node without recursive call.
+ * Retrieves all node arrays without recursive call.
*/
private static void readUnigramsAndBigramsBinaryInner(
final FusionDictionaryBufferInterface buffer, final int headerSize,
@@ -86,7 +86,7 @@
if (index != p.mLength) index = p.mLength;
if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) {
- p.mNumOfCharGroup = BinaryDictInputOutput.readCharGroupCount(buffer);
+ p.mNumOfCharGroup = BinaryDictDecoder.readCharGroupCount(buffer);
p.mAddress += getGroupCountSize(p.mNumOfCharGroup);
p.mPosition = 0;
}
@@ -94,7 +94,7 @@
stack.pop();
continue;
}
- CharGroupInfo info = BinaryDictInputOutput.readCharGroup(buffer,
+ CharGroupInfo info = BinaryDictDecoder.readCharGroup(buffer,
p.mAddress - headerSize, formatOptions);
for (int i = 0; i < info.mCharacters.length; ++i) {
pushedChars[index++] = info.mCharacters[i];
@@ -116,7 +116,7 @@
if (formatOptions.mSupportsDynamicUpdate) {
final int forwardLinkAddress = buffer.readUnsignedInt24();
if (forwardLinkAddress != FormatSpec.NO_FORWARD_LINK_ADDRESS) {
- // the node has a forward link.
+ // The node array has a forward link.
p.mNumOfCharGroup = Position.NOT_READ_GROUPCOUNT;
p.mAddress = forwardLinkAddress;
} else {
@@ -126,7 +126,7 @@
stack.pop();
}
} else {
- // the node has more groups.
+ // The node array has more groups.
p.mAddress = buffer.position();
}
@@ -139,22 +139,22 @@
/**
* Reads unigrams and bigrams from the binary file.
- * Doesn't make the memory representation of the dictionary.
+ * Doesn't store a full memory representation of the dictionary.
*
- * @param reader the reader.
+ * @param dictReader the dict reader.
* @param words the map to store the address as a key and the word as a value.
* @param frequencies the map to store the address as a key and the frequency as a value.
* @param bigrams the map to store the address as a key and the list of address as a value.
- * @throws IOException
- * @throws UnsupportedFormatException
+ * @throws IOException if the file can't be read.
+ * @throws UnsupportedFormatException if the format of the file is not recognized.
*/
- public static void readUnigramsAndBigramsBinary(final BinaryDictReader reader,
+ public static void readUnigramsAndBigramsBinary(final BinaryDictReader dictReader,
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
UnsupportedFormatException {
// Read header
- final FileHeader header = BinaryDictInputOutput.readHeader(reader.getBuffer());
- readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words,
+ final FileHeader header = BinaryDictDecoder.readHeader(dictReader);
+ readUnigramsAndBigramsBinaryInner(dictReader.getBuffer(), header.mHeaderSize, words,
frequencies, bigrams, header.mFormatOptions);
}
@@ -162,30 +162,31 @@
* Gets the address of the last CharGroup of the exact matching word in the dictionary.
* If no match is found, returns NOT_VALID_WORD.
*
- * @param buffer the buffer to read.
+ * @param dictReader the dict reader.
* @param word the word we search for.
* @return the address of the terminal node.
- * @throws IOException
- * @throws UnsupportedFormatException
+ * @throws IOException if the file can't be read.
+ * @throws UnsupportedFormatException if the format of the file is not recognized.
*/
@UsedForTesting
- public static int getTerminalPosition(final FusionDictionaryBufferInterface buffer,
+ public static int getTerminalPosition(final BinaryDictReader dictReader,
final String word) throws IOException, UnsupportedFormatException {
+ final FusionDictionaryBufferInterface buffer = dictReader.getBuffer();
if (word == null) return FormatSpec.NOT_VALID_WORD;
if (buffer.position() != 0) buffer.position(0);
- final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
+ final FileHeader header = BinaryDictDecoder.readHeader(dictReader);
int wordPos = 0;
final int wordLen = word.codePointCount(0, word.length());
for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) {
if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD;
do {
- final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer);
+ final int charGroupCount = BinaryDictDecoder.readCharGroupCount(buffer);
boolean foundNextCharGroup = false;
for (int i = 0; i < charGroupCount; ++i) {
final int charGroupPos = buffer.position();
- final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer,
+ final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer,
buffer.position(), header.mFormatOptions);
final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags,
header.mFormatOptions);
@@ -224,9 +225,9 @@
}
// If we found the next char group, it is under the file pointer.
- // But if not, we are at the end of this node so we expect to have
+ // But if not, we are at the end of this node array so we expect to have
// a forward link address that we need to consult and possibly resume
- // search on the next node in the linked list.
+ // search on the next node array in the linked list.
if (foundNextCharGroup) break;
if (!header.mFormatOptions.mSupportsDynamicUpdate) {
return FormatSpec.NOT_VALID_WORD;
@@ -271,7 +272,7 @@
*/
private static int writeVariableAddress(final OutputStream destination, final int value)
throws IOException {
- switch (BinaryDictInputOutput.getByteSize(value)) {
+ switch (BinaryDictEncoder.getByteSize(value)) {
case 1:
destination.write((byte)value);
break;
@@ -285,15 +286,15 @@
destination.write((byte)(0xFF & value));
break;
}
- return BinaryDictInputOutput.getByteSize(value);
+ return BinaryDictEncoder.getByteSize(value);
}
static void skipCharGroup(final FusionDictionaryBufferInterface buffer,
final FormatOptions formatOptions) {
final int flags = buffer.readUnsignedByte();
- BinaryDictInputOutput.readParentAddress(buffer, formatOptions);
+ BinaryDictDecoder.readParentAddress(buffer, formatOptions);
skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
- BinaryDictInputOutput.readChildrenAddress(buffer, flags, formatOptions);
+ BinaryDictDecoder.readChildrenAddress(buffer, flags, formatOptions);
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte();
if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) {
final int shortcutsSize = buffer.readUnsignedShort();
@@ -365,9 +366,10 @@
}
/**
- * Write a char group to an output stream.
- * A char group is an in-memory representation of a node in trie.
- * A char group info is an on-disk representation of a node.
+ * Write a char group to an output stream from a CharGroupInfo.
+ * A char group is an in-memory representation of a node in the patricia trie.
+ * A char group info is a container for low-level information about how the
+ * char group is stored in the binary format.
*
* @param destination the stream to write.
* @param info the char group info to be written.
@@ -411,14 +413,14 @@
if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) {
final int shortcutListSize =
- BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets);
+ BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets);
destination.write((byte)(shortcutListSize >> 8));
destination.write((byte)(shortcutListSize & 0xFF));
size += 2;
final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator();
while (shortcutIterator.hasNext()) {
final WeightedString target = shortcutIterator.next();
- destination.write((byte)BinaryDictInputOutput.makeShortcutFlags(
+ destination.write((byte)BinaryDictEncoder.makeShortcutFlags(
shortcutIterator.hasNext(), target.mFrequency));
size++;
size += writeString(destination, target.mWord);
@@ -427,7 +429,7 @@
if (info.mBigrams != null) {
// TODO: Consolidate this code with the code that computes the size of the bigram list
- // in BinaryDictionaryInputOutput#computeActualNodeSize
+ // in BinaryDictEncoder#computeActualNodeArraySize
for (int i = 0; i < info.mBigrams.size(); ++i) {
final int bigramFrequency = info.mBigrams.get(i).mFrequency;
@@ -437,7 +439,7 @@
final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress
+ size);
bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0;
- switch (BinaryDictInputOutput.getByteSize(bigramOffset)) {
+ switch (BinaryDictEncoder.getByteSize(bigramOffset)) {
case 1:
bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
break;
@@ -461,32 +463,32 @@
*/
static int computeGroupSize(final CharGroupInfo info, final FormatOptions formatOptions) {
int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
- + BinaryDictInputOutput.getGroupCharactersSize(info.mCharacters)
- + BinaryDictInputOutput.getChildrenAddressSize(info.mFlags, formatOptions);
+ + BinaryDictEncoder.getGroupCharactersSize(info.mCharacters)
+ + getChildrenAddressSize(info.mFlags, formatOptions);
if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) {
size += FormatSpec.GROUP_FREQUENCY_SIZE;
}
if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) {
- size += BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets);
+ size += BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets);
}
if (info.mBigrams != null) {
for (final PendingAttribute attr : info.mBigrams) {
size += FormatSpec.GROUP_FLAGS_SIZE;
- size += BinaryDictInputOutput.getByteSize(attr.mAddress);
+ size += BinaryDictEncoder.getByteSize(attr.mAddress);
}
}
return size;
}
/**
- * Write a node to the stream.
+ * Write a node array to the stream.
*
* @param destination the stream to write.
- * @param infos groups to be written.
+ * @param infos an array of CharGroupInfo to be written.
* @return the size written, in bytes.
* @throws IOException
*/
- static int writeNode(final OutputStream destination, final CharGroupInfo[] infos)
+ static int writeNodes(final OutputStream destination, final CharGroupInfo[] infos)
throws IOException {
int size = getGroupCountSize(infos.length);
switch (getGroupCountSize(infos.length)) {
@@ -506,23 +508,24 @@
}
/**
- * Find a word from the buffer.
+ * Find a word using the BinaryDictReader.
*
- * @param buffer the buffer representing the body of the dictionary file.
+ * @param dictReader the dict reader
* @param word the word searched
* @return the found group
* @throws IOException
* @throws UnsupportedFormatException
*/
@UsedForTesting
- public static CharGroupInfo findWordFromBuffer(final FusionDictionaryBufferInterface buffer,
+ public static CharGroupInfo findWordByBinaryDictReader(final BinaryDictReader dictReader,
final String word) throws IOException, UnsupportedFormatException {
- int position = getTerminalPosition(buffer, word);
+ int position = getTerminalPosition(dictReader, word);
+ final FusionDictionaryBufferInterface buffer = dictReader.getBuffer();
if (position != FormatSpec.NOT_VALID_WORD) {
buffer.position(0);
- final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
+ final FileHeader header = BinaryDictDecoder.readHeader(dictReader);
buffer.position(position);
- return BinaryDictInputOutput.readCharGroup(buffer, position, header.mFormatOptions);
+ return BinaryDictDecoder.readCharGroup(buffer, position, header.mFormatOptions);
}
return null;
}
@@ -541,16 +544,21 @@
final File file, final long offset, final long length)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE];
- final FileInputStream inStream = new FileInputStream(file);
- try {
- inStream.read(buffer);
- final BinaryDictInputOutput.ByteBufferWrapper wrapper =
- new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map(
- FileChannel.MapMode.READ_ONLY, offset, length));
- return BinaryDictInputOutput.readHeader(wrapper);
- } finally {
- inStream.close();
- }
+ final BinaryDictReader dictReader = new BinaryDictReader(file);
+ dictReader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFactory() {
+ @Override
+ public FusionDictionaryBufferInterface getFusionDictionaryBuffer(File file)
+ throws FileNotFoundException, IOException {
+ final FileInputStream inStream = new FileInputStream(file);
+ try {
+ inStream.read(buffer);
+ return new ByteArrayWrapper(buffer);
+ } finally {
+ inStream.close();
+ }
+ }
+ });
+ return BinaryDictDecoder.readHeader(dictReader);
}
public static FileHeader getDictionaryFileHeaderOrNull(final File file, final long offset,
@@ -604,12 +612,28 @@
public static int getGroupCountSize(final int count) {
if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= count) {
return 1;
- } else if (FormatSpec.MAX_CHARGROUPS_IN_A_NODE >= count) {
+ } else if (FormatSpec.MAX_CHARGROUPS_IN_A_PT_NODE_ARRAY >= count) {
return 2;
} else {
throw new RuntimeException("Can't have more than "
- + FormatSpec.MAX_CHARGROUPS_IN_A_NODE + " groups in a node (found " + count
- + ")");
+ + FormatSpec.MAX_CHARGROUPS_IN_A_PT_NODE_ARRAY + " groups in a node (found "
+ + count + ")");
+ }
+ }
+
+ static int getChildrenAddressSize(final int optionFlags,
+ final FormatOptions formatOptions) {
+ if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
+ switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
+ return 1;
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
+ return 2;
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
+ return 3;
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
+ default:
+ return 0;
}
}
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
deleted file mode 100644
index a33cddb..0000000
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
+++ /dev/null
@@ -1,1783 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.inputmethod.latin.makedict;
-
-import com.android.inputmethod.annotations.UsedForTesting;
-import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
-import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
-import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
-import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
-import com.android.inputmethod.latin.utils.JniUtils;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * Reads and writes XML files for a FusionDictionary.
- *
- * All the methods in this class are static.
- */
-public final class BinaryDictInputOutput {
-
- private static final boolean DBG = MakedictLog.DBG;
-
- static {
- JniUtils.loadNativeLibrary();
- }
-
- // TODO: implement something sensical instead of just a phony method
- private static native int doNothing();
-
- // Arbitrary limit to how much passes we consider address size compression should
- // terminate in. At the time of this writing, our largest dictionary completes
- // compression in five passes.
- // If the number of passes exceeds this number, makedict bails with an exception on
- // suspicion that a bug might be causing an infinite loop.
- private static final int MAX_PASSES = 24;
- private static final int MAX_JUMPS = 12;
-
- @UsedForTesting
- public interface FusionDictionaryBufferInterface {
- public int readUnsignedByte();
- public int readUnsignedShort();
- public int readUnsignedInt24();
- public int readInt();
- public int position();
- public void position(int newPosition);
- public void put(final byte b);
- public int limit();
- @UsedForTesting
- public int capacity();
- }
-
- public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
- private ByteBuffer mBuffer;
-
- public ByteBufferWrapper(final ByteBuffer buffer) {
- mBuffer = buffer;
- }
-
- @Override
- public int readUnsignedByte() {
- return mBuffer.get() & 0xFF;
- }
-
- @Override
- public int readUnsignedShort() {
- return mBuffer.getShort() & 0xFFFF;
- }
-
- @Override
- public int readUnsignedInt24() {
- final int retval = readUnsignedByte();
- return (retval << 16) + readUnsignedShort();
- }
-
- @Override
- public int readInt() {
- return mBuffer.getInt();
- }
-
- @Override
- public int position() {
- return mBuffer.position();
- }
-
- @Override
- public void position(int newPos) {
- mBuffer.position(newPos);
- }
-
- @Override
- public void put(final byte b) {
- mBuffer.put(b);
- }
-
- @Override
- public int limit() {
- return mBuffer.limit();
- }
-
- @Override
- public int capacity() {
- return mBuffer.capacity();
- }
- }
-
- /**
- * A class grouping utility function for our specific character encoding.
- */
- static final class CharEncoding {
- private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
- private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF;
-
- /**
- * Helper method to find out whether this code fits on one byte
- */
- private static boolean fitsOnOneByte(final int character) {
- return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE
- && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE;
- }
-
- /**
- * Compute the size of a character given its character code.
- *
- * Char format is:
- * 1 byte = bbbbbbbb match
- * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
- * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
- * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
- * 00011111 would be outside unicode.
- * else: iso-latin-1 code
- * This allows for the whole unicode range to be encoded, including chars outside of
- * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
- * characters which should never happen anyway (and still work, but take 3 bytes).
- *
- * @param character the character code.
- * @return the size in binary encoded-form, either 1 or 3 bytes.
- */
- static int getCharSize(final int character) {
- // See char encoding in FusionDictionary.java
- if (fitsOnOneByte(character)) return 1;
- if (FormatSpec.INVALID_CHARACTER == character) return 1;
- return 3;
- }
-
- /**
- * Compute the byte size of a character array.
- */
- private static int getCharArraySize(final int[] chars) {
- int size = 0;
- for (int character : chars) size += getCharSize(character);
- return size;
- }
-
- /**
- * Writes a char array to a byte buffer.
- *
- * @param codePoints the code point array to write.
- * @param buffer the byte buffer to write to.
- * @param index the index in buffer to write the character array to.
- * @return the index after the last character.
- */
- private static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) {
- for (int codePoint : codePoints) {
- if (1 == getCharSize(codePoint)) {
- buffer[index++] = (byte)codePoint;
- } else {
- buffer[index++] = (byte)(0xFF & (codePoint >> 16));
- buffer[index++] = (byte)(0xFF & (codePoint >> 8));
- buffer[index++] = (byte)(0xFF & codePoint);
- }
- }
- return index;
- }
-
- /**
- * Writes a string with our character format to a byte buffer.
- *
- * This will also write the terminator byte.
- *
- * @param buffer the byte buffer to write to.
- * @param origin the offset to write from.
- * @param word the string to write.
- * @return the size written, in bytes.
- */
- private static int writeString(final byte[] buffer, final int origin,
- final String word) {
- final int length = word.length();
- int index = origin;
- for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
- final int codePoint = word.codePointAt(i);
- if (1 == getCharSize(codePoint)) {
- buffer[index++] = (byte)codePoint;
- } else {
- buffer[index++] = (byte)(0xFF & (codePoint >> 16));
- buffer[index++] = (byte)(0xFF & (codePoint >> 8));
- buffer[index++] = (byte)(0xFF & codePoint);
- }
- }
- buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
- return index - origin;
- }
-
- /**
- * Writes a string with our character format to a ByteArrayOutputStream.
- *
- * This will also write the terminator byte.
- *
- * @param buffer the ByteArrayOutputStream to write to.
- * @param word the string to write.
- */
- private static void writeString(final ByteArrayOutputStream buffer, final String word) {
- final int length = word.length();
- for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
- final int codePoint = word.codePointAt(i);
- if (1 == getCharSize(codePoint)) {
- buffer.write((byte) codePoint);
- } else {
- buffer.write((byte) (0xFF & (codePoint >> 16)));
- buffer.write((byte) (0xFF & (codePoint >> 8)));
- buffer.write((byte) (0xFF & codePoint));
- }
- }
- buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR);
- }
-
- /**
- * Reads a string from a buffer. This is the converse of the above method.
- */
- private static String readString(final FusionDictionaryBufferInterface buffer) {
- final StringBuilder s = new StringBuilder();
- int character = readChar(buffer);
- while (character != FormatSpec.INVALID_CHARACTER) {
- s.appendCodePoint(character);
- character = readChar(buffer);
- }
- return s.toString();
- }
-
- /**
- * Reads a character from the buffer.
- *
- * This follows the character format documented earlier in this source file.
- *
- * @param buffer the buffer, positioned over an encoded character.
- * @return the character code.
- */
- static int readChar(final FusionDictionaryBufferInterface buffer) {
- int character = buffer.readUnsignedByte();
- if (!fitsOnOneByte(character)) {
- if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) {
- return FormatSpec.INVALID_CHARACTER;
- }
- character <<= 16;
- character += buffer.readUnsignedShort();
- }
- return character;
- }
- }
-
- /**
- * Compute the binary size of the character array.
- *
- * If only one character, this is the size of this character. If many, it's the sum of their
- * sizes + 1 byte for the terminator.
- *
- * @param characters the character array
- * @return the size of the char array, including the terminator if any
- */
- static int getGroupCharactersSize(final int[] characters) {
- int size = CharEncoding.getCharArraySize(characters);
- if (characters.length > 1) size += FormatSpec.GROUP_TERMINATOR_SIZE;
- return size;
- }
-
- /**
- * Compute the binary size of the character array in a group
- *
- * If only one character, this is the size of this character. If many, it's the sum of their
- * sizes + 1 byte for the terminator.
- *
- * @param group the group
- * @return the size of the char array, including the terminator if any
- */
- private static int getGroupCharactersSize(final CharGroup group) {
- return getGroupCharactersSize(group.mChars);
- }
-
- /**
- * Compute the binary size of the group count for a node
- * @param node the node
- * @return the size of the group count, either 1 or 2 bytes.
- */
- private static int getGroupCountSize(final Node node) {
- return BinaryDictIOUtils.getGroupCountSize(node.mData.size());
- }
-
- /**
- * Compute the size of a shortcut in bytes.
- */
- private static int getShortcutSize(final WeightedString shortcut) {
- int size = FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE;
- final String word = shortcut.mWord;
- final int length = word.length();
- for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
- final int codePoint = word.codePointAt(i);
- size += CharEncoding.getCharSize(codePoint);
- }
- size += FormatSpec.GROUP_TERMINATOR_SIZE;
- return size;
- }
-
- /**
- * Compute the size of a shortcut list in bytes.
- *
- * This is known in advance and does not change according to position in the file
- * like address lists do.
- */
- static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) {
- if (null == shortcutList) return 0;
- int size = FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
- for (final WeightedString shortcut : shortcutList) {
- size += getShortcutSize(shortcut);
- }
- return size;
- }
-
- /**
- * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything.
- *
- * @param group the CharGroup to compute the size of.
- * @param options file format options.
- * @return the maximum size of the group.
- */
- private static int getCharGroupMaximumSize(final CharGroup group, final FormatOptions options) {
- int size = getGroupHeaderSize(group, options);
- // If terminal, one byte for the frequency
- if (group.isTerminal()) size += FormatSpec.GROUP_FREQUENCY_SIZE;
- size += FormatSpec.GROUP_MAX_ADDRESS_SIZE; // For children address
- size += getShortcutListSize(group.mShortcutTargets);
- if (null != group.mBigrams) {
- size += (FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE
- + FormatSpec.GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE)
- * group.mBigrams.size();
- }
- return size;
- }
-
- /**
- * Compute the maximum size of a node, assuming 3-byte addresses for everything, and caches
- * it in the 'actualSize' member of the node.
- *
- * @param node the node to compute the maximum size of.
- * @param options file format options.
- */
- private static void calculateNodeMaximumSize(final Node node, final FormatOptions options) {
- int size = getGroupCountSize(node);
- for (CharGroup g : node.mData) {
- final int groupSize = getCharGroupMaximumSize(g, options);
- g.mCachedSize = groupSize;
- size += groupSize;
- }
- if (options.mSupportsDynamicUpdate) {
- size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
- }
- node.mCachedSize = size;
- }
-
- /**
- * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup.
- *
- * @param group the group of which to compute the size of the header
- * @param options file format options.
- */
- private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) {
- if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
- return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
- + getGroupCharactersSize(group);
- } else {
- return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group);
- }
- }
-
- /**
- * Compute the size, in bytes, that an address will occupy.
- *
- * This can be used either for children addresses (which are always positive) or for
- * attribute, which may be positive or negative but
- * store their sign bit separately.
- *
- * @param address the address
- * @return the byte size.
- */
- static int getByteSize(final int address) {
- assert(address <= FormatSpec.UINT24_MAX);
- if (!BinaryDictIOUtils.hasChildrenAddress(address)) {
- return 0;
- } else if (Math.abs(address) <= FormatSpec.UINT8_MAX) {
- return 1;
- } else if (Math.abs(address) <= FormatSpec.UINT16_MAX) {
- return 2;
- } else {
- return 3;
- }
- }
-
- // End utility methods.
-
- // This method is responsible for finding a nice ordering of the nodes that favors run-time
- // cache performance and dictionary size.
- /* package for tests */ static ArrayList<Node> flattenTree(final Node root) {
- final int treeSize = FusionDictionary.countCharGroups(root);
- MakedictLog.i("Counted nodes : " + treeSize);
- final ArrayList<Node> flatTree = new ArrayList<Node>(treeSize);
- return flattenTreeInner(flatTree, root);
- }
-
- private static ArrayList<Node> flattenTreeInner(final ArrayList<Node> list, final Node node) {
- // Removing the node is necessary if the tails are merged, because we would then
- // add the same node several times when we only want it once. A number of places in
- // the code also depends on any node being only once in the list.
- // Merging tails can only be done if there are no attributes. Searching for attributes
- // in LatinIME code depends on a total breadth-first ordering, which merging tails
- // breaks. If there are no attributes, it should be fine (and reduce the file size)
- // to merge tails, and removing the node from the list would be necessary. However,
- // we don't merge tails because breaking the breadth-first ordering would result in
- // extreme overhead at bigram lookup time (it would make the search function O(n) instead
- // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty
- // high).
- // If no nodes are ever merged, we can't have the same node twice in the list, hence
- // searching for duplicates in unnecessary. It is also very performance consuming,
- // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making
- // this simple list.remove operation O(n*n) overall. On Android this overhead is very
- // high.
- // For future reference, the code to remove duplicate is a simple : list.remove(node);
- list.add(node);
- final ArrayList<CharGroup> branches = node.mData;
- final int nodeSize = branches.size();
- for (CharGroup group : branches) {
- if (null != group.mChildren) flattenTreeInner(list, group.mChildren);
- }
- return list;
- }
-
- /**
- * Get the offset from a position inside a current node to a target node, during update.
- *
- * If the current node is before the target node, the target node has not been updated yet,
- * so we should return the offset from the old position of the current node to the old position
- * of the target node. If on the other hand the target is before the current node, it already
- * has been updated, so we should return the offset from the new position in the current node
- * to the new position in the target node.
- * @param currentNode the node containing the CharGroup where the offset will be written
- * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode
- * @param targetNode the target node to get the offset to
- * @return the offset to the target node
- */
- private static int getOffsetToTargetNodeDuringUpdate(final Node currentNode,
- final int offsetFromStartOfCurrentNode, final Node targetNode) {
- final boolean isTargetBeforeCurrent = (targetNode.mCachedAddressBeforeUpdate
- < currentNode.mCachedAddressBeforeUpdate);
- if (isTargetBeforeCurrent) {
- return targetNode.mCachedAddressAfterUpdate
- - (currentNode.mCachedAddressAfterUpdate + offsetFromStartOfCurrentNode);
- } else {
- return targetNode.mCachedAddressBeforeUpdate
- - (currentNode.mCachedAddressBeforeUpdate + offsetFromStartOfCurrentNode);
- }
- }
-
- /**
- * Get the offset from a position inside a current node to a target CharGroup, during update.
- * @param currentNode the node containing the CharGroup where the offset will be written
- * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode
- * @param targetCharGroup the target CharGroup to get the offset to
- * @return the offset to the target CharGroup
- */
- // TODO: is there any way to factorize this method with the one above?
- private static int getOffsetToTargetCharGroupDuringUpdate(final Node currentNode,
- final int offsetFromStartOfCurrentNode, final CharGroup targetCharGroup) {
- final int oldOffsetBasePoint = currentNode.mCachedAddressBeforeUpdate
- + offsetFromStartOfCurrentNode;
- final boolean isTargetBeforeCurrent = (targetCharGroup.mCachedAddressBeforeUpdate
- < oldOffsetBasePoint);
- // If the target is before the current node, then its address has already been updated.
- // We can use the AfterUpdate member, and compare it to our own member after update.
- // Otherwise, the AfterUpdate member is not updated yet, so we need to use the BeforeUpdate
- // member, and of course we have to compare this to our own address before update.
- if (isTargetBeforeCurrent) {
- final int newOffsetBasePoint = currentNode.mCachedAddressAfterUpdate
- + offsetFromStartOfCurrentNode;
- return targetCharGroup.mCachedAddressAfterUpdate - newOffsetBasePoint;
- } else {
- return targetCharGroup.mCachedAddressBeforeUpdate - oldOffsetBasePoint;
- }
- }
-
- /**
- * Computes the actual node size, based on the cached addresses of the children nodes.
- *
- * Each node stores its tentative address. During dictionary address computing, these
- * are not final, but they can be used to compute the node size (the node size depends
- * on the address of the children because the number of bytes necessary to store an
- * address depends on its numeric value. The return value indicates whether the node
- * contents (as in, any of the addresses stored in the cache fields) have changed with
- * respect to their previous value.
- *
- * @param node the node to compute the size of.
- * @param dict the dictionary in which the word/attributes are to be found.
- * @param formatOptions file format options.
- * @return false if none of the cached addresses inside the node changed, true otherwise.
- */
- private static boolean computeActualNodeSize(final Node node, final FusionDictionary dict,
- final FormatOptions formatOptions) {
- boolean changed = false;
- int size = getGroupCountSize(node);
- for (CharGroup group : node.mData) {
- group.mCachedAddressAfterUpdate = node.mCachedAddressAfterUpdate + size;
- if (group.mCachedAddressAfterUpdate != group.mCachedAddressBeforeUpdate) {
- changed = true;
- }
- int groupSize = getGroupHeaderSize(group, formatOptions);
- if (group.isTerminal()) groupSize += FormatSpec.GROUP_FREQUENCY_SIZE;
- if (null == group.mChildren && formatOptions.mSupportsDynamicUpdate) {
- groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
- } else if (null != group.mChildren) {
- if (formatOptions.mSupportsDynamicUpdate) {
- groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
- } else {
- groupSize += getByteSize(getOffsetToTargetNodeDuringUpdate(node,
- groupSize + size, group.mChildren));
- }
- }
- groupSize += getShortcutListSize(group.mShortcutTargets);
- if (null != group.mBigrams) {
- for (WeightedString bigram : group.mBigrams) {
- final int offset = getOffsetToTargetCharGroupDuringUpdate(node,
- groupSize + size + FormatSpec.GROUP_FLAGS_SIZE,
- FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord));
- groupSize += getByteSize(offset) + FormatSpec.GROUP_FLAGS_SIZE;
- }
- }
- group.mCachedSize = groupSize;
- size += groupSize;
- }
- if (formatOptions.mSupportsDynamicUpdate) {
- size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
- }
- if (node.mCachedSize != size) {
- node.mCachedSize = size;
- changed = true;
- }
- return changed;
- }
-
- /**
- * Initializes the cached addresses of nodes from their size.
- *
- * @param flatNodes the array of nodes.
- * @param formatOptions file format options.
- * @return the byte size of the entire stack.
- */
- private static int initializeNodesCachedAddresses(final ArrayList<Node> flatNodes,
- final FormatOptions formatOptions) {
- int nodeOffset = 0;
- for (final Node n : flatNodes) {
- n.mCachedAddressBeforeUpdate = nodeOffset;
- int groupCountSize = getGroupCountSize(n);
- int groupOffset = 0;
- for (final CharGroup g : n.mData) {
- g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate =
- groupCountSize + nodeOffset + groupOffset;
- groupOffset += g.mCachedSize;
- }
- final int nodeSize = groupCountSize + groupOffset
- + (formatOptions.mSupportsDynamicUpdate
- ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0);
- nodeOffset += n.mCachedSize;
- }
- return nodeOffset;
- }
-
- /**
- * Updates the cached addresses of nodes after recomputing their new positions.
- *
- * @param flatNodes the array of nodes.
- */
- private static void updateNodeCachedAddresses(final ArrayList<Node> flatNodes) {
- for (final Node n : flatNodes) {
- n.mCachedAddressBeforeUpdate = n.mCachedAddressAfterUpdate;
- for (final CharGroup g : n.mData) {
- g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate;
- }
- }
- }
-
- /**
- * Compute the cached parent addresses after all has been updated.
- *
- * The parent addresses are used by some binary formats at write-to-disk time. Not all formats
- * need them. In particular, version 2 does not need them, and version 3 does.
- *
- * @param flatNodes the flat array of nodes to fill in
- */
- private static void computeParentAddresses(final ArrayList<Node> flatNodes) {
- for (final Node node : flatNodes) {
- for (final CharGroup group : node.mData) {
- if (null != group.mChildren) {
- // Assign my address to children's parent address
- // Here BeforeUpdate and AfterUpdate addresses have the same value, so it
- // does not matter which we use.
- group.mChildren.mCachedParentAddress = group.mCachedAddressAfterUpdate
- - group.mChildren.mCachedAddressAfterUpdate;
- }
- }
- }
- }
-
- /**
- * Compute the addresses and sizes of an ordered node array.
- *
- * This method takes a node array and will update its cached address and size values
- * so that they can be written into a file. It determines the smallest size each of the
- * nodes can be given the addresses of its children and attributes, and store that into
- * each node.
- * The order of the node is given by the order of the array. This method makes no effort
- * to find a good order; it only mechanically computes the size this order results in.
- *
- * @param dict the dictionary
- * @param flatNodes the ordered array of nodes
- * @param formatOptions file format options.
- * @return the same array it was passed. The nodes have been updated for address and size.
- */
- private static ArrayList<Node> computeAddresses(final FusionDictionary dict,
- final ArrayList<Node> flatNodes, final FormatOptions formatOptions) {
- // First get the worst possible sizes and offsets
- for (final Node n : flatNodes) calculateNodeMaximumSize(n, formatOptions);
- final int offset = initializeNodesCachedAddresses(flatNodes, formatOptions);
-
- MakedictLog.i("Compressing the array addresses. Original size : " + offset);
- MakedictLog.i("(Recursively seen size : " + offset + ")");
-
- int passes = 0;
- boolean changesDone = false;
- do {
- changesDone = false;
- int nodeStartOffset = 0;
- for (final Node n : flatNodes) {
- n.mCachedAddressAfterUpdate = nodeStartOffset;
- final int oldNodeSize = n.mCachedSize;
- final boolean changed = computeActualNodeSize(n, dict, formatOptions);
- final int newNodeSize = n.mCachedSize;
- if (oldNodeSize < newNodeSize) throw new RuntimeException("Increased size ?!");
- nodeStartOffset += newNodeSize;
- changesDone |= changed;
- }
- updateNodeCachedAddresses(flatNodes);
- ++passes;
- if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug");
- } while (changesDone);
-
- if (formatOptions.mSupportsDynamicUpdate) {
- computeParentAddresses(flatNodes);
- }
- final Node lastNode = flatNodes.get(flatNodes.size() - 1);
- MakedictLog.i("Compression complete in " + passes + " passes.");
- MakedictLog.i("After address compression : "
- + (lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize));
-
- return flatNodes;
- }
-
- /**
- * Sanity-checking method.
- *
- * This method checks an array of node for juxtaposition, that is, it will do
- * nothing if each node's cached address is actually the previous node's address
- * plus the previous node's size.
- * If this is not the case, it will throw an exception.
- *
- * @param array the array node to check
- */
- private static void checkFlatNodeArray(final ArrayList<Node> array) {
- int offset = 0;
- int index = 0;
- for (final Node n : array) {
- // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter
- // which we use.
- if (n.mCachedAddressAfterUpdate != offset) {
- throw new RuntimeException("Wrong address for node " + index
- + " : expected " + offset + ", got " + n.mCachedAddressAfterUpdate);
- }
- ++index;
- offset += n.mCachedSize;
- }
- }
-
- /**
- * Helper method to write a variable-size address to a file.
- *
- * @param buffer the buffer to write to.
- * @param index the index in the buffer to write the address to.
- * @param address the address to write.
- * @return the size in bytes the address actually took.
- */
- private static int writeVariableAddress(final byte[] buffer, int index, final int address) {
- switch (getByteSize(address)) {
- case 1:
- buffer[index++] = (byte)address;
- return 1;
- case 2:
- buffer[index++] = (byte)(0xFF & (address >> 8));
- buffer[index++] = (byte)(0xFF & address);
- return 2;
- case 3:
- buffer[index++] = (byte)(0xFF & (address >> 16));
- buffer[index++] = (byte)(0xFF & (address >> 8));
- buffer[index++] = (byte)(0xFF & address);
- return 3;
- case 0:
- return 0;
- default:
- throw new RuntimeException("Address " + address + " has a strange size");
- }
- }
-
- /**
- * Helper method to write a variable-size signed address to a file.
- *
- * @param buffer the buffer to write to.
- * @param index the index in the buffer to write the address to.
- * @param address the address to write.
- * @return the size in bytes the address actually took.
- */
- private static int writeVariableSignedAddress(final byte[] buffer, int index,
- final int address) {
- if (!BinaryDictIOUtils.hasChildrenAddress(address)) {
- buffer[index] = buffer[index + 1] = buffer[index + 2] = 0;
- } else {
- final int absAddress = Math.abs(address);
- buffer[index++] =
- (byte)((address < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absAddress >> 16)));
- buffer[index++] = (byte)(0xFF & (absAddress >> 8));
- buffer[index++] = (byte)(0xFF & absAddress);
- }
- return 3;
- }
-
- /**
- * Makes the flag value for a char group.
- *
- * @param hasMultipleChars whether the group has multiple chars.
- * @param isTerminal whether the group is terminal.
- * @param childrenAddressSize the size of a children address.
- * @param hasShortcuts whether the group has shortcuts.
- * @param hasBigrams whether the group has bigrams.
- * @param isNotAWord whether the group is not a word.
- * @param isBlackListEntry whether the group is a blacklist entry.
- * @param formatOptions file format options.
- * @return the flags
- */
- static int makeCharGroupFlags(final boolean hasMultipleChars, final boolean isTerminal,
- final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams,
- final boolean isNotAWord, final boolean isBlackListEntry,
- final FormatOptions formatOptions) {
- byte flags = 0;
- if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS;
- if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL;
- if (formatOptions.mSupportsDynamicUpdate) {
- flags |= FormatSpec.FLAG_IS_NOT_MOVED;
- } else if (true) {
- switch (childrenAddressSize) {
- case 1:
- flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE;
- break;
- case 2:
- flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES;
- break;
- case 3:
- flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES;
- break;
- case 0:
- flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS;
- break;
- default:
- throw new RuntimeException("Node with a strange address");
- }
- }
- if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS;
- if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS;
- if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD;
- if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED;
- return flags;
- }
-
- private static byte makeCharGroupFlags(final CharGroup group, final int groupAddress,
- final int childrenOffset, final FormatOptions formatOptions) {
- return (byte) makeCharGroupFlags(group.mChars.length > 1, group.mFrequency >= 0,
- getByteSize(childrenOffset), group.mShortcutTargets != null, group.mBigrams != null,
- group.mIsNotAWord, group.mIsBlacklistEntry, formatOptions);
- }
-
- /**
- * Makes the flag value for a bigram.
- *
- * @param more whether there are more bigrams after this one.
- * @param offset the offset of the bigram.
- * @param bigramFrequency the frequency of the bigram, 0..255.
- * @param unigramFrequency the unigram frequency of the same word, 0..255.
- * @param word the second bigram, for debugging purposes
- * @return the flags
- */
- private static final int makeBigramFlags(final boolean more, final int offset,
- int bigramFrequency, final int unigramFrequency, final String word) {
- int bigramFlags = (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0)
- + (offset < 0 ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0);
- switch (getByteSize(offset)) {
- case 1:
- bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
- break;
- case 2:
- bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES;
- break;
- case 3:
- bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES;
- break;
- default:
- throw new RuntimeException("Strange offset size");
- }
- if (unigramFrequency > bigramFrequency) {
- MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word
- + "\". Bigram freq is " + bigramFrequency + ", unigram freq for "
- + word + " is " + unigramFrequency);
- bigramFrequency = unigramFrequency;
- }
- // We compute the difference between 255 (which means probability = 1) and the
- // unigram score. We split this into a number of discrete steps.
- // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15
- // represents an increase of 16 steps: a value of 15 will be interpreted as the median
- // value of the 16th step. In all justice, if the bigram frequency is low enough to be
- // rounded below the first step (which means it is less than half a step higher than the
- // unigram frequency) then the unigram frequency itself is the best approximation of the
- // bigram freq that we could possibly supply, hence we should *not* include this bigram
- // in the file at all.
- // until this is done, we'll write 0 and slightly overestimate this case.
- // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step
- // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to
- // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the
- // step size. Then we compute the start of the first step (the one where value 0 starts)
- // by adding half-a-step to the unigramFrequency. From there, we compute the integer
- // number of steps to the bigramFrequency. One last thing: we want our steps to include
- // their lower bound and exclude their higher bound so we need to have the first step
- // start at exactly 1 unit higher than floor(unigramFreq + half a step).
- // Note : to reconstruct the score, the dictionary reader will need to divide
- // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step,
- // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best
- // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the
- // step pointed by the discretized frequency.
- final float stepSize =
- (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
- / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
- final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f);
- final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize);
- // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1
- // here. The best approximation would be the unigram freq itself, so we should not
- // include this bigram in the dictionary. For now, register as 0, and live with the
- // small over-estimation that we get in this case. TODO: actually remove this bigram
- // if discretizedFrequency < 0.
- final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0;
- bigramFlags += finalBigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY;
- return bigramFlags;
- }
-
- /**
- * Makes the 2-byte value for options flags.
- */
- private static final int makeOptionsValue(final FusionDictionary dictionary,
- final FormatOptions formatOptions) {
- final DictionaryOptions options = dictionary.mOptions;
- final boolean hasBigrams = dictionary.hasBigrams();
- return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0)
- + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0)
- + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0)
- + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0);
- }
-
- /**
- * Makes the flag value for a shortcut.
- *
- * @param more whether there are more attributes after this one.
- * @param frequency the frequency of the attribute, 0..15
- * @return the flags
- */
- static final int makeShortcutFlags(final boolean more, final int frequency) {
- return (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0)
- + (frequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY);
- }
-
- private static final int writeParentAddress(final byte[] buffer, final int index,
- final int address, final FormatOptions formatOptions) {
- if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
- if (address == FormatSpec.NO_PARENT_ADDRESS) {
- buffer[index] = buffer[index + 1] = buffer[index + 2] = 0;
- } else {
- final int absAddress = Math.abs(address);
- assert(absAddress <= FormatSpec.SINT24_MAX);
- buffer[index] = (byte)((address < 0 ? FormatSpec.MSB8 : 0)
- | ((absAddress >> 16) & 0xFF));
- buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF);
- buffer[index + 2] = (byte)(absAddress & 0xFF);
- }
- return index + 3;
- } else {
- return index;
- }
- }
-
- /**
- * Write a node to memory. The node is expected to have its final position cached.
- *
- * This can be an empty map, but the more is inside the faster the lookups will be. It can
- * be carried on as long as nodes do not move.
- *
- * @param dict the dictionary the node is a part of (for relative offsets).
- * @param buffer the memory buffer to write to.
- * @param node the node to write.
- * @param formatOptions file format options.
- * @return the address of the END of the node.
- */
- @SuppressWarnings("unused")
- private static int writePlacedNode(final FusionDictionary dict, byte[] buffer,
- final Node node, final FormatOptions formatOptions) {
- // TODO: Make the code in common with BinaryDictIOUtils#writeCharGroup
- int index = node.mCachedAddressAfterUpdate;
-
- final int groupCount = node.mData.size();
- final int countSize = getGroupCountSize(node);
- final int parentAddress = node.mCachedParentAddress;
- if (1 == countSize) {
- buffer[index++] = (byte)groupCount;
- } else if (2 == countSize) {
- // We need to signal 2-byte size by setting the top bit of the MSB to 1, so
- // we | 0x80 to do this.
- buffer[index++] = (byte)((groupCount >> 8) | 0x80);
- buffer[index++] = (byte)(groupCount & 0xFF);
- } else {
- throw new RuntimeException("Strange size from getGroupCountSize : " + countSize);
- }
- int groupAddress = index;
- for (int i = 0; i < groupCount; ++i) {
- final CharGroup group = node.mData.get(i);
- if (index != group.mCachedAddressAfterUpdate) {
- throw new RuntimeException("Bug: write index is not the same as the cached address "
- + "of the group : " + index + " <> " + group.mCachedAddressAfterUpdate);
- }
- groupAddress += getGroupHeaderSize(group, formatOptions);
- // Sanity checks.
- if (DBG && group.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) {
- throw new RuntimeException("A node has a frequency > "
- + FormatSpec.MAX_TERMINAL_FREQUENCY
- + " : " + group.mFrequency);
- }
- if (group.mFrequency >= 0) groupAddress += FormatSpec.GROUP_FREQUENCY_SIZE;
- final int childrenOffset = null == group.mChildren
- ? FormatSpec.NO_CHILDREN_ADDRESS
- : group.mChildren.mCachedAddressAfterUpdate - groupAddress;
- buffer[index++] =
- makeCharGroupFlags(group, groupAddress, childrenOffset, formatOptions);
-
- if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) {
- index = writeParentAddress(buffer, index, parentAddress, formatOptions);
- } else {
- index = writeParentAddress(buffer, index, parentAddress
- + (node.mCachedAddressAfterUpdate - group.mCachedAddressAfterUpdate),
- formatOptions);
- }
-
- index = CharEncoding.writeCharArray(group.mChars, buffer, index);
- if (group.hasSeveralChars()) {
- buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
- }
- if (group.mFrequency >= 0) {
- buffer[index++] = (byte) group.mFrequency;
- }
-
- final int shift;
- if (formatOptions.mSupportsDynamicUpdate) {
- shift = writeVariableSignedAddress(buffer, index, childrenOffset);
- } else {
- shift = writeVariableAddress(buffer, index, childrenOffset);
- }
- index += shift;
- groupAddress += shift;
-
- // Write shortcuts
- if (null != group.mShortcutTargets) {
- final int indexOfShortcutByteSize = index;
- index += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
- groupAddress += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
- final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator();
- while (shortcutIterator.hasNext()) {
- final WeightedString target = shortcutIterator.next();
- ++groupAddress;
- int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(),
- target.mFrequency);
- buffer[index++] = (byte)shortcutFlags;
- final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord);
- index += shortcutShift;
- groupAddress += shortcutShift;
- }
- final int shortcutByteSize = index - indexOfShortcutByteSize;
- if (shortcutByteSize > 0xFFFF) {
- throw new RuntimeException("Shortcut list too large");
- }
- buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8);
- buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF);
- }
- // Write bigrams
- if (null != group.mBigrams) {
- final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator();
- while (bigramIterator.hasNext()) {
- final WeightedString bigram = bigramIterator.next();
- final CharGroup target =
- FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord);
- final int addressOfBigram = target.mCachedAddressAfterUpdate;
- final int unigramFrequencyForThisWord = target.mFrequency;
- ++groupAddress;
- final int offset = addressOfBigram - groupAddress;
- int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset,
- bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
- buffer[index++] = (byte)bigramFlags;
- final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset));
- index += bigramShift;
- groupAddress += bigramShift;
- }
- }
-
- }
- if (formatOptions.mSupportsDynamicUpdate) {
- buffer[index] = buffer[index + 1] = buffer[index + 2]
- = FormatSpec.NO_FORWARD_LINK_ADDRESS;
- index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
- }
- if (index != node.mCachedAddressAfterUpdate + node.mCachedSize) throw new RuntimeException(
- "Not the same size : written "
- + (index - node.mCachedAddressAfterUpdate) + " bytes from a node that should have "
- + node.mCachedSize + " bytes");
- return index;
- }
-
- /**
- * Dumps a collection of useful statistics about a node array.
- *
- * This prints purely informative stuff, like the total estimated file size, the
- * number of nodes, of character groups, the repartition of each address size, etc
- *
- * @param nodes the node array.
- */
- private static void showStatistics(ArrayList<Node> nodes) {
- int firstTerminalAddress = Integer.MAX_VALUE;
- int lastTerminalAddress = Integer.MIN_VALUE;
- int size = 0;
- int charGroups = 0;
- int maxGroups = 0;
- int maxRuns = 0;
- for (final Node n : nodes) {
- if (maxGroups < n.mData.size()) maxGroups = n.mData.size();
- for (final CharGroup cg : n.mData) {
- ++charGroups;
- if (cg.mChars.length > maxRuns) maxRuns = cg.mChars.length;
- if (cg.mFrequency >= 0) {
- if (n.mCachedAddressAfterUpdate < firstTerminalAddress)
- firstTerminalAddress = n.mCachedAddressAfterUpdate;
- if (n.mCachedAddressAfterUpdate > lastTerminalAddress)
- lastTerminalAddress = n.mCachedAddressAfterUpdate;
- }
- }
- if (n.mCachedAddressAfterUpdate + n.mCachedSize > size) {
- size = n.mCachedAddressAfterUpdate + n.mCachedSize;
- }
- }
- final int[] groupCounts = new int[maxGroups + 1];
- final int[] runCounts = new int[maxRuns + 1];
- for (final Node n : nodes) {
- ++groupCounts[n.mData.size()];
- for (final CharGroup cg : n.mData) {
- ++runCounts[cg.mChars.length];
- }
- }
-
- MakedictLog.i("Statistics:\n"
- + " total file size " + size + "\n"
- + " " + nodes.size() + " nodes\n"
- + " " + charGroups + " groups (" + ((float)charGroups / nodes.size())
- + " groups per node)\n"
- + " first terminal at " + firstTerminalAddress + "\n"
- + " last terminal at " + lastTerminalAddress + "\n"
- + " Group stats : max = " + maxGroups);
- for (int i = 0; i < groupCounts.length; ++i) {
- MakedictLog.i(" " + i + " : " + groupCounts[i]);
- }
- MakedictLog.i(" Character run stats : max = " + maxRuns);
- for (int i = 0; i < runCounts.length; ++i) {
- MakedictLog.i(" " + i + " : " + runCounts[i]);
- }
- }
-
- /**
- * Dumps a FusionDictionary to a file.
- *
- * This is the public entry point to write a dictionary to a file.
- *
- * @param destination the stream to write the binary data to.
- * @param dict the dictionary to write.
- * @param formatOptions file format options.
- */
- public static void writeDictionaryBinary(final OutputStream destination,
- final FusionDictionary dict, final FormatOptions formatOptions)
- throws IOException, UnsupportedFormatException {
-
- // Addresses are limited to 3 bytes, but since addresses can be relative to each node, the
- // structure itself is not limited to 16MB. However, if it is over 16MB deciding the order
- // of the nodes becomes a quite complicated problem, because though the dictionary itself
- // does not have a size limit, each node must still be within 16MB of all its children and
- // parents. As long as this is ensured, the dictionary file may grow to any size.
-
- final int version = formatOptions.mVersion;
- if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
- || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
- throw new UnsupportedFormatException("Requested file format version " + version
- + ", but this implementation only supports versions "
- + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through "
- + FormatSpec.MAXIMUM_SUPPORTED_VERSION);
- }
-
- ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256);
-
- // The magic number in big-endian order.
- // Magic number for all versions.
- headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 24)));
- headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 16)));
- headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 8)));
- headerBuffer.write((byte) (0xFF & FormatSpec.MAGIC_NUMBER));
- // Dictionary version.
- headerBuffer.write((byte) (0xFF & (version >> 8)));
- headerBuffer.write((byte) (0xFF & version));
-
- // Options flags
- final int options = makeOptionsValue(dict, formatOptions);
- headerBuffer.write((byte) (0xFF & (options >> 8)));
- headerBuffer.write((byte) (0xFF & options));
- final int headerSizeOffset = headerBuffer.size();
- // Placeholder to be written later with header size.
- for (int i = 0; i < 4; ++i) {
- headerBuffer.write(0);
- }
- // Write out the options.
- for (final String key : dict.mOptions.mAttributes.keySet()) {
- final String value = dict.mOptions.mAttributes.get(key);
- CharEncoding.writeString(headerBuffer, key);
- CharEncoding.writeString(headerBuffer, value);
- }
- final int size = headerBuffer.size();
- final byte[] bytes = headerBuffer.toByteArray();
- // Write out the header size.
- bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24));
- bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16));
- bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8));
- bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0));
- destination.write(bytes);
-
- headerBuffer.close();
-
- // Leave the choice of the optimal node order to the flattenTree function.
- MakedictLog.i("Flattening the tree...");
- ArrayList<Node> flatNodes = flattenTree(dict.mRoot);
-
- MakedictLog.i("Computing addresses...");
- computeAddresses(dict, flatNodes, formatOptions);
- MakedictLog.i("Checking array...");
- if (DBG) checkFlatNodeArray(flatNodes);
-
- // Create a buffer that matches the final dictionary size.
- final Node lastNode = flatNodes.get(flatNodes.size() - 1);
- final int bufferSize = lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize;
- final byte[] buffer = new byte[bufferSize];
- int index = 0;
-
- MakedictLog.i("Writing file...");
- int dataEndOffset = 0;
- for (Node n : flatNodes) {
- dataEndOffset = writePlacedNode(dict, buffer, n, formatOptions);
- }
-
- if (DBG) showStatistics(flatNodes);
-
- destination.write(buffer, 0, dataEndOffset);
-
- destination.close();
- MakedictLog.i("Done");
- }
-
-
- // Input methods: Read a binary dictionary to memory.
- // readDictionaryBinary is the public entry point for them.
-
- static int getChildrenAddressSize(final int optionFlags,
- final FormatOptions formatOptions) {
- if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
- switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
- return 1;
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
- return 2;
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
- return 3;
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
- default:
- return 0;
- }
- }
-
- static int readChildrenAddress(final FusionDictionaryBufferInterface buffer,
- final int optionFlags, final FormatOptions options) {
- if (options.mSupportsDynamicUpdate) {
- final int address = buffer.readUnsignedInt24();
- if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS;
- if ((address & FormatSpec.MSB24) != 0) {
- return -(address & FormatSpec.SINT24_MAX);
- } else {
- return address;
- }
- }
- int address;
- switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
- return buffer.readUnsignedByte();
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
- return buffer.readUnsignedShort();
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
- return buffer.readUnsignedInt24();
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
- default:
- return FormatSpec.NO_CHILDREN_ADDRESS;
- }
- }
-
- static int readParentAddress(final FusionDictionaryBufferInterface buffer,
- final FormatOptions formatOptions) {
- if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
- final int parentAddress = buffer.readUnsignedInt24();
- final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1;
- return sign * (parentAddress & FormatSpec.SINT24_MAX);
- } else {
- return FormatSpec.NO_PARENT_ADDRESS;
- }
- }
-
- private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH];
- public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
- final int originalGroupAddress, final FormatOptions options) {
- int addressPointer = originalGroupAddress;
- final int flags = buffer.readUnsignedByte();
- ++addressPointer;
-
- final int parentAddress = readParentAddress(buffer, options);
- if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
- addressPointer += 3;
- }
-
- final int characters[];
- if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) {
- int index = 0;
- int character = CharEncoding.readChar(buffer);
- addressPointer += CharEncoding.getCharSize(character);
- while (-1 != character) {
- // FusionDictionary is making sure that the length of the word is smaller than
- // MAX_WORD_LENGTH.
- // So we'll never write past the end of CHARACTER_BUFFER.
- CHARACTER_BUFFER[index++] = character;
- character = CharEncoding.readChar(buffer);
- addressPointer += CharEncoding.getCharSize(character);
- }
- characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index);
- } else {
- final int character = CharEncoding.readChar(buffer);
- addressPointer += CharEncoding.getCharSize(character);
- characters = new int[] { character };
- }
- final int frequency;
- if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) {
- ++addressPointer;
- frequency = buffer.readUnsignedByte();
- } else {
- frequency = CharGroup.NOT_A_TERMINAL;
- }
- int childrenAddress = readChildrenAddress(buffer, flags, options);
- if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
- childrenAddress += addressPointer;
- }
- addressPointer += getChildrenAddressSize(flags, options);
- ArrayList<WeightedString> shortcutTargets = null;
- if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) {
- final int pointerBefore = buffer.position();
- shortcutTargets = new ArrayList<WeightedString>();
- buffer.readUnsignedShort(); // Skip the size
- while (true) {
- final int targetFlags = buffer.readUnsignedByte();
- final String word = CharEncoding.readString(buffer);
- shortcutTargets.add(new WeightedString(word,
- targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY));
- if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
- }
- addressPointer += buffer.position() - pointerBefore;
- }
- ArrayList<PendingAttribute> bigrams = null;
- if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
- bigrams = new ArrayList<PendingAttribute>();
- int bigramCount = 0;
- while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
- final int bigramFlags = buffer.readUnsignedByte();
- ++addressPointer;
- final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE)
- ? 1 : -1;
- int bigramAddress = addressPointer;
- switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) {
- case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
- bigramAddress += sign * buffer.readUnsignedByte();
- addressPointer += 1;
- break;
- case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
- bigramAddress += sign * buffer.readUnsignedShort();
- addressPointer += 2;
- break;
- case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
- final int offset = (buffer.readUnsignedByte() << 16)
- + buffer.readUnsignedShort();
- bigramAddress += sign * offset;
- addressPointer += 3;
- break;
- default:
- throw new RuntimeException("Has bigrams with no address");
- }
- bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY,
- bigramAddress));
- if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
- }
- if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
- MakedictLog.d("too many bigrams in a group.");
- }
- }
- return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency,
- parentAddress, childrenAddress, shortcutTargets, bigrams);
- }
-
- /**
- * Reads and returns the char group count out of a buffer and forwards the pointer.
- */
- public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
- final int msb = buffer.readUnsignedByte();
- if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) {
- return msb;
- } else {
- return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8)
- + buffer.readUnsignedByte();
- }
- }
-
- // The word cache here is a stopgap bandaid to help the catastrophic performance
- // of this method. Since it performs direct, unbuffered random access to the file and
- // may be called hundreds of thousands of times, the resulting performance is not
- // reasonable without some kind of cache. Thus:
- private static TreeMap<Integer, WeightedString> wordCache =
- new TreeMap<Integer, WeightedString>();
- /**
- * Finds, as a string, the word at the address passed as an argument.
- *
- * @param buffer the buffer to read from.
- * @param headerSize the size of the header.
- * @param address the address to seek.
- * @param formatOptions file format options.
- * @return the word with its frequency, as a weighted string.
- */
- /* package for tests */ static WeightedString getWordAtAddress(
- final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
- final FormatOptions formatOptions) {
- final WeightedString cachedString = wordCache.get(address);
- if (null != cachedString) return cachedString;
-
- final WeightedString result;
- final int originalPointer = buffer.position();
- buffer.position(address);
-
- if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
- result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions);
- } else {
- result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address,
- formatOptions);
- }
-
- wordCache.put(address, result);
- buffer.position(originalPointer);
- return result;
- }
-
- // TODO: static!? This will behave erratically when used in multi-threaded code.
- // We need to fix this
- private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH];
- @SuppressWarnings("unused")
- private static WeightedString getWordAtAddressWithParentAddress(
- final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
- final FormatOptions options) {
- int currentAddress = address;
- int index = FormatSpec.MAX_WORD_LENGTH - 1;
- int frequency = Integer.MIN_VALUE;
- // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH
- for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) {
- CharGroupInfo currentInfo;
- int loopCounter = 0;
- do {
- buffer.position(currentAddress + headerSize);
- currentInfo = readCharGroup(buffer, currentAddress, options);
- if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) {
- currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
- }
- if (DBG && loopCounter++ > MAX_JUMPS) {
- MakedictLog.d("Too many jumps - probably a bug");
- }
- } while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options));
- if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency;
- for (int i = 0; i < currentInfo.mCharacters.length; ++i) {
- sGetWordBuffer[index--] =
- currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1];
- }
- if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break;
- currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
- }
-
- return new WeightedString(
- new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1),
- frequency);
- }
-
- private static WeightedString getWordAtAddressWithoutParentAddress(
- final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
- final FormatOptions options) {
- buffer.position(headerSize);
- final int count = readCharGroupCount(buffer);
- int groupOffset = BinaryDictIOUtils.getGroupCountSize(count);
- final StringBuilder builder = new StringBuilder();
- WeightedString result = null;
-
- CharGroupInfo last = null;
- for (int i = count - 1; i >= 0; --i) {
- CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
- groupOffset = info.mEndAddress;
- if (info.mOriginalAddress == address) {
- builder.append(new String(info.mCharacters, 0, info.mCharacters.length));
- result = new WeightedString(builder.toString(), info.mFrequency);
- break; // and return
- }
- if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
- if (info.mChildrenAddress > address) {
- if (null == last) continue;
- builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
- buffer.position(last.mChildrenAddress + headerSize);
- i = readCharGroupCount(buffer);
- groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
- last = null;
- continue;
- }
- last = info;
- }
- if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) {
- builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
- buffer.position(last.mChildrenAddress + headerSize);
- i = readCharGroupCount(buffer);
- groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
- last = null;
- continue;
- }
- }
- return result;
- }
-
- /**
- * Reads a single node from a buffer.
- *
- * This methods reads the file at the current position. A node is fully expected to start at
- * the current position.
- * This will recursively read other nodes into the structure, populating the reverse
- * maps on the fly and using them to keep track of already read nodes.
- *
- * @param buffer the buffer, correctly positioned at the start of a node.
- * @param headerSize the size, in bytes, of the file header.
- * @param reverseNodeMap a mapping from addresses to already read nodes.
- * @param reverseGroupMap a mapping from addresses to already read character groups.
- * @param options file format options.
- * @return the read node with all his children already read.
- */
- private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize,
- final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap,
- final FormatOptions options)
- throws IOException {
- final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>();
- final int nodeOrigin = buffer.position() - headerSize;
-
- do { // Scan the linked-list node.
- final int nodeHeadPosition = buffer.position() - headerSize;
- final int count = readCharGroupCount(buffer);
- int groupOffset = nodeHeadPosition + BinaryDictIOUtils.getGroupCountSize(count);
- for (int i = count; i > 0; --i) { // Scan the array of CharGroup.
- CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
- if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue;
- ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets;
- ArrayList<WeightedString> bigrams = null;
- if (null != info.mBigrams) {
- bigrams = new ArrayList<WeightedString>();
- for (PendingAttribute bigram : info.mBigrams) {
- final WeightedString word = getWordAtAddress(
- buffer, headerSize, bigram.mAddress, options);
- final int reconstructedFrequency =
- reconstructBigramFrequency(word.mFrequency, bigram.mFrequency);
- bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
- }
- }
- if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
- Node children = reverseNodeMap.get(info.mChildrenAddress);
- if (null == children) {
- final int currentPosition = buffer.position();
- buffer.position(info.mChildrenAddress + headerSize);
- children = readNode(
- buffer, headerSize, reverseNodeMap, reverseGroupMap, options);
- buffer.position(currentPosition);
- }
- nodeContents.add(
- new CharGroup(info.mCharacters, shortcutTargets, bigrams,
- info.mFrequency,
- 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
- 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children));
- } else {
- nodeContents.add(
- new CharGroup(info.mCharacters, shortcutTargets, bigrams,
- info.mFrequency,
- 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
- 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED)));
- }
- groupOffset = info.mEndAddress;
- }
-
- // reach the end of the array.
- if (options.mSupportsDynamicUpdate) {
- final int nextAddress = buffer.readUnsignedInt24();
- if (nextAddress >= 0 && nextAddress < buffer.limit()) {
- buffer.position(nextAddress);
- } else {
- break;
- }
- }
- } while (options.mSupportsDynamicUpdate &&
- buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS);
-
- final Node node = new Node(nodeContents);
- node.mCachedAddressBeforeUpdate = nodeOrigin;
- node.mCachedAddressAfterUpdate = nodeOrigin;
- reverseNodeMap.put(node.mCachedAddressAfterUpdate, node);
- return node;
- }
-
- /**
- * Helper function to get the binary format version from the header.
- * @throws IOException
- */
- private static int getFormatVersion(final FusionDictionaryBufferInterface buffer)
- throws IOException {
- final int magic = buffer.readInt();
- if (FormatSpec.MAGIC_NUMBER == magic) return buffer.readUnsignedShort();
- return FormatSpec.NOT_A_VERSION_NUMBER;
- }
-
- /**
- * Helper function to get and validate the binary format version.
- * @throws UnsupportedFormatException
- * @throws IOException
- */
- private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer)
- throws IOException, UnsupportedFormatException {
- final int version = getFormatVersion(buffer);
- if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
- || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
- throw new UnsupportedFormatException("This file has version " + version
- + ", but this implementation does not support versions above "
- + FormatSpec.MAXIMUM_SUPPORTED_VERSION);
- }
- return version;
- }
-
- /**
- * Reads a header from a buffer.
- * @param buffer the buffer to read.
- * @throws IOException
- * @throws UnsupportedFormatException
- */
- public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer)
- throws IOException, UnsupportedFormatException {
- final int version = checkFormatVersion(buffer);
- final int optionsFlags = buffer.readUnsignedShort();
-
- final HashMap<String, String> attributes = new HashMap<String, String>();
- final int headerSize;
- headerSize = buffer.readInt();
-
- if (headerSize < 0) {
- throw new UnsupportedFormatException("header size can't be negative.");
- }
-
- populateOptions(buffer, headerSize, attributes);
- buffer.position(headerSize);
-
- final FileHeader header = new FileHeader(headerSize,
- new FusionDictionary.DictionaryOptions(attributes,
- 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG),
- 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)),
- new FormatOptions(version,
- 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE)));
- return header;
- }
-
- /**
- * Reads options from a buffer and populate a map with their contents.
- *
- * The buffer is read at the current position, so the caller must take care the pointer
- * is in the right place before calling this.
- */
- public static void populateOptions(final FusionDictionaryBufferInterface buffer,
- final int headerSize, final HashMap<String, String> options) {
- while (buffer.position() < headerSize) {
- final String key = CharEncoding.readString(buffer);
- final String value = CharEncoding.readString(buffer);
- options.put(key, value);
- }
- }
-
- /**
- * Reads a buffer and returns the memory representation of the dictionary.
- *
- * This high-level method takes a buffer and reads its contents, populating a
- * FusionDictionary structure. The optional dict argument is an existing dictionary to
- * which words from the buffer should be added. If it is null, a new dictionary is created.
- *
- * @param reader the reader.
- * @param dict an optional dictionary to add words to, or null.
- * @return the created (or merged) dictionary.
- */
- @UsedForTesting
- public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
- final FusionDictionary dict) throws FileNotFoundException, IOException,
- UnsupportedFormatException {
- // clear cache
- wordCache.clear();
-
- // if the buffer has not been opened, open the buffer with bytebuffer.
- if (reader.getBuffer() == null) reader.openBuffer(
- new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
- if (reader.getBuffer() == null) {
- MakedictLog.e("Cannot open the buffer");
- }
-
- // Read header
- final FileHeader header = readHeader(reader.getBuffer());
-
- Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
- Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
- final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
- reverseGroupMapping, header.mFormatOptions);
-
- FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);
- if (null != dict) {
- for (final Word w : dict) {
- if (w.mIsBlacklistEntry) {
- newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord);
- } else {
- newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord);
- }
- }
- for (final Word w : dict) {
- // By construction a binary dictionary may not have bigrams pointing to
- // words that are not also registered as unigrams so we don't have to avoid
- // them explicitly here.
- for (final WeightedString bigram : w.mBigrams) {
- newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency);
- }
- }
- }
-
- return newDict;
- }
-
- /**
- * Helper method to pass a file name instead of a File object to isBinaryDictionary.
- */
- public static boolean isBinaryDictionary(final String filename) {
- final File file = new File(filename);
- return isBinaryDictionary(file);
- }
-
- /**
- * Basic test to find out whether the file is a binary dictionary or not.
- *
- * Concretely this only tests the magic number.
- *
- * @param file The file to test.
- * @return true if it's a binary dictionary, false otherwise
- */
- public static boolean isBinaryDictionary(final File file) {
- FileInputStream inStream = null;
- try {
- inStream = new FileInputStream(file);
- final ByteBuffer buffer = inStream.getChannel().map(
- FileChannel.MapMode.READ_ONLY, 0, file.length());
- final int version = getFormatVersion(new ByteBufferWrapper(buffer));
- return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION
- && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION);
- } catch (FileNotFoundException e) {
- return false;
- } catch (IOException e) {
- return false;
- } finally {
- if (inStream != null) {
- try {
- inStream.close();
- } catch (IOException e) {
- // do nothing
- }
- }
- }
- }
-
- /**
- * Calculate bigram frequency from compressed value
- *
- * @see #makeBigramFlags
- *
- * @param unigramFrequency
- * @param bigramFrequency compressed frequency
- * @return approximate bigram frequency
- */
- public static int reconstructBigramFrequency(final int unigramFrequency,
- final int bigramFrequency) {
- final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
- / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
- final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f);
- return (int)resultFreqFloat;
- }
-}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java
index a4a7ce4..6d3b31a 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java
@@ -17,7 +17,9 @@
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.CharEncoding;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.decoder.HeaderReaderInterface;
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import java.io.File;
@@ -27,8 +29,9 @@
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.util.HashMap;
-public class BinaryDictReader {
+public class BinaryDictReader implements HeaderReaderInterface {
public interface FusionDictionaryBufferFactory {
public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
@@ -55,7 +58,7 @@
}
}
if (buffer != null) {
- return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
+ return new BinaryDictDecoder.ByteBufferWrapper(buffer);
}
return null;
}
@@ -103,7 +106,7 @@
}
}
if (buffer != null) {
- return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
+ return new BinaryDictDecoder.ByteBufferWrapper(buffer);
}
return null;
}
@@ -133,4 +136,34 @@
openBuffer(factory);
return getBuffer();
}
+
+ // The implementation of HeaderReaderInterface
+ @Override
+ public int readVersion() throws IOException, UnsupportedFormatException {
+ return BinaryDictDecoder.checkFormatVersion(mFusionDictionaryBuffer);
+ }
+
+ @Override
+ public int readOptionFlags() {
+ return mFusionDictionaryBuffer.readUnsignedShort();
+ }
+
+ @Override
+ public int readHeaderSize() {
+ return mFusionDictionaryBuffer.readInt();
+ }
+
+ @Override
+ public HashMap<String, String> readAttributes(final int headerSize) {
+ final HashMap<String, String> attributes = new HashMap<String, String>();
+ while (mFusionDictionaryBuffer.position() < headerSize) {
+ // We can avoid infinite loop here since mFusionDictonary.position() is always increased
+ // by calling CharEncoding.readString.
+ final String key = CharEncoding.readString(mFusionDictionaryBuffer);
+ final String value = CharEncoding.readString(mFusionDictionaryBuffer);
+ attributes.put(key, value);
+ }
+ mFusionDictionaryBuffer.position(headerSize);
+ return attributes;
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
index 5b10912..584b793 100644
--- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
@@ -18,7 +18,7 @@
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
@@ -49,17 +49,18 @@
/**
* Delete the word from the binary file.
*
- * @param buffer the buffer to write.
+ * @param dictReader the dict reader.
* @param word the word we delete
* @throws IOException
* @throws UnsupportedFormatException
*/
@UsedForTesting
- public static void deleteWord(final FusionDictionaryBufferInterface buffer,
- final String word) throws IOException, UnsupportedFormatException {
+ public static void deleteWord(final BinaryDictReader dictReader, final String word)
+ throws IOException, UnsupportedFormatException {
+ final FusionDictionaryBufferInterface buffer = dictReader.getBuffer();
buffer.position(0);
- final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
- final int wordPosition = BinaryDictIOUtils.getTerminalPosition(buffer, word);
+ final FileHeader header = BinaryDictDecoder.readHeader(dictReader);
+ final int wordPosition = BinaryDictIOUtils.getTerminalPosition(dictReader, word);
if (wordPosition == FormatSpec.NOT_VALID_WORD) return;
buffer.position(wordPosition);
@@ -86,7 +87,7 @@
}
final int flags = buffer.readUnsignedByte();
if (BinaryDictIOUtils.isMovedGroup(flags, formatOptions)) {
- // if the group is moved, the parent address is stored in the destination group.
+ // If the group is moved, the parent address is stored in the destination group.
// We are guaranteed to process the destination group later, so there is no need to
// update anything here.
buffer.position(originalPosition);
@@ -101,10 +102,10 @@
}
/**
- * Update parent addresses in a Node that is referred to by nodeOriginAddress.
+ * Update parent addresses in a node array stored at nodeOriginAddress.
*
* @param buffer the buffer to be modified.
- * @param nodeOriginAddress the address of a modified Node.
+ * @param nodeOriginAddress the address of the node array to update.
* @param newParentAddress the address to be written.
* @param formatOptions file format options.
*/
@@ -114,7 +115,7 @@
final int originalPosition = buffer.position();
buffer.position(nodeOriginAddress);
do {
- final int count = BinaryDictInputOutput.readCharGroupCount(buffer);
+ final int count = BinaryDictDecoder.readCharGroupCount(buffer);
for (int i = 0; i < count; ++i) {
updateParentAddress(buffer, buffer.position(), newParentAddress, formatOptions);
BinaryDictIOUtils.skipCharGroup(buffer, formatOptions);
@@ -140,7 +141,7 @@
final int originalPosition = buffer.position();
buffer.position(groupOriginAddress);
final int flags = buffer.readUnsignedByte();
- final int parentAddress = BinaryDictInputOutput.readParentAddress(buffer, formatOptions);
+ final int parentAddress = BinaryDictDecoder.readParentAddress(buffer, formatOptions);
BinaryDictIOUtils.skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte();
final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS
@@ -154,7 +155,7 @@
*/
private static int moveCharGroup(final OutputStream destination,
final FusionDictionaryBufferInterface buffer, final CharGroupInfo info,
- final int nodeOriginAddress, final int oldGroupAddress,
+ final int nodeArrayOriginAddress, final int oldGroupAddress,
final FormatOptions formatOptions) throws IOException {
updateParentAddress(buffer, oldGroupAddress, buffer.limit() + 1, formatOptions);
buffer.position(oldGroupAddress);
@@ -163,23 +164,24 @@
buffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags
& (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG))));
int size = FormatSpec.GROUP_FLAGS_SIZE;
- updateForwardLink(buffer, nodeOriginAddress, buffer.limit(), formatOptions);
- size += BinaryDictIOUtils.writeNode(destination, new CharGroupInfo[] { info });
+ updateForwardLink(buffer, nodeArrayOriginAddress, buffer.limit(), formatOptions);
+ size += BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[] { info });
return size;
}
+
@SuppressWarnings("unused")
private static void updateForwardLink(final FusionDictionaryBufferInterface buffer,
- final int nodeOriginAddress, final int newNodeAddress,
+ final int nodeArrayOriginAddress, final int newNodeArrayAddress,
final FormatOptions formatOptions) {
- buffer.position(nodeOriginAddress);
+ buffer.position(nodeArrayOriginAddress);
int jumpCount = 0;
while (jumpCount++ < MAX_JUMPS) {
- final int count = BinaryDictInputOutput.readCharGroupCount(buffer);
+ final int count = BinaryDictDecoder.readCharGroupCount(buffer);
for (int i = 0; i < count; ++i) BinaryDictIOUtils.skipCharGroup(buffer, formatOptions);
final int forwardLinkAddress = buffer.readUnsignedInt24();
if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) {
buffer.position(buffer.position() - FormatSpec.FORWARD_LINK_ADDRESS_SIZE);
- BinaryDictIOUtils.writeSInt24ToBuffer(buffer, newNodeAddress);
+ BinaryDictIOUtils.writeSInt24ToBuffer(buffer, newNodeArrayAddress);
return;
}
buffer.position(forwardLinkAddress);
@@ -190,71 +192,73 @@
}
/**
- * Move a group that is referred to by oldGroupOrigin to the tail of the file.
- * And set the children address to the byte after the group.
+ * Move a group that is referred to by oldGroupOrigin to the tail of the file, and set the
+ * children address to the byte after the group
*
- * @param nodeOrigin the address of the tail of the file.
- * @param characters
- * @param length
- * @param flags
- * @param frequency
- * @param parentAddress
- * @param shortcutTargets
- * @param bigrams
+ * @param fileEndAddress the address of the tail of the file.
+ * @param codePoints the characters to put inside the group.
+ * @param length how many code points to read from codePoints.
+ * @param flags the flags for this group.
+ * @param frequency the frequency of this terminal.
+ * @param parentAddress the address of the parent group of this group.
+ * @param shortcutTargets the shortcut targets for this group.
+ * @param bigrams the bigrams for this group.
* @param destination the stream representing the tail of the file.
* @param buffer the buffer representing the (constant-size) body of the file.
- * @param oldNodeOrigin
- * @param oldGroupOrigin
- * @param formatOptions
+ * @param oldNodeArrayOrigin the origin of the old node array this group was a part of.
+ * @param oldGroupOrigin the old origin where this group used to be stored.
+ * @param formatOptions format options for this dictionary.
* @return the size written, in bytes.
- * @throws IOException
+ * @throws IOException if the file can't be accessed
*/
- private static int moveGroup(final int nodeOrigin, final int[] characters, final int length,
- final int flags, final int frequency, final int parentAddress,
+ private static int moveGroup(final int fileEndAddress, final int[] codePoints,
+ final int length, final int flags, final int frequency, final int parentAddress,
final ArrayList<WeightedString> shortcutTargets,
final ArrayList<PendingAttribute> bigrams, final OutputStream destination,
- final FusionDictionaryBufferInterface buffer, final int oldNodeOrigin,
+ final FusionDictionaryBufferInterface buffer, final int oldNodeArrayOrigin,
final int oldGroupOrigin, final FormatOptions formatOptions) throws IOException {
int size = 0;
- final int newGroupOrigin = nodeOrigin + 1;
- final int[] writtenCharacters = Arrays.copyOfRange(characters, 0, length);
+ final int newGroupOrigin = fileEndAddress + 1;
+ final int[] writtenCharacters = Arrays.copyOfRange(codePoints, 0, length);
final CharGroupInfo tmpInfo = new CharGroupInfo(newGroupOrigin, -1 /* endAddress */,
flags, writtenCharacters, frequency, parentAddress, FormatSpec.NO_CHILDREN_ADDRESS,
shortcutTargets, bigrams);
size = BinaryDictIOUtils.computeGroupSize(tmpInfo, formatOptions);
final CharGroupInfo newInfo = new CharGroupInfo(newGroupOrigin, newGroupOrigin + size,
flags, writtenCharacters, frequency, parentAddress,
- nodeOrigin + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets,
+ fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets,
bigrams);
- moveCharGroup(destination, buffer, newInfo, oldNodeOrigin, oldGroupOrigin, formatOptions);
+ moveCharGroup(destination, buffer, newInfo, oldNodeArrayOrigin, oldGroupOrigin,
+ formatOptions);
return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
}
/**
* Insert a word into a binary dictionary.
*
- * @param buffer
- * @param destination
- * @param word
- * @param frequency
- * @param bigramStrings
- * @param shortcuts
- * @throws IOException
- * @throws UnsupportedFormatException
+ * @param dictReader the dict reader.
+ * @param destination a stream to the underlying file, with the pointer at the end of the file.
+ * @param word the word to insert.
+ * @param frequency the frequency of the new word.
+ * @param bigramStrings bigram list, or null if none.
+ * @param shortcuts shortcut list, or null if none.
+ * @param isBlackListEntry whether this should be a blacklist entry.
+ * @throws IOException if the file can't be accessed.
+ * @throws UnsupportedFormatException if the existing dictionary is in an unexpected format.
*/
// TODO: Support batch insertion.
// TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary.
@UsedForTesting
- public static void insertWord(final FusionDictionaryBufferInterface buffer,
- final OutputStream destination, final String word, final int frequency,
- final ArrayList<WeightedString> bigramStrings,
+ public static void insertWord(final BinaryDictReader dictReader, final OutputStream destination,
+ final String word, final int frequency, final ArrayList<WeightedString> bigramStrings,
final ArrayList<WeightedString> shortcuts, final boolean isNotAWord,
final boolean isBlackListEntry)
throws IOException, UnsupportedFormatException {
final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>();
+ final FusionDictionaryBufferInterface buffer = dictReader.getBuffer();
if (bigramStrings != null) {
for (final WeightedString bigram : bigramStrings) {
- int position = BinaryDictIOUtils.getTerminalPosition(buffer, bigram.mWord);
+ int position = BinaryDictIOUtils.getTerminalPosition(dictReader, bigram.mWord);
if (position == FormatSpec.NOT_VALID_WORD) {
// TODO: figure out what is the correct thing to do here.
} else {
@@ -269,7 +273,7 @@
// find the insert position of the word.
if (buffer.position() != 0) buffer.position(0);
- final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
+ final FileHeader fileHeader = BinaryDictDecoder.readHeader(dictReader);
int wordPos = 0, address = buffer.position(), nodeOriginAddress = buffer.position();
final int[] codePoints = FusionDictionary.getCodePoints(word);
@@ -279,15 +283,15 @@
if (wordPos >= wordLen) break;
nodeOriginAddress = buffer.position();
int nodeParentAddress = -1;
- final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer);
+ final int charGroupCount = BinaryDictDecoder.readCharGroupCount(buffer);
boolean foundNextGroup = false;
for (int i = 0; i < charGroupCount; ++i) {
address = buffer.position();
- final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer,
- buffer.position(), header.mFormatOptions);
+ final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer,
+ buffer.position(), fileHeader.mFormatOptions);
final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags,
- header.mFormatOptions);
+ fileHeader.mFormatOptions);
if (isMovedGroup) continue;
nodeParentAddress = (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS)
? FormatSpec.NO_PARENT_ADDRESS : currentInfo.mParentAddress + address;
@@ -305,25 +309,25 @@
* abc - d - ef
*/
final int newNodeAddress = buffer.limit();
- final int flags = BinaryDictInputOutput.makeCharGroupFlags(p > 1,
+ final int flags = BinaryDictEncoder.makeCharGroupFlags(p > 1,
isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */,
- false /* isBlackListEntry */, header.mFormatOptions);
+ false /* isBlackListEntry */, fileHeader.mFormatOptions);
int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags,
frequency, nodeParentAddress, shortcuts, bigrams, destination,
- buffer, nodeOriginAddress, address, header.mFormatOptions);
+ buffer, nodeOriginAddress, address, fileHeader.mFormatOptions);
final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p,
currentInfo.mCharacters.length);
if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
updateParentAddresses(buffer, currentInfo.mChildrenAddress,
- newNodeAddress + written + 1, header.mFormatOptions);
+ newNodeAddress + written + 1, fileHeader.mFormatOptions);
}
final CharGroupInfo newInfo2 = new CharGroupInfo(
newNodeAddress + written + 1, -1 /* endAddress */,
currentInfo.mFlags, characters2, currentInfo.mFrequency,
newNodeAddress + 1, currentInfo.mChildrenAddress,
currentInfo.mShortcutTargets, currentInfo.mBigrams);
- BinaryDictIOUtils.writeNode(destination, new CharGroupInfo[] { newInfo2 });
+ BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[] { newInfo2 });
return;
} else if (codePoints[wordPos + p] != currentInfo.mCharacters[p]) {
if (p > 0) {
@@ -344,49 +348,49 @@
final int childrenAddress = currentInfo.mChildrenAddress;
// move prefix
- final int prefixFlags = BinaryDictInputOutput.makeCharGroupFlags(p > 1,
+ final int prefixFlags = BinaryDictEncoder.makeCharGroupFlags(p > 1,
false /* isTerminal */, 0 /* childrenAddressSize*/,
false /* hasShortcut */, false /* hasBigrams */,
false /* isNotAWord */, false /* isBlackListEntry */,
- header.mFormatOptions);
+ fileHeader.mFormatOptions);
int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p,
prefixFlags, -1 /* frequency */, nodeParentAddress, null, null,
destination, buffer, nodeOriginAddress, address,
- header.mFormatOptions);
+ fileHeader.mFormatOptions);
final int[] suffixCharacters = Arrays.copyOfRange(
currentInfo.mCharacters, p, currentInfo.mCharacters.length);
if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
updateParentAddresses(buffer, currentInfo.mChildrenAddress,
- newNodeAddress + written + 1, header.mFormatOptions);
+ newNodeAddress + written + 1, fileHeader.mFormatOptions);
}
- final int suffixFlags = BinaryDictInputOutput.makeCharGroupFlags(
+ final int suffixFlags = BinaryDictEncoder.makeCharGroupFlags(
suffixCharacters.length > 1,
(currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0,
0 /* childrenAddressSize */,
(currentInfo.mFlags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)
!= 0,
(currentInfo.mFlags & FormatSpec.FLAG_HAS_BIGRAMS) != 0,
- isNotAWord, isBlackListEntry, header.mFormatOptions);
+ isNotAWord, isBlackListEntry, fileHeader.mFormatOptions);
final CharGroupInfo suffixInfo = new CharGroupInfo(
newNodeAddress + written + 1, -1 /* endAddress */, suffixFlags,
suffixCharacters, currentInfo.mFrequency, newNodeAddress + 1,
currentInfo.mChildrenAddress, currentInfo.mShortcutTargets,
currentInfo.mBigrams);
written += BinaryDictIOUtils.computeGroupSize(suffixInfo,
- header.mFormatOptions) + 1;
+ fileHeader.mFormatOptions) + 1;
final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p,
codePoints.length);
- final int flags = BinaryDictInputOutput.makeCharGroupFlags(
+ final int flags = BinaryDictEncoder.makeCharGroupFlags(
newCharacters.length > 1, isTerminal,
0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
- isNotAWord, isBlackListEntry, header.mFormatOptions);
+ isNotAWord, isBlackListEntry, fileHeader.mFormatOptions);
final CharGroupInfo newInfo = new CharGroupInfo(
newNodeAddress + written, -1 /* endAddress */, flags,
newCharacters, frequency, newNodeAddress + 1,
FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams);
- BinaryDictIOUtils.writeNode(destination,
+ BinaryDictIOUtils.writeNodes(destination,
new CharGroupInfo[] { suffixInfo, newInfo });
return;
}
@@ -401,15 +405,15 @@
// only update group.
final int newNodeAddress = buffer.limit();
final boolean hasMultipleChars = currentInfo.mCharacters.length > 1;
- final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars,
+ final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars,
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
- isNotAWord, isBlackListEntry, header.mFormatOptions);
+ isNotAWord, isBlackListEntry, fileHeader.mFormatOptions);
final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1,
-1 /* endAddress */, flags, currentInfo.mCharacters, frequency,
nodeParentAddress, currentInfo.mChildrenAddress, shortcuts,
bigrams);
moveCharGroup(destination, buffer, newInfo, nodeOriginAddress, address,
- header.mFormatOptions);
+ fileHeader.mFormatOptions);
return;
}
wordPos += currentInfo.mCharacters.length;
@@ -428,17 +432,17 @@
*/
final int newNodeAddress = buffer.limit();
updateChildrenAddress(buffer, address, newNodeAddress,
- header.mFormatOptions);
+ fileHeader.mFormatOptions);
final int newGroupAddress = newNodeAddress + 1;
final boolean hasMultipleChars = (wordLen - wordPos) > 1;
- final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars,
+ final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars,
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
- isNotAWord, isBlackListEntry, header.mFormatOptions);
+ isNotAWord, isBlackListEntry, fileHeader.mFormatOptions);
final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen);
final CharGroupInfo newInfo = new CharGroupInfo(newGroupAddress, -1, flags,
characters, frequency, address, FormatSpec.NO_CHILDREN_ADDRESS,
shortcuts, bigrams);
- BinaryDictIOUtils.writeNode(destination, new CharGroupInfo[] { newInfo });
+ BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[] { newInfo });
return;
}
buffer.position(currentInfo.mChildrenAddress);
@@ -476,13 +480,13 @@
BinaryDictIOUtils.writeSInt24ToBuffer(buffer, newNodeAddress);
final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen);
- final int flags = BinaryDictInputOutput.makeCharGroupFlags(characters.length > 1,
+ final int flags = BinaryDictEncoder.makeCharGroupFlags(characters.length > 1,
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
- isNotAWord, isBlackListEntry, header.mFormatOptions);
+ isNotAWord, isBlackListEntry, fileHeader.mFormatOptions);
final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1,
-1 /* endAddress */, flags, characters, frequency, nodeParentAddress,
FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams);
- BinaryDictIOUtils.writeNode(destination, new CharGroupInfo[]{ newInfo });
+ BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[]{ newInfo });
return;
} else {
depth--;
diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
index 9af66ed..5e33121 100644
--- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
+++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
@@ -60,7 +60,7 @@
*/
/*
- * Array of Node(FusionDictionary.Node) layout is as follows:
+ * Node array (FusionDictionary.PtNodeArray) layout is as follows:
*
* g |
* r | the number of groups, 1 or 2 bytes.
@@ -86,7 +86,7 @@
* linkaddress
*/
- /* Node(CharGroup) layout is as follows:
+ /* Node (FusionDictionary.CharGroup) layout is as follows:
* | IF !SUPPORTS_DYNAMIC_UPDATE
* | addressType xx : mask with MASK_GROUP_ADDRESS_TYPE
* | 2 bits, 00 = no children : FLAG_GROUP_ADDRESS_TYPE_NOADDRESS
@@ -251,7 +251,7 @@
static final int INVALID_CHARACTER = -1;
static final int MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT = 0x7F; // 127
- static final int MAX_CHARGROUPS_IN_A_NODE = 0x7FFF; // 32767
+ static final int MAX_CHARGROUPS_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767
static final int MAX_BIGRAMS_IN_A_GROUP = 10000;
static final int MAX_TERMINAL_FREQUENCY = 255;
diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
index 118dc22..fce1c5c 100644
--- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
+++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
@@ -37,14 +37,14 @@
private static int CHARACTER_NOT_FOUND_INDEX = -1;
/**
- * A node of the dictionary, containing several CharGroups.
+ * A node array of the dictionary, containing several CharGroups.
*
- * A node is but an ordered array of CharGroups, which essentially contain all the
+ * A PtNodeArray is but an ordered array of CharGroups, which essentially contain all the
* real information.
* This class also contains fields to cache size and address, to help with binary
* generation.
*/
- public static final class Node {
+ public static final class PtNodeArray {
ArrayList<CharGroup> mData;
// To help with binary generation
int mCachedSize = Integer.MIN_VALUE;
@@ -57,10 +57,10 @@
int mCachedAddressAfterUpdate = Integer.MIN_VALUE;
int mCachedParentAddress = 0;
- public Node() {
+ public PtNodeArray() {
mData = new ArrayList<CharGroup>();
}
- public Node(ArrayList<CharGroup> data) {
+ public PtNodeArray(ArrayList<CharGroup> data) {
mData = data;
}
}
@@ -98,7 +98,7 @@
* This is the central class of the in-memory representation. A CharGroup is what can
* be seen as a traditional "trie node", except it can hold several characters at the
* same time. A CharGroup essentially represents one or several characters in the middle
- * of the trie trie; as such, it can be a terminal, and it can have children.
+ * of the trie tree; as such, it can be a terminal, and it can have children.
* In this in-memory representation, whether the CharGroup is a terminal or not is represented
* in the frequency, where NOT_A_TERMINAL (= -1) means this is not a terminal and any other
* value is the frequency of this terminal. A terminal may have non-null shortcuts and/or
@@ -110,7 +110,7 @@
ArrayList<WeightedString> mShortcutTargets;
ArrayList<WeightedString> mBigrams;
int mFrequency; // NOT_A_TERMINAL == mFrequency indicates this is not a terminal.
- Node mChildren;
+ PtNodeArray mChildren;
boolean mIsNotAWord; // Only a shortcut
boolean mIsBlacklistEntry;
// mCachedSize and mCachedAddressBefore/AfterUpdate are helpers for binary dictionary
@@ -137,7 +137,8 @@
public CharGroup(final int[] chars, final ArrayList<WeightedString> shortcutTargets,
final ArrayList<WeightedString> bigrams, final int frequency,
- final boolean isNotAWord, final boolean isBlacklistEntry, final Node children) {
+ final boolean isNotAWord, final boolean isBlacklistEntry,
+ final PtNodeArray children) {
mChars = chars;
mFrequency = frequency;
mShortcutTargets = shortcutTargets;
@@ -149,7 +150,7 @@
public void addChild(CharGroup n) {
if (null == mChildren) {
- mChildren = new Node();
+ mChildren = new PtNodeArray();
}
mChildren.mData.add(n);
}
@@ -344,10 +345,10 @@
}
public final DictionaryOptions mOptions;
- public final Node mRoot;
+ public final PtNodeArray mRootNodeArray;
- public FusionDictionary(final Node root, final DictionaryOptions options) {
- mRoot = root;
+ public FusionDictionary(final PtNodeArray rootNodeArray, final DictionaryOptions options) {
+ mRootNodeArray = rootNodeArray;
mOptions = options;
}
@@ -406,13 +407,13 @@
}
/**
- * Sanity check for a node.
+ * Sanity check for a node array.
*
- * This method checks that all CharGroups in a node are ordered as expected.
+ * This method checks that all CharGroups in a node array are ordered as expected.
* If they are, nothing happens. If they aren't, an exception is thrown.
*/
- private void checkStack(Node node) {
- ArrayList<CharGroup> stack = node.mData;
+ private void checkStack(PtNodeArray nodeArray) {
+ ArrayList<CharGroup> stack = nodeArray.mData;
int lastValue = -1;
for (int i = 0; i < stack.size(); ++i) {
int currentValue = stack.get(i).mChars[0];
@@ -431,16 +432,16 @@
* @param frequency the bigram frequency
*/
public void setBigram(final String word1, final String word2, final int frequency) {
- CharGroup charGroup = findWordInTree(mRoot, word1);
+ CharGroup charGroup = findWordInTree(mRootNodeArray, word1);
if (charGroup != null) {
- final CharGroup charGroup2 = findWordInTree(mRoot, word2);
+ final CharGroup charGroup2 = findWordInTree(mRootNodeArray, word2);
if (charGroup2 == null) {
add(getCodePoints(word2), 0, null, false /* isNotAWord */,
false /* isBlacklistEntry */);
// The chargroup for the first word may have moved by the above insertion,
// if word1 and word2 share a common stem that happens not to have been
// a cutting point until now. In this case, we need to refresh charGroup.
- charGroup = findWordInTree(mRoot, word1);
+ charGroup = findWordInTree(mRootNodeArray, word1);
}
charGroup.addBigram(word2, frequency);
} else {
@@ -469,38 +470,38 @@
return;
}
- Node currentNode = mRoot;
+ PtNodeArray currentNodeArray = mRootNodeArray;
int charIndex = 0;
CharGroup currentGroup = null;
int differentCharIndex = 0; // Set by the loop to the index of the char that differs
- int nodeIndex = findIndexOfChar(mRoot, word[charIndex]);
+ int nodeIndex = findIndexOfChar(mRootNodeArray, word[charIndex]);
while (CHARACTER_NOT_FOUND_INDEX != nodeIndex) {
- currentGroup = currentNode.mData.get(nodeIndex);
- differentCharIndex = compareArrays(currentGroup.mChars, word, charIndex);
+ currentGroup = currentNodeArray.mData.get(nodeIndex);
+ differentCharIndex = compareCharArrays(currentGroup.mChars, word, charIndex);
if (ARRAYS_ARE_EQUAL != differentCharIndex
&& differentCharIndex < currentGroup.mChars.length) break;
if (null == currentGroup.mChildren) break;
charIndex += currentGroup.mChars.length;
if (charIndex >= word.length) break;
- currentNode = currentGroup.mChildren;
- nodeIndex = findIndexOfChar(currentNode, word[charIndex]);
+ currentNodeArray = currentGroup.mChildren;
+ nodeIndex = findIndexOfChar(currentNodeArray, word[charIndex]);
}
if (CHARACTER_NOT_FOUND_INDEX == nodeIndex) {
// No node at this point to accept the word. Create one.
- final int insertionIndex = findInsertionIndex(currentNode, word[charIndex]);
+ final int insertionIndex = findInsertionIndex(currentNodeArray, word[charIndex]);
final CharGroup newGroup = new CharGroup(
Arrays.copyOfRange(word, charIndex, word.length),
shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry);
- currentNode.mData.add(insertionIndex, newGroup);
- if (DBG) checkStack(currentNode);
+ currentNodeArray.mData.add(insertionIndex, newGroup);
+ if (DBG) checkStack(currentNodeArray);
} else {
// There is a word with a common prefix.
if (differentCharIndex == currentGroup.mChars.length) {
if (charIndex + differentCharIndex >= word.length) {
// The new word is a prefix of an existing word, but the node on which it
- // should end already exists as is. Since the old CharNode was not a terminal,
+ // should end already exists as is. Since the old CharGroup was not a terminal,
// make it one by filling in its frequency and other attributes
currentGroup.update(frequency, shortcutTargets, null, isNotAWord,
isBlacklistEntry);
@@ -511,7 +512,7 @@
Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length),
shortcutTargets, null /* bigrams */, frequency, isNotAWord,
isBlacklistEntry);
- currentGroup.mChildren = new Node();
+ currentGroup.mChildren = new PtNodeArray();
currentGroup.mChildren.mData.add(newNode);
}
} else {
@@ -524,7 +525,7 @@
} else {
// Partial prefix match only. We have to replace the current node with a node
// containing the current prefix and create two new ones for the tails.
- Node newChildren = new Node();
+ PtNodeArray newChildren = new PtNodeArray();
final CharGroup newOldWord = new CharGroup(
Arrays.copyOfRange(currentGroup.mChars, differentCharIndex,
currentGroup.mChars.length), currentGroup.mShortcutTargets,
@@ -552,9 +553,9 @@
> currentGroup.mChars[differentCharIndex] ? 1 : 0;
newChildren.mData.add(addIndex, newWord);
}
- currentNode.mData.set(nodeIndex, newParent);
+ currentNodeArray.mData.set(nodeIndex, newParent);
}
- if (DBG) checkStack(currentNode);
+ if (DBG) checkStack(currentNodeArray);
}
}
}
@@ -576,7 +577,7 @@
* @param dstOffset the offset in the right-hand side string.
* @return the index at which the strings differ, or ARRAYS_ARE_EQUAL = 0 if they don't.
*/
- private static int compareArrays(final int[] src, final int[] dst, int dstOffset) {
+ private static int compareCharArrays(final int[] src, final int[] dst, int dstOffset) {
// We do NOT test the first char, because we come from a method that already
// tested it.
for (int i = 1; i < src.length; ++i) {
@@ -603,10 +604,10 @@
final static private CharGroupComparator CHARGROUP_COMPARATOR = new CharGroupComparator();
/**
- * Finds the insertion index of a character within a node.
+ * Finds the insertion index of a character within a node array.
*/
- private static int findInsertionIndex(final Node node, int character) {
- final ArrayList<CharGroup> data = node.mData;
+ private static int findInsertionIndex(final PtNodeArray nodeArray, int character) {
+ final ArrayList<CharGroup> data = nodeArray.mData;
final CharGroup reference = new CharGroup(new int[] { character },
null /* shortcutTargets */, null /* bigrams */, 0, false /* isNotAWord */,
false /* isBlacklistEntry */);
@@ -615,16 +616,16 @@
}
/**
- * Find the index of a char in a node, if it exists.
+ * Find the index of a char in a node array, if it exists.
*
- * @param node the node to search in.
+ * @param nodeArray the node array to search in.
* @param character the character to search for.
* @return the position of the character if it's there, or CHARACTER_NOT_FOUND_INDEX = -1 else.
*/
- private static int findIndexOfChar(final Node node, int character) {
- final int insertionIndex = findInsertionIndex(node, character);
- if (node.mData.size() <= insertionIndex) return CHARACTER_NOT_FOUND_INDEX;
- return character == node.mData.get(insertionIndex).mChars[0] ? insertionIndex
+ private static int findIndexOfChar(final PtNodeArray nodeArray, int character) {
+ final int insertionIndex = findInsertionIndex(nodeArray, character);
+ if (nodeArray.mData.size() <= insertionIndex) return CHARACTER_NOT_FOUND_INDEX;
+ return character == nodeArray.mData.get(insertionIndex).mChars[0] ? insertionIndex
: CHARACTER_NOT_FOUND_INDEX;
}
@@ -632,16 +633,16 @@
* Helper method to find a word in a given branch.
*/
@SuppressWarnings("unused")
- public static CharGroup findWordInTree(Node node, final String string) {
+ public static CharGroup findWordInTree(PtNodeArray nodeArray, final String string) {
int index = 0;
final StringBuilder checker = DBG ? new StringBuilder() : null;
final int[] codePoints = getCodePoints(string);
CharGroup currentGroup;
do {
- int indexOfGroup = findIndexOfChar(node, codePoints[index]);
+ int indexOfGroup = findIndexOfChar(nodeArray, codePoints[index]);
if (CHARACTER_NOT_FOUND_INDEX == indexOfGroup) return null;
- currentGroup = node.mData.get(indexOfGroup);
+ currentGroup = nodeArray.mData.get(indexOfGroup);
if (codePoints.length - index < currentGroup.mChars.length) return null;
int newIndex = index;
@@ -653,9 +654,9 @@
if (DBG) checker.append(new String(currentGroup.mChars, 0, currentGroup.mChars.length));
if (index < codePoints.length) {
- node = currentGroup.mChildren;
+ nodeArray = currentGroup.mChildren;
}
- } while (null != node && index < codePoints.length);
+ } while (null != nodeArray && index < codePoints.length);
if (index < codePoints.length) return null;
if (!currentGroup.isTerminal()) return null;
@@ -670,20 +671,20 @@
if (null == s || "".equals(s)) {
throw new RuntimeException("Can't search for a null or empty string");
}
- return null != findWordInTree(mRoot, s);
+ return null != findWordInTree(mRootNodeArray, s);
}
/**
* Recursively count the number of character groups in a given branch of the trie.
*
- * @param node the parent node.
+ * @param nodeArray the parent node.
* @return the number of char groups in all the branch under this node.
*/
- public static int countCharGroups(final Node node) {
- final int nodeSize = node.mData.size();
+ public static int countCharGroups(final PtNodeArray nodeArray) {
+ final int nodeSize = nodeArray.mData.size();
int size = nodeSize;
for (int i = nodeSize - 1; i >= 0; --i) {
- CharGroup group = node.mData.get(i);
+ CharGroup group = nodeArray.mData.get(i);
if (null != group.mChildren)
size += countCharGroups(group.mChildren);
}
@@ -693,15 +694,15 @@
/**
* Recursively count the number of nodes in a given branch of the trie.
*
- * @param node the node to count.
+ * @param nodeArray the node array to count.
* @return the number of nodes in this branch.
*/
- public static int countNodes(final Node node) {
+ public static int countNodeArrays(final PtNodeArray nodeArray) {
int size = 1;
- for (int i = node.mData.size() - 1; i >= 0; --i) {
- CharGroup group = node.mData.get(i);
+ for (int i = nodeArray.mData.size() - 1; i >= 0; --i) {
+ CharGroup group = nodeArray.mData.get(i);
if (null != group.mChildren)
- size += countNodes(group.mChildren);
+ size += countNodeArrays(group.mChildren);
}
return size;
}
@@ -709,10 +710,10 @@
// Recursively find out whether there are any bigrams.
// This can be pretty expensive especially if there aren't any (we return as soon
// as we find one, so it's much cheaper if there are bigrams)
- private static boolean hasBigramsInternal(final Node node) {
- if (null == node) return false;
- for (int i = node.mData.size() - 1; i >= 0; --i) {
- CharGroup group = node.mData.get(i);
+ private static boolean hasBigramsInternal(final PtNodeArray nodeArray) {
+ if (null == nodeArray) return false;
+ for (int i = nodeArray.mData.size() - 1; i >= 0; --i) {
+ CharGroup group = nodeArray.mData.get(i);
if (null != group.mBigrams) return true;
if (hasBigramsInternal(group.mChildren)) return true;
}
@@ -729,7 +730,7 @@
// find a more efficient way of doing this, without compromising too much on memory
// and ease of use.
public boolean hasBigrams() {
- return hasBigramsInternal(mRoot);
+ return hasBigramsInternal(mRootNodeArray);
}
// Historically, the tails of the words were going to be merged to save space.
@@ -750,13 +751,13 @@
// MakedictLog.i("Merging nodes. Number of nodes : " + countNodes(root));
// MakedictLog.i("Number of groups : " + countCharGroups(root));
//
-// final HashMap<String, ArrayList<Node>> repository =
-// new HashMap<String, ArrayList<Node>>();
+// final HashMap<String, ArrayList<PtNodeArray>> repository =
+// new HashMap<String, ArrayList<PtNodeArray>>();
// mergeTailsInner(repository, root);
//
// MakedictLog.i("Number of different pseudohashes : " + repository.size());
// int size = 0;
-// for (ArrayList<Node> a : repository.values()) {
+// for (ArrayList<PtNodeArray> a : repository.values()) {
// size += a.size();
// }
// MakedictLog.i("Number of nodes after merge : " + (1 + size));
@@ -764,7 +765,7 @@
}
// The following methods are used by the deactivated mergeTails()
-// private static boolean isEqual(Node a, Node b) {
+// private static boolean isEqual(PtNodeArray a, PtNodeArray b) {
// if (null == a && null == b) return true;
// if (null == a || null == b) return false;
// if (a.data.size() != b.data.size()) return false;
@@ -781,21 +782,21 @@
// return true;
// }
-// static private HashMap<String, ArrayList<Node>> mergeTailsInner(
-// final HashMap<String, ArrayList<Node>> map, final Node node) {
-// final ArrayList<CharGroup> branches = node.data;
+// static private HashMap<String, ArrayList<PtNodeArray>> mergeTailsInner(
+// final HashMap<String, ArrayList<PtNodeArray>> map, final PtNodeArray nodeArray) {
+// final ArrayList<CharGroup> branches = nodeArray.data;
// final int nodeSize = branches.size();
// for (int i = 0; i < nodeSize; ++i) {
// CharGroup group = branches.get(i);
// if (null != group.children) {
// String pseudoHash = getPseudoHash(group.children);
-// ArrayList<Node> similarList = map.get(pseudoHash);
+// ArrayList<PtNodeArray> similarList = map.get(pseudoHash);
// if (null == similarList) {
-// similarList = new ArrayList<Node>();
+// similarList = new ArrayList<PtNodeArray>();
// map.put(pseudoHash, similarList);
// }
// boolean merged = false;
-// for (Node similar : similarList) {
+// for (PtNodeArray similar : similarList) {
// if (isEqual(group.children, similar)) {
// group.children = similar;
// merged = true;
@@ -811,9 +812,9 @@
// return map;
// }
-// private static String getPseudoHash(final Node node) {
+// private static String getPseudoHash(final PtNodeArray nodeArray) {
// StringBuilder s = new StringBuilder();
-// for (CharGroup g : node.data) {
+// for (CharGroup g : nodeArray.data) {
// s.append(g.frequency);
// for (int ch : g.chars) {
// s.append(Character.toChars(ch));
@@ -901,6 +902,6 @@
*/
@Override
public Iterator<Word> iterator() {
- return new DictionaryIterator(mRoot.mData);
+ return new DictionaryIterator(mRootNodeArray.mData);
}
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/decoder/HeaderReaderInterface.java b/java/src/com/android/inputmethod/latin/makedict/decoder/HeaderReaderInterface.java
new file mode 100644
index 0000000..7cddef2
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/makedict/decoder/HeaderReaderInterface.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.makedict.decoder;
+
+import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/**
+ * An interface to read a binary dictionary file header.
+ */
+public interface HeaderReaderInterface {
+ public int readVersion() throws IOException, UnsupportedFormatException;
+ public int readOptionFlags();
+ public int readHeaderSize();
+ public HashMap<String, String> readAttributes(final int headerSize);
+}
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
index 065e00e..525d3cd 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
@@ -43,6 +43,7 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
/**
@@ -75,6 +76,8 @@
private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions =
CollectionUtils.newArrayList();
+ private final AtomicReference<AsyncTask<Void, Void, Void>> mWaitingTask;
+
// Should always be false except when we use this class for test
@UsedForTesting boolean mIsTest = false;
@@ -83,6 +86,7 @@
super(context, dictionaryType);
mLocale = locale;
mPrefs = sp;
+ mWaitingTask = new AtomicReference<AsyncTask<Void, Void, Void>>();
if (mLocale != null && mLocale.length() > 1) {
loadDictionary();
}
@@ -174,7 +178,11 @@
*/
private void flushPendingWrites() {
// Create a background thread to write the pending entries
- new UpdateBinaryTask(mBigramList, mLocale, this, mPrefs, getContext()).execute();
+ final AsyncTask<Void, Void, Void> old = mWaitingTask.getAndSet(new UpdateBinaryTask(
+ mBigramList, mLocale, this, mPrefs, getContext()).execute());
+ if (old != null) {
+ old.cancel(false);
+ }
}
@Override
@@ -287,6 +295,7 @@
@Override
protected Void doInBackground(final Void... v) {
+ if (isCancelled()) return null;
if (mDynamicPredictionDictionary.mIsTest) {
// If mIsTest == true, wait until the lock is released.
mDynamicPredictionDictionary.mBigramListLock.lock();
@@ -306,6 +315,9 @@
}
private void doWriteTaskLocked() {
+ if (isCancelled()) return;
+ mDynamicPredictionDictionary.mWaitingTask.compareAndSet(this, null);
+
if (DBG_STRESS_TEST) {
try {
Log.w(TAG, "Start stress in closing: " + mLocale);
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
index a755f90..858aa32 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
@@ -48,6 +48,8 @@
public abstract void onDictionaryReady();
+ public abstract void onDictionaryClosed();
+
public void setPredictionDictionary(String locale, DynamicPredictionDictionaryBase dictionary) {
mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary);
mLocale = locale;
@@ -68,6 +70,7 @@
public void closeSession() {
unsetPredictionDictionary();
+ onDictionaryClosed();
}
public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
index 955bd27..a038d0a 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
@@ -17,6 +17,7 @@
package com.android.inputmethod.latin.personalization;
import com.android.inputmethod.latin.Dictionary;
+import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import android.content.Context;
import android.content.SharedPreferences;
@@ -31,6 +32,6 @@
@Override
protected String getDictionaryFileName() {
- return NAME + "." + getLocale() + ".dict";
+ return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
}
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
index d117844..76e48c7 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
@@ -17,6 +17,7 @@
package com.android.inputmethod.latin.personalization;
import com.android.inputmethod.latin.Dictionary;
+import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import android.content.Context;
import android.content.SharedPreferences;
@@ -26,7 +27,8 @@
* cancellation or manual picks. This allows the keyboard to adapt to the typist over time.
*/
public class UserHistoryPredictionDictionary extends DynamicPredictionDictionaryBase {
- private static final String NAME = UserHistoryPredictionDictionary.class.getSimpleName();
+ /* package for tests */ static final String NAME =
+ UserHistoryPredictionDictionary.class.getSimpleName();
/* package */ UserHistoryPredictionDictionary(final Context context, final String locale,
final SharedPreferences sp) {
super(context, locale, sp, Dictionary.TYPE_USER_HISTORY);
@@ -34,6 +36,6 @@
@Override
protected String getDictionaryFileName() {
- return NAME + "." + getLocale() + ".dict";
+ return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
}
}
diff --git a/java/src/com/android/inputmethod/latin/settings/Settings.java b/java/src/com/android/inputmethod/latin/settings/Settings.java
index d432087..20b6752 100644
--- a/java/src/com/android/inputmethod/latin/settings/Settings.java
+++ b/java/src/com/android/inputmethod/latin/settings/Settings.java
@@ -27,10 +27,10 @@
import com.android.inputmethod.latin.InputAttributes;
import com.android.inputmethod.latin.R;
import com.android.inputmethod.latin.utils.AdditionalSubtypeUtils;
-import com.android.inputmethod.latin.utils.DebugLogUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
import com.android.inputmethod.latin.utils.ResourceUtils;
import com.android.inputmethod.latin.utils.RunInLocale;
+import com.android.inputmethod.latin.utils.StringUtils;
import java.util.HashMap;
import java.util.Locale;
@@ -90,6 +90,8 @@
private static final String PREF_SUPPRESS_LANGUAGE_SWITCH_KEY =
"pref_suppress_language_switch_key";
+ private static final String PREF_LAST_USED_PERSONALIZATION_TOKEN =
+ "pref_last_used_personalization_token";
public static final String PREF_SEND_FEEDBACK = "send_feedback";
public static final String PREF_ABOUT_KEYBOARD = "about_keyboard";
@@ -343,4 +345,14 @@
return prefs.getBoolean(
DebugSettings.PREF_USE_ONLY_PERSONALIZATION_DICTIONARY_FOR_DEBUG, false);
}
+
+ public void writeLastUsedPersonalizationToken(byte[] token) {
+ final String tokenStr = StringUtils.byteArrayToHexString(token);
+ mPrefs.edit().putString(PREF_LAST_USED_PERSONALIZATION_TOKEN, tokenStr).apply();
+ }
+
+ public byte[] readLastUsedPersonalizationToken() {
+ final String tokenStr = mPrefs.getString(PREF_LAST_USED_PERSONALIZATION_TOKEN, null);
+ return StringUtils.hexStringToByteArray(tokenStr);
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java
index 195f9f8..a0b744d 100644
--- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java
+++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java
@@ -57,6 +57,7 @@
public final SuggestedWords mSuggestPuncList;
public final String mWordSeparators;
public final CharSequence mHintToSaveText;
+ public final boolean mCurrentLanguageHasSpaces;
// From preferences, in the same order as xml/prefs.xml:
public final boolean mAutoCap;
@@ -118,6 +119,7 @@
mSuggestPuncList = createSuggestPuncList(suggestPuncsSpec);
mWordSeparators = res.getString(R.string.symbols_word_separators);
mHintToSaveText = res.getText(R.string.hint_add_to_dictionary);
+ mCurrentLanguageHasSpaces = res.getBoolean(R.bool.current_language_has_spaces);
// Store the input attributes
if (null == inputAttributes) {
@@ -186,6 +188,7 @@
mSuggestPuncList = createSuggestPuncList(suggestPuncsSpec);
mWordSeparators = "&\t \n()[]{}*&<>+=|.,;:!?/_\"";
mHintToSaveText = "Touch again to save";
+ mCurrentLanguageHasSpaces = true;
mInputAttributes = new InputAttributes(null, false /* isFullscreenMode */);
mAutoCap = true;
mVibrateOn = true;
diff --git a/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java b/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java
index 1bb27aa..d93b614 100644
--- a/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java
+++ b/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java
@@ -16,7 +16,7 @@
package com.android.inputmethod.latin.utils;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
/**
* This class provides an implementation for the FusionDictionary buffer interface that is backed
diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
index 0b4838c..be41840 100644
--- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
@@ -357,4 +357,37 @@
}
return true;
}
+
+ @UsedForTesting
+ public static String byteArrayToHexString(byte[] bytes) {
+ if (bytes == null || bytes.length == 0) {
+ return "";
+ }
+ final StringBuilder sb = new StringBuilder();
+ for (byte b : bytes) {
+ sb.append(String.format("%02x", b & 0xff));
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Convert hex string to byte array. The string length must be an even number.
+ */
+ @UsedForTesting
+ public static byte[] hexStringToByteArray(String hexString) {
+ if (TextUtils.isEmpty(hexString)) {
+ return null;
+ }
+ final int N = hexString.length();
+ if (N % 2 != 0) {
+ throw new NumberFormatException("Input hex string length must be an even number."
+ + " Length = " + N);
+ }
+ final byte[] bytes = new byte[N / 2];
+ for (int i = 0; i < N; i += 2) {
+ bytes[i / 2] = (byte) ((Character.digit(hexString.charAt(i), 16) << 4)
+ + Character.digit(hexString.charAt(i + 1), 16));
+ }
+ return bytes;
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
index a0ad27c..cd03b38 100644
--- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
@@ -19,12 +19,13 @@
import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
+import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.PendingAttribute;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
@@ -62,7 +63,7 @@
final FormatOptions formatOptions) {
final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams);
try {
- BinaryDictInputOutput.writeDictionaryBinary(destination, fusionDict, formatOptions);
+ BinaryDictEncoder.writeDictionaryBinary(destination, fusionDict, formatOptions);
Log.d(TAG, "end writing");
} catch (IOException e) {
Log.e(TAG, "IO exception while writing file", e);
@@ -77,7 +78,7 @@
@UsedForTesting
static FusionDictionary constructFusionDictionary(
final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams) {
- final FusionDictionary fusionDict = new FusionDictionary(new Node(),
+ final FusionDictionary fusionDict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false,
false));
int profTotal = 0;
@@ -101,7 +102,7 @@
if (word1 == null) { // unigram
fusionDict.add(word2, freq, null, false /* isNotAWord */);
} else { // bigram
- if (FusionDictionary.findWordInTree(fusionDict.mRoot, word1) == null) {
+ if (FusionDictionary.findWordInTree(fusionDict.mRootNodeArray, word1) == null) {
fusionDict.add(word1, 2, null, false /* isNotAWord */);
}
fusionDict.setBigram(word1, word2, freq);
@@ -156,7 +157,7 @@
continue;
}
to.setBigram(word1, word2,
- BinaryDictInputOutput.reconstructBigramFrequency(unigramFrequency,
+ BinaryDictDecoder.reconstructBigramFrequency(unigramFrequency,
attr.mFrequency));
}
}
diff --git a/java/src/com/android/inputmethod/research/JsonUtils.java b/java/src/com/android/inputmethod/research/JsonUtils.java
index 63d524d..977f843 100644
--- a/java/src/com/android/inputmethod/research/JsonUtils.java
+++ b/java/src/com/android/inputmethod/research/JsonUtils.java
@@ -75,12 +75,12 @@
private static void writeJson(final Key key, final JsonWriter jsonWriter) throws IOException {
jsonWriter.beginObject();
- jsonWriter.name("code").value(key.mCode);
+ jsonWriter.name("code").value(key.getCode());
jsonWriter.name("altCode").value(key.getAltCode());
- jsonWriter.name("x").value(key.mX);
- jsonWriter.name("y").value(key.mY);
- jsonWriter.name("w").value(key.mWidth);
- jsonWriter.name("h").value(key.mHeight);
+ jsonWriter.name("x").value(key.getX());
+ jsonWriter.name("y").value(key.getY());
+ jsonWriter.name("w").value(key.getWidth());
+ jsonWriter.name("h").value(key.getHeight());
jsonWriter.endObject();
}
diff --git a/native/jni/Android.mk b/native/jni/Android.mk
index 8d51a2f..9e7407a 100644
--- a/native/jni/Android.mk
+++ b/native/jni/Android.mk
@@ -43,7 +43,7 @@
com_android_inputmethod_keyboard_ProximityInfo.cpp \
com_android_inputmethod_latin_BinaryDictionary.cpp \
com_android_inputmethod_latin_DicTraverseSession.cpp \
- com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp \
+ com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp \
jni_common.cpp
LATIN_IME_CORE_SRC_FILES := \
diff --git a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp
similarity index 75%
rename from native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp
rename to native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp
index f78883c..457b226 100644
--- a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp
+++ b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp
@@ -14,16 +14,16 @@
* limitations under the License.
*/
-#define LOG_TAG "LatinIME: jni: BinaryDictInputOutput"
+#define LOG_TAG "LatinIME: jni: BinaryDictDecoder"
-#include "com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h"
+#include "com_android_inputmethod_latin_makedict_BinaryDictDecoder.h"
#include "defines.h"
#include "jni.h"
#include "jni_common.h"
namespace latinime {
-static int latinime_BinaryDictInputOutput_doNothing(JNIEnv *env, jclass clazz) {
+static int latinime_BinaryDictDecoder_doNothing(JNIEnv *env, jclass clazz) {
// This is a phony method for test - it does nothing. It just returns some value
// unlikely to be in memory by chance for testing purposes.
// TODO: remove this method.
@@ -35,13 +35,13 @@
// TODO: remove this entry when we have one useful method in here
const_cast<char *>("doNothing"),
const_cast<char *>("()I"),
- reinterpret_cast<void *>(latinime_BinaryDictInputOutput_doNothing)
+ reinterpret_cast<void *>(latinime_BinaryDictDecoder_doNothing)
},
};
-int register_BinaryDictInputOutput(JNIEnv *env) {
+int register_BinaryDictDecoder(JNIEnv *env) {
const char *const kClassPathName =
- "com/android/inputmethod/latin/makedict/BinaryDictInputOutput";
+ "com/android/inputmethod/latin/makedict/BinaryDictDecoder";
return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods));
}
} // namespace latinime
diff --git a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.h
similarity index 71%
rename from native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h
rename to native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.h
index e622ed4..7f3cb67 100644
--- a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h
+++ b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.h
@@ -14,12 +14,12 @@
* limitations under the License.
*/
-#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H
-#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H
+#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H
+#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H
#include "jni.h"
namespace latinime {
-int register_BinaryDictInputOutput(JNIEnv *env);
+int register_BinaryDictDecoder(JNIEnv *env);
} // namespace latinime
-#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H
+#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H
diff --git a/native/jni/jni_common.cpp b/native/jni/jni_common.cpp
index 733e15f..d44be67 100644
--- a/native/jni/jni_common.cpp
+++ b/native/jni/jni_common.cpp
@@ -23,7 +23,7 @@
#include "com_android_inputmethod_latin_BinaryDictionary.h"
#include "com_android_inputmethod_latin_DicTraverseSession.h"
#endif
-#include "com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h"
+#include "com_android_inputmethod_latin_makedict_BinaryDictDecoder.h"
#include "defines.h"
/*
@@ -55,8 +55,8 @@
return -1;
}
#endif
- if (!latinime::register_BinaryDictInputOutput(env)) {
- AKLOGE("ERROR: BinaryDictInputOutput native registration failed");
+ if (!latinime::register_BinaryDictDecoder(env)) {
+ AKLOGE("ERROR: BinaryDictDecoder native registration failed");
return -1;
}
/* success -- return valid version number */
diff --git a/native/jni/src/suggest/core/dictionary/probability_utils.h b/native/jni/src/suggest/core/dictionary/probability_utils.h
index f450087..2192135 100644
--- a/native/jni/src/suggest/core/dictionary/probability_utils.h
+++ b/native/jni/src/suggest/core/dictionary/probability_utils.h
@@ -41,7 +41,7 @@
// the unigram probability to be the median value of the 17th step from the top. A value of
// 0 for the bigram probability represents the middle of the 16th step from the top,
// while a value of 15 represents the middle of the top step.
- // See makedict.BinaryDictInputOutput for details.
+ // See makedict.BinaryDictDecoder for details.
const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability)
/ (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY);
return unigramProbability
diff --git a/native/jni/src/suggest/policyimpl/dictionary/binary_format.h b/native/jni/src/suggest/policyimpl/dictionary/binary_format.h
deleted file mode 100644
index 23f4c7f..0000000
--- a/native/jni/src/suggest/policyimpl/dictionary/binary_format.h
+++ /dev/null
@@ -1,470 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_BINARY_FORMAT_H
-#define LATINIME_BINARY_FORMAT_H
-
-#include <stdint.h>
-
-#include "suggest/core/dictionary/probability_utils.h"
-#include "utils/char_utils.h"
-
-namespace latinime {
-
-class BinaryFormat {
- public:
- // Mask and flags for children address type selection.
- static const int MASK_GROUP_ADDRESS_TYPE = 0xC0;
-
- // Flag for single/multiple char group
- static const int FLAG_HAS_MULTIPLE_CHARS = 0x20;
-
- // Flag for terminal groups
- static const int FLAG_IS_TERMINAL = 0x10;
-
- // Flag for shortcut targets presence
- static const int FLAG_HAS_SHORTCUT_TARGETS = 0x08;
- // Flag for bigram presence
- static const int FLAG_HAS_BIGRAMS = 0x04;
- // Flag for non-words (typically, shortcut only entries)
- static const int FLAG_IS_NOT_A_WORD = 0x02;
- // Flag for blacklist
- static const int FLAG_IS_BLACKLISTED = 0x01;
-
- // Attribute (bigram/shortcut) related flags:
- // Flag for presence of more attributes
- static const int FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
- // Flag for sign of offset. If this flag is set, the offset value must be negated.
- static const int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
-
- // Mask for attribute probability, stored on 4 bits inside the flags byte.
- static const int MASK_ATTRIBUTE_PROBABILITY = 0x0F;
-
- // Mask and flags for attribute address type selection.
- static const int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30;
-
- static int getGroupCountAndForwardPointer(const uint8_t *const dict, int *pos);
- static uint8_t getFlagsAndForwardPointer(const uint8_t *const dict, int *pos);
- static int getCodePointAndForwardPointer(const uint8_t *const dict, int *pos);
- static int readProbabilityWithoutMovingPointer(const uint8_t *const dict, const int pos);
- static int skipOtherCharacters(const uint8_t *const dict, const int pos);
- static int skipChildrenPosition(const uint8_t flags, const int pos);
- static int skipProbability(const uint8_t flags, const int pos);
- static int skipShortcuts(const uint8_t *const dict, const uint8_t flags, const int pos);
- static int skipChildrenPosAndAttributes(const uint8_t *const dict, const uint8_t flags,
- const int pos);
- static int readChildrenPosition(const uint8_t *const dict, const uint8_t flags, const int pos);
- static bool hasChildrenInFlags(const uint8_t flags);
- static int getTerminalPosition(const uint8_t *const root, const int *const inWord,
- const int length, const bool forceLowerCaseSearch);
- static int getCodePointsAndProbabilityAndReturnCodePointCount(
- const uint8_t *const root, const int nodePos, const int maxCodePointCount,
- int *const outCodePoints, int *const outUnigramProbability);
-
- private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryFormat);
-
- static const int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00;
- static const int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40;
- static const int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80;
- static const int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0;
- static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
- static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
- static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
-
- static const int CHARACTER_ARRAY_TERMINATOR_SIZE = 1;
- static const int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
- static const int CHARACTER_ARRAY_TERMINATOR = 0x1F;
- static const int MULTIPLE_BYTE_CHARACTER_ADDITIONAL_SIZE = 2;
- static const int NO_FLAGS = 0;
- static int skipAllAttributes(const uint8_t *const dict, const uint8_t flags, const int pos);
- static int skipBigrams(const uint8_t *const dict, const uint8_t flags, const int pos);
-};
-
-AK_FORCE_INLINE int BinaryFormat::getGroupCountAndForwardPointer(const uint8_t *const dict,
- int *pos) {
- const int msb = dict[(*pos)++];
- if (msb < 0x80) return msb;
- return ((msb & 0x7F) << 8) | dict[(*pos)++];
-}
-
-inline uint8_t BinaryFormat::getFlagsAndForwardPointer(const uint8_t *const dict, int *pos) {
- return dict[(*pos)++];
-}
-
-AK_FORCE_INLINE int BinaryFormat::getCodePointAndForwardPointer(const uint8_t *const dict,
- int *pos) {
- const int origin = *pos;
- const int codePoint = dict[origin];
- if (codePoint < MINIMAL_ONE_BYTE_CHARACTER_VALUE) {
- if (codePoint == CHARACTER_ARRAY_TERMINATOR) {
- *pos = origin + 1;
- return NOT_A_CODE_POINT;
- } else {
- *pos = origin + 3;
- const int char_1 = codePoint << 16;
- const int char_2 = char_1 + (dict[origin + 1] << 8);
- return char_2 + dict[origin + 2];
- }
- } else {
- *pos = origin + 1;
- return codePoint;
- }
-}
-
-inline int BinaryFormat::readProbabilityWithoutMovingPointer(const uint8_t *const dict,
- const int pos) {
- return dict[pos];
-}
-
-AK_FORCE_INLINE int BinaryFormat::skipOtherCharacters(const uint8_t *const dict, const int pos) {
- int currentPos = pos;
- int character = dict[currentPos++];
- while (CHARACTER_ARRAY_TERMINATOR != character) {
- if (character < MINIMAL_ONE_BYTE_CHARACTER_VALUE) {
- currentPos += MULTIPLE_BYTE_CHARACTER_ADDITIONAL_SIZE;
- }
- character = dict[currentPos++];
- }
- return currentPos;
-}
-
-static inline int attributeAddressSize(const uint8_t flags) {
- static const int ATTRIBUTE_ADDRESS_SHIFT = 4;
- return (flags & BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT;
- /* Note: this is a value-dependant optimization of what may probably be
- more readably written this way:
- switch (flags * BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) {
- case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1;
- case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2;
- case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3;
- default: return 0;
- }
- */
-}
-
-static AK_FORCE_INLINE int skipExistingBigrams(const uint8_t *const dict, const int pos) {
- int currentPos = pos;
- uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(dict, ¤tPos);
- while (flags & BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT) {
- currentPos += attributeAddressSize(flags);
- flags = BinaryFormat::getFlagsAndForwardPointer(dict, ¤tPos);
- }
- currentPos += attributeAddressSize(flags);
- return currentPos;
-}
-
-static inline int childrenAddressSize(const uint8_t flags) {
- static const int CHILDREN_ADDRESS_SHIFT = 6;
- return (BinaryFormat::MASK_GROUP_ADDRESS_TYPE & flags) >> CHILDREN_ADDRESS_SHIFT;
- /* See the note in attributeAddressSize. The same applies here */
-}
-
-static AK_FORCE_INLINE int shortcutByteSize(const uint8_t *const dict, const int pos) {
- return (static_cast<int>(dict[pos] << 8)) + (dict[pos + 1]);
-}
-
-inline int BinaryFormat::skipChildrenPosition(const uint8_t flags, const int pos) {
- return pos + childrenAddressSize(flags);
-}
-
-inline int BinaryFormat::skipProbability(const uint8_t flags, const int pos) {
- return FLAG_IS_TERMINAL & flags ? pos + 1 : pos;
-}
-
-AK_FORCE_INLINE int BinaryFormat::skipShortcuts(const uint8_t *const dict, const uint8_t flags,
- const int pos) {
- if (FLAG_HAS_SHORTCUT_TARGETS & flags) {
- return pos + shortcutByteSize(dict, pos);
- } else {
- return pos;
- }
-}
-
-AK_FORCE_INLINE int BinaryFormat::skipBigrams(const uint8_t *const dict, const uint8_t flags,
- const int pos) {
- if (FLAG_HAS_BIGRAMS & flags) {
- return skipExistingBigrams(dict, pos);
- } else {
- return pos;
- }
-}
-
-AK_FORCE_INLINE int BinaryFormat::skipAllAttributes(const uint8_t *const dict, const uint8_t flags,
- const int pos) {
- // This function skips all attributes: shortcuts and bigrams.
- int newPos = pos;
- newPos = skipShortcuts(dict, flags, newPos);
- newPos = skipBigrams(dict, flags, newPos);
- return newPos;
-}
-
-AK_FORCE_INLINE int BinaryFormat::skipChildrenPosAndAttributes(const uint8_t *const dict,
- const uint8_t flags, const int pos) {
- int currentPos = pos;
- currentPos = skipChildrenPosition(flags, currentPos);
- currentPos = skipAllAttributes(dict, flags, currentPos);
- return currentPos;
-}
-
-AK_FORCE_INLINE int BinaryFormat::readChildrenPosition(const uint8_t *const dict,
- const uint8_t flags, const int pos) {
- int offset = 0;
- switch (MASK_GROUP_ADDRESS_TYPE & flags) {
- case FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
- offset = dict[pos];
- break;
- case FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
- offset = dict[pos] << 8;
- offset += dict[pos + 1];
- break;
- case FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
- offset = dict[pos] << 16;
- offset += dict[pos + 1] << 8;
- offset += dict[pos + 2];
- break;
- default:
- // If we come here, it means we asked for the children of a word with
- // no children.
- return -1;
- }
- return pos + offset;
-}
-
-inline bool BinaryFormat::hasChildrenInFlags(const uint8_t flags) {
- return (FLAG_GROUP_ADDRESS_TYPE_NOADDRESS != (MASK_GROUP_ADDRESS_TYPE & flags));
-}
-
-// This function gets the byte position of the last chargroup of the exact matching word in the
-// dictionary. If no match is found, it returns NOT_A_VALID_WORD_POS.
-AK_FORCE_INLINE int BinaryFormat::getTerminalPosition(const uint8_t *const root,
- const int *const inWord, const int length, const bool forceLowerCaseSearch) {
- int pos = 0;
- int wordPos = 0;
-
- while (true) {
- // If we already traversed the tree further than the word is long, there means
- // there was no match (or we would have found it).
- if (wordPos >= length) return NOT_A_VALID_WORD_POS;
- int charGroupCount = BinaryFormat::getGroupCountAndForwardPointer(root, &pos);
- const int wChar = forceLowerCaseSearch
- ? CharUtils::toLowerCase(inWord[wordPos]) : inWord[wordPos];
- while (true) {
- // If there are no more character groups in this node, it means we could not
- // find a matching character for this depth, therefore there is no match.
- if (0 >= charGroupCount) return NOT_A_VALID_WORD_POS;
- const int charGroupPos = pos;
- const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
- int character = BinaryFormat::getCodePointAndForwardPointer(root, &pos);
- if (character == wChar) {
- // This is the correct node. Only one character group may start with the same
- // char within a node, so either we found our match in this node, or there is
- // no match and we can return NOT_A_VALID_WORD_POS. So we will check all the
- // characters in this character group indeed does match.
- if (FLAG_HAS_MULTIPLE_CHARS & flags) {
- character = BinaryFormat::getCodePointAndForwardPointer(root, &pos);
- while (NOT_A_CODE_POINT != character) {
- ++wordPos;
- // If we shoot the length of the word we search for, or if we find a single
- // character that does not match, as explained above, it means the word is
- // not in the dictionary (by virtue of this chargroup being the only one to
- // match the word on the first character, but not matching the whole word).
- if (wordPos >= length) return NOT_A_VALID_WORD_POS;
- if (inWord[wordPos] != character) return NOT_A_VALID_WORD_POS;
- character = BinaryFormat::getCodePointAndForwardPointer(root, &pos);
- }
- }
- // If we come here we know that so far, we do match. Either we are on a terminal
- // and we match the length, in which case we found it, or we traverse children.
- // If we don't match the length AND don't have children, then a word in the
- // dictionary fully matches a prefix of the searched word but not the full word.
- ++wordPos;
- if (FLAG_IS_TERMINAL & flags) {
- if (wordPos == length) {
- return charGroupPos;
- }
- pos = BinaryFormat::skipProbability(FLAG_IS_TERMINAL, pos);
- }
- if (FLAG_GROUP_ADDRESS_TYPE_NOADDRESS == (MASK_GROUP_ADDRESS_TYPE & flags)) {
- return NOT_A_VALID_WORD_POS;
- }
- // We have children and we are still shorter than the word we are searching for, so
- // we need to traverse children. Put the pointer on the children position, and
- // break
- pos = BinaryFormat::readChildrenPosition(root, flags, pos);
- break;
- } else {
- // This chargroup does not match, so skip the remaining part and go to the next.
- if (FLAG_HAS_MULTIPLE_CHARS & flags) {
- pos = BinaryFormat::skipOtherCharacters(root, pos);
- }
- pos = BinaryFormat::skipProbability(flags, pos);
- pos = BinaryFormat::skipChildrenPosAndAttributes(root, flags, pos);
- }
- --charGroupCount;
- }
- }
-}
-
-// This function searches for a terminal in the dictionary by its address.
-// Due to the fact that words are ordered in the dictionary in a strict breadth-first order,
-// it is possible to check for this with advantageous complexity. For each node, we search
-// for groups with children and compare the children address with the address we look for.
-// When we shoot the address we look for, it means the word we look for is in the children
-// of the previous group. The only tricky part is the fact that if we arrive at the end of a
-// node with the last group's children address still less than what we are searching for, we
-// must descend the last group's children (for example, if the word we are searching for starts
-// with a z, it's the last group of the root node, so all children addresses will be smaller
-// than the address we look for, and we have to descend the z node).
-/* Parameters :
- * root: the dictionary buffer
- * address: the byte position of the last chargroup of the word we are searching for (this is
- * what is stored as the "bigram address" in each bigram)
- * outword: an array to write the found word, with MAX_WORD_LENGTH size.
- * outUnigramProbability: a pointer to an int to write the probability into.
- * Return value : the length of the word, of 0 if the word was not found.
- */
-AK_FORCE_INLINE int BinaryFormat::getCodePointsAndProbabilityAndReturnCodePointCount(
- const uint8_t *const root, const int nodePos, const int maxCodePointCount,
- int *const outCodePoints, int *const outUnigramProbability) {
- int pos = 0;
- int wordPos = 0;
-
- // One iteration of the outer loop iterates through nodes. As stated above, we will only
- // traverse nodes that are actually a part of the terminal we are searching, so each time
- // we enter this loop we are one depth level further than last time.
- // The only reason we count nodes is because we want to reduce the probability of infinite
- // looping in case there is a bug. Since we know there is an upper bound to the depth we are
- // supposed to traverse, it does not hurt to count iterations.
- for (int loopCount = maxCodePointCount; loopCount > 0; --loopCount) {
- int lastCandidateGroupPos = 0;
- // Let's loop through char groups in this node searching for either the terminal
- // or one of its ascendants.
- for (int charGroupCount = getGroupCountAndForwardPointer(root, &pos); charGroupCount > 0;
- --charGroupCount) {
- const int startPos = pos;
- const uint8_t flags = getFlagsAndForwardPointer(root, &pos);
- const int character = getCodePointAndForwardPointer(root, &pos);
- if (nodePos == startPos) {
- // We found the address. Copy the rest of the word in the buffer and return
- // the length.
- outCodePoints[wordPos] = character;
- if (FLAG_HAS_MULTIPLE_CHARS & flags) {
- int nextChar = getCodePointAndForwardPointer(root, &pos);
- // We count chars in order to avoid infinite loops if the file is broken or
- // if there is some other bug
- int charCount = maxCodePointCount;
- while (NOT_A_CODE_POINT != nextChar && --charCount > 0) {
- outCodePoints[++wordPos] = nextChar;
- nextChar = getCodePointAndForwardPointer(root, &pos);
- }
- }
- *outUnigramProbability = readProbabilityWithoutMovingPointer(root, pos);
- return ++wordPos;
- }
- // We need to skip past this char group, so skip any remaining chars after the
- // first and possibly the probability.
- if (FLAG_HAS_MULTIPLE_CHARS & flags) {
- pos = skipOtherCharacters(root, pos);
- }
- pos = skipProbability(flags, pos);
-
- // The fact that this group has children is very important. Since we already know
- // that this group does not match, if it has no children we know it is irrelevant
- // to what we are searching for.
- const bool hasChildren = (FLAG_GROUP_ADDRESS_TYPE_NOADDRESS !=
- (MASK_GROUP_ADDRESS_TYPE & flags));
- // We will write in `found' whether we have passed the children address we are
- // searching for. For example if we search for "beer", the children of b are less
- // than the address we are searching for and the children of c are greater. When we
- // come here for c, we realize this is too big, and that we should descend b.
- bool found;
- if (hasChildren) {
- // Here comes the tricky part. First, read the children position.
- const int childrenPos = readChildrenPosition(root, flags, pos);
- if (childrenPos > nodePos) {
- // If the children pos is greater than address, it means the previous chargroup,
- // which address is stored in lastCandidateGroupPos, was the right one.
- found = true;
- } else if (1 >= charGroupCount) {
- // However if we are on the LAST group of this node, and we have NOT shot the
- // address we should descend THIS node. So we trick the lastCandidateGroupPos
- // so that we will descend this node, not the previous one.
- lastCandidateGroupPos = startPos;
- found = true;
- } else {
- // Else, we should continue looking.
- found = false;
- }
- } else {
- // Even if we don't have children here, we could still be on the last group of this
- // node. If this is the case, we should descend the last group that had children,
- // and their address is already in lastCandidateGroup.
- found = (1 >= charGroupCount);
- }
-
- if (found) {
- // Okay, we found the group we should descend. Its address is in
- // the lastCandidateGroupPos variable, so we just re-read it.
- if (0 != lastCandidateGroupPos) {
- const uint8_t lastFlags =
- getFlagsAndForwardPointer(root, &lastCandidateGroupPos);
- const int lastChar =
- getCodePointAndForwardPointer(root, &lastCandidateGroupPos);
- // We copy all the characters in this group to the buffer
- outCodePoints[wordPos] = lastChar;
- if (FLAG_HAS_MULTIPLE_CHARS & lastFlags) {
- int nextChar = getCodePointAndForwardPointer(root, &lastCandidateGroupPos);
- int charCount = maxCodePointCount;
- while (-1 != nextChar && --charCount > 0) {
- outCodePoints[++wordPos] = nextChar;
- nextChar = getCodePointAndForwardPointer(root, &lastCandidateGroupPos);
- }
- }
- ++wordPos;
- // Now we only need to branch to the children address. Skip the probability if
- // it's there, read pos, and break to resume the search at pos.
- lastCandidateGroupPos = skipProbability(lastFlags, lastCandidateGroupPos);
- pos = readChildrenPosition(root, lastFlags, lastCandidateGroupPos);
- break;
- } else {
- // Here is a little tricky part: we come here if we found out that all children
- // addresses in this group are bigger than the address we are searching for.
- // Should we conclude the word is not in the dictionary? No! It could still be
- // one of the remaining chargroups in this node, so we have to keep looking in
- // this node until we find it (or we realize it's not there either, in which
- // case it's actually not in the dictionary). Pass the end of this group, ready
- // to start the next one.
- pos = skipChildrenPosAndAttributes(root, flags, pos);
- }
- } else {
- // If we did not find it, we should record the last children address for the next
- // iteration.
- if (hasChildren) lastCandidateGroupPos = startPos;
- // Now skip the end of this group (children pos and the attributes if any) so that
- // our pos is after the end of this char group, at the start of the next one.
- pos = skipChildrenPosAndAttributes(root, flags, pos);
- }
-
- }
- }
- // If we have looked through all the chargroups and found no match, the address is
- // not the address of a terminal in this dictionary.
- return 0;
-}
-
-} // namespace latinime
-#endif // LATINIME_BINARY_FORMAT_H
diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp
index 3e664a2..15eb067 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp
@@ -20,7 +20,6 @@
#include "defines.h"
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_vector.h"
-#include "suggest/policyimpl/dictionary/binary_format.h"
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
namespace latinime {
@@ -38,17 +37,273 @@
}
}
+// This retrieves code points and the probability of the word by its terminal position.
+// Due to the fact that words are ordered in the dictionary in a strict breadth-first order,
+// it is possible to check for this with advantageous complexity. For each node, we search
+// for groups with children and compare the children position with the position we look for.
+// When we shoot the position we look for, it means the word we look for is in the children
+// of the previous group. The only tricky part is the fact that if we arrive at the end of a
+// node with the last group's children position still less than what we are searching for, we
+// must descend the last group's children (for example, if the word we are searching for starts
+// with a z, it's the last group of the root node, so all children addresses will be smaller
+// than the position we look for, and we have to descend the z node).
+/* Parameters :
+ * nodePos: the byte position of the terminal chargroup of the word we are searching for (this is
+ * what is stored as the "bigram position" in each bigram)
+ * outCodePoints: an array to write the found word, with MAX_WORD_LENGTH size.
+ * outUnigramProbability: a pointer to an int to write the probability into.
+ * Return value : the code point count, of 0 if the word was not found.
+ */
+// TODO: Split this function to be more readable
int PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
const int nodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const {
- return BinaryFormat::getCodePointsAndProbabilityAndReturnCodePointCount(mDictRoot, nodePos,
- maxCodePointCount, outCodePoints, outUnigramProbability);
+ int pos = getRootPosition();
+ int wordPos = 0;
+ // One iteration of the outer loop iterates through nodes. As stated above, we will only
+ // traverse nodes that are actually a part of the terminal we are searching, so each time
+ // we enter this loop we are one depth level further than last time.
+ // The only reason we count nodes is because we want to reduce the probability of infinite
+ // looping in case there is a bug. Since we know there is an upper bound to the depth we are
+ // supposed to traverse, it does not hurt to count iterations.
+ for (int loopCount = maxCodePointCount; loopCount > 0; --loopCount) {
+ int lastCandidateGroupPos = 0;
+ // Let's loop through char groups in this node searching for either the terminal
+ // or one of its ascendants.
+ for (int charGroupCount = PatriciaTrieReadingUtils::getGroupCountAndAdvancePosition(
+ mDictRoot, &pos); charGroupCount > 0; --charGroupCount) {
+ const int startPos = pos;
+ const PatriciaTrieReadingUtils::NodeFlags flags =
+ PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(mDictRoot, &pos);
+ const int character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+ mDictRoot, &pos);
+ if (nodePos == startPos) {
+ // We found the position. Copy the rest of the code points in the buffer and return
+ // the length.
+ outCodePoints[wordPos] = character;
+ if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) {
+ int nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+ mDictRoot, &pos);
+ // We count code points in order to avoid infinite loops if the file is broken
+ // or if there is some other bug
+ int charCount = maxCodePointCount;
+ while (NOT_A_CODE_POINT != nextChar && --charCount > 0) {
+ outCodePoints[++wordPos] = nextChar;
+ nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+ mDictRoot, &pos);
+ }
+ }
+ *outUnigramProbability =
+ PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot,
+ &pos);
+ return ++wordPos;
+ }
+ // We need to skip past this char group, so skip any remaining code points after the
+ // first and possibly the probability.
+ if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) {
+ PatriciaTrieReadingUtils::skipCharacters(mDictRoot, flags, MAX_WORD_LENGTH, &pos);
+ }
+ if (PatriciaTrieReadingUtils::isTerminal(flags)) {
+ PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos);
+ }
+ // The fact that this group has children is very important. Since we already know
+ // that this group does not match, if it has no children we know it is irrelevant
+ // to what we are searching for.
+ const bool hasChildren = PatriciaTrieReadingUtils::hasChildrenInFlags(flags);
+ // We will write in `found' whether we have passed the children position we are
+ // searching for. For example if we search for "beer", the children of b are less
+ // than the address we are searching for and the children of c are greater. When we
+ // come here for c, we realize this is too big, and that we should descend b.
+ bool found;
+ if (hasChildren) {
+ int currentPos = pos;
+ // Here comes the tricky part. First, read the children position.
+ const int childrenPos = PatriciaTrieReadingUtils
+ ::readChildrenPositionAndAdvancePosition(mDictRoot, flags, ¤tPos);
+ if (childrenPos > nodePos) {
+ // If the children pos is greater than the position, it means the previous
+ // chargroup, which position is stored in lastCandidateGroupPos, was the right
+ // one.
+ found = true;
+ } else if (1 >= charGroupCount) {
+ // However if we are on the LAST group of this node, and we have NOT shot the
+ // position we should descend THIS node. So we trick the lastCandidateGroupPos
+ // so that we will descend this node, not the previous one.
+ lastCandidateGroupPos = startPos;
+ found = true;
+ } else {
+ // Else, we should continue looking.
+ found = false;
+ }
+ } else {
+ // Even if we don't have children here, we could still be on the last group of this
+ // node. If this is the case, we should descend the last group that had children,
+ // and their position is already in lastCandidateGroup.
+ found = (1 >= charGroupCount);
+ }
+
+ if (found) {
+ // Okay, we found the group we should descend. Its position is in
+ // the lastCandidateGroupPos variable, so we just re-read it.
+ if (0 != lastCandidateGroupPos) {
+ const PatriciaTrieReadingUtils::NodeFlags lastFlags =
+ PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(
+ mDictRoot, &lastCandidateGroupPos);
+ const int lastChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+ mDictRoot, &lastCandidateGroupPos);
+ // We copy all the characters in this group to the buffer
+ outCodePoints[wordPos] = lastChar;
+ if (PatriciaTrieReadingUtils::hasMultipleChars(lastFlags)) {
+ int nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+ mDictRoot, &lastCandidateGroupPos);
+ int charCount = maxCodePointCount;
+ while (-1 != nextChar && --charCount > 0) {
+ outCodePoints[++wordPos] = nextChar;
+ nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+ mDictRoot, &lastCandidateGroupPos);
+ }
+ }
+ ++wordPos;
+ // Now we only need to branch to the children address. Skip the probability if
+ // it's there, read pos, and break to resume the search at pos.
+ if (PatriciaTrieReadingUtils::isTerminal(lastFlags)) {
+ PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot,
+ &lastCandidateGroupPos);
+ }
+ pos = PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
+ mDictRoot, lastFlags, &lastCandidateGroupPos);
+ break;
+ } else {
+ // Here is a little tricky part: we come here if we found out that all children
+ // addresses in this group are bigger than the address we are searching for.
+ // Should we conclude the word is not in the dictionary? No! It could still be
+ // one of the remaining chargroups in this node, so we have to keep looking in
+ // this node until we find it (or we realize it's not there either, in which
+ // case it's actually not in the dictionary). Pass the end of this group, ready
+ // to start the next one.
+ if (PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) {
+ PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
+ mDictRoot, flags, &pos);
+ }
+ if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) {
+ mShortcutListPolicy.skipAllShortcuts(&pos);
+ }
+ if (PatriciaTrieReadingUtils::hasBigrams(flags)) {
+ mBigramListPolicy.skipAllBigrams(&pos);
+ }
+ }
+ } else {
+ // If we did not find it, we should record the last children address for the next
+ // iteration.
+ if (hasChildren) lastCandidateGroupPos = startPos;
+ // Now skip the end of this group (children pos and the attributes if any) so that
+ // our pos is after the end of this char group, at the start of the next one.
+ if (PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) {
+ PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
+ mDictRoot, flags, &pos);
+ }
+ if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) {
+ mShortcutListPolicy.skipAllShortcuts(&pos);
+ }
+ if (PatriciaTrieReadingUtils::hasBigrams(flags)) {
+ mBigramListPolicy.skipAllBigrams(&pos);
+ }
+ }
+
+ }
+ }
+ // If we have looked through all the chargroups and found no match, the nodePos is
+ // not the position of a terminal in this dictionary.
+ return 0;
}
+// This function gets the position of the terminal node of the exact matching word in the
+// dictionary. If no match is found, it returns NOT_A_VALID_WORD_POS.
int PatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const inWord,
const int length, const bool forceLowerCaseSearch) const {
- return BinaryFormat::getTerminalPosition(mDictRoot, inWord,
- length, forceLowerCaseSearch);
+ int pos = getRootPosition();
+ int wordPos = 0;
+
+ while (true) {
+ // If we already traversed the tree further than the word is long, there means
+ // there was no match (or we would have found it).
+ if (wordPos >= length) return NOT_A_VALID_WORD_POS;
+ int charGroupCount = PatriciaTrieReadingUtils::getGroupCountAndAdvancePosition(mDictRoot,
+ &pos);
+ const int wChar = forceLowerCaseSearch
+ ? CharUtils::toLowerCase(inWord[wordPos]) : inWord[wordPos];
+ while (true) {
+ // If there are no more character groups in this node, it means we could not
+ // find a matching character for this depth, therefore there is no match.
+ if (0 >= charGroupCount) return NOT_A_VALID_WORD_POS;
+ const int charGroupPos = pos;
+ const PatriciaTrieReadingUtils::NodeFlags flags =
+ PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(mDictRoot, &pos);
+ int character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(mDictRoot,
+ &pos);
+ if (character == wChar) {
+ // This is the correct node. Only one character group may start with the same
+ // char within a node, so either we found our match in this node, or there is
+ // no match and we can return NOT_A_VALID_WORD_POS. So we will check all the
+ // characters in this character group indeed does match.
+ if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) {
+ character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(mDictRoot,
+ &pos);
+ while (NOT_A_CODE_POINT != character) {
+ ++wordPos;
+ // If we shoot the length of the word we search for, or if we find a single
+ // character that does not match, as explained above, it means the word is
+ // not in the dictionary (by virtue of this chargroup being the only one to
+ // match the word on the first character, but not matching the whole word).
+ if (wordPos >= length) return NOT_A_VALID_WORD_POS;
+ if (inWord[wordPos] != character) return NOT_A_VALID_WORD_POS;
+ character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+ mDictRoot, &pos);
+ }
+ }
+ // If we come here we know that so far, we do match. Either we are on a terminal
+ // and we match the length, in which case we found it, or we traverse children.
+ // If we don't match the length AND don't have children, then a word in the
+ // dictionary fully matches a prefix of the searched word but not the full word.
+ ++wordPos;
+ if (PatriciaTrieReadingUtils::isTerminal(flags)) {
+ if (wordPos == length) {
+ return charGroupPos;
+ }
+ PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos);
+ }
+ if (!PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) {
+ return NOT_A_VALID_WORD_POS;
+ }
+ // We have children and we are still shorter than the word we are searching for, so
+ // we need to traverse children. Put the pointer on the children position, and
+ // break
+ pos = PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(mDictRoot,
+ flags, &pos);
+ break;
+ } else {
+ // This chargroup does not match, so skip the remaining part and go to the next.
+ if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) {
+ PatriciaTrieReadingUtils::skipCharacters(mDictRoot, flags, MAX_WORD_LENGTH,
+ &pos);
+ }
+ if (PatriciaTrieReadingUtils::isTerminal(flags)) {
+ PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos);
+ }
+ if (PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) {
+ PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(mDictRoot,
+ flags, &pos);
+ }
+ if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) {
+ mShortcutListPolicy.skipAllShortcuts(&pos);
+ }
+ if (PatriciaTrieReadingUtils::hasBigrams(flags)) {
+ mBigramListPolicy.skipAllBigrams(&pos);
+ }
+ }
+ --charGroupCount;
+ }
+ }
}
int PatriciaTriePolicy::getUnigramProbability(const int nodePos) const {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h
index e3aabb0..08add03 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h
@@ -48,13 +48,13 @@
close(fd);
return 0;
}
- uint8_t *const buffer = static_cast<uint8_t *>(mmapedBuffer) + bufOffset;
+ uint8_t *const buffer = static_cast<uint8_t *>(mmapedBuffer) + offset;
if (!buffer) {
AKLOGE("DICT: buffer is null");
close(fd);
return 0;
}
- return new MmapedBuffer(buffer, adjSize, fd, adjOffset, isUpdatable);
+ return new MmapedBuffer(buffer, adjSize, fd, offset, isUpdatable);
}
~MmapedBuffer() {
diff --git a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java
index 65dfd2d..cadd0f8 100644
--- a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java
@@ -20,7 +20,7 @@
import android.test.suitebuilder.annotation.SmallTest;
import com.android.inputmethod.latin.makedict.FusionDictionary;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import java.util.HashMap;
@@ -30,21 +30,21 @@
@SmallTest
public class FusionDictionaryTests extends AndroidTestCase {
public void testFindWordInTree() {
- FusionDictionary dict = new FusionDictionary(new Node(),
+ FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false));
dict.add("abc", 10, null, false /* isNotAWord */);
- assertNull(FusionDictionary.findWordInTree(dict.mRoot, "aaa"));
- assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "abc"));
+ assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa"));
+ assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "abc"));
dict.add("aa", 10, null, false /* isNotAWord */);
- assertNull(FusionDictionary.findWordInTree(dict.mRoot, "aaa"));
- assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "aa"));
+ assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa"));
+ assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aa"));
dict.add("babcd", 10, null, false /* isNotAWord */);
dict.add("bacde", 10, null, false /* isNotAWord */);
- assertNull(FusionDictionary.findWordInTree(dict.mRoot, "ba"));
- assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "babcd"));
- assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "bacde"));
+ assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "ba"));
+ assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "babcd"));
+ assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "bacde"));
}
}
diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTests.java b/tests/src/com/android/inputmethod/latin/InputLogicTests.java
index d27a7a9..6cc4bef 100644
--- a/tests/src/com/android/inputmethod/latin/InputLogicTests.java
+++ b/tests/src/com/android/inputmethod/latin/InputLogicTests.java
@@ -17,6 +17,7 @@
package com.android.inputmethod.latin;
import android.test.suitebuilder.annotation.LargeTest;
+import android.view.inputmethod.BaseInputConnection;
@LargeTest
public class InputLogicTests extends InputTestsBase {
@@ -290,5 +291,19 @@
}
assertEquals("delete whole composing word", "", mEditText.getText().toString());
}
+
+ public void testResumeSuggestionOnBackspace() {
+ final String WORD_TO_TYPE = "and this ";
+ type(WORD_TO_TYPE);
+ assertEquals("resume suggestion on backspace", -1,
+ BaseInputConnection.getComposingSpanStart(mEditText.getText()));
+ assertEquals("resume suggestion on backspace", -1,
+ BaseInputConnection.getComposingSpanEnd(mEditText.getText()));
+ type(Constants.CODE_DELETE);
+ assertEquals("resume suggestion on backspace", 4,
+ BaseInputConnection.getComposingSpanStart(mEditText.getText()));
+ assertEquals("resume suggestion on backspace", 8,
+ BaseInputConnection.getComposingSpanEnd(mEditText.getText()));
+ }
// TODO: Add some tests for non-BMP characters
}
diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java b/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java
new file mode 100644
index 0000000..0f0ebaf
--- /dev/null
+++ b/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin;
+
+import android.test.suitebuilder.annotation.LargeTest;
+import android.view.inputmethod.BaseInputConnection;
+
+import com.android.inputmethod.latin.suggestions.SuggestionStripView;
+
+@LargeTest
+public class InputLogicTestsLanguageWithoutSpaces extends InputTestsBase {
+ public void testAutoCorrectForLanguageWithoutSpaces() {
+ final String STRING_TO_TYPE = "tgis is";
+ final String EXPECTED_RESULT = "thisis";
+ changeKeyboardLocaleAndDictLocale("th", "en_US");
+ type(STRING_TO_TYPE);
+ assertEquals("simple auto-correct for language without spaces", EXPECTED_RESULT,
+ mEditText.getText().toString());
+ }
+
+ public void testRevertAutoCorrectForLanguageWithoutSpaces() {
+ final String STRING_TO_TYPE = "tgis ";
+ final String EXPECTED_INTERMEDIATE_RESULT = "this";
+ final String EXPECTED_FINAL_RESULT = "tgis";
+ changeKeyboardLocaleAndDictLocale("th", "en_US");
+ type(STRING_TO_TYPE);
+ assertEquals("simple auto-correct for language without spaces",
+ EXPECTED_INTERMEDIATE_RESULT, mEditText.getText().toString());
+ type(Constants.CODE_DELETE);
+ assertEquals("simple auto-correct for language without spaces",
+ EXPECTED_FINAL_RESULT, mEditText.getText().toString());
+ // Check we are back to composing the word
+ assertEquals("don't resume suggestion on backspace", 0,
+ BaseInputConnection.getComposingSpanStart(mEditText.getText()));
+ assertEquals("don't resume suggestion on backspace", 4,
+ BaseInputConnection.getComposingSpanEnd(mEditText.getText()));
+ }
+
+ public void testDontResumeSuggestionOnBackspace() {
+ final String WORD_TO_TYPE = "and this ";
+ changeKeyboardLocaleAndDictLocale("th", "en_US");
+ type(WORD_TO_TYPE);
+ assertEquals("don't resume suggestion on backspace", -1,
+ BaseInputConnection.getComposingSpanStart(mEditText.getText()));
+ assertEquals("don't resume suggestion on backspace", -1,
+ BaseInputConnection.getComposingSpanEnd(mEditText.getText()));
+ type(" ");
+ type(Constants.CODE_DELETE);
+ assertEquals("don't resume suggestion on backspace", -1,
+ BaseInputConnection.getComposingSpanStart(mEditText.getText()));
+ assertEquals("don't resume suggestion on backspace", -1,
+ BaseInputConnection.getComposingSpanEnd(mEditText.getText()));
+ }
+
+ public void testStartComposingInsideText() {
+ final String WORD_TO_TYPE = "abcdefgh ";
+ final int typedLength = WORD_TO_TYPE.length() - 1; // -1 because space gets eaten
+ final int CURSOR_POS = 4;
+ changeKeyboardLocaleAndDictLocale("th", "en_US");
+ type(WORD_TO_TYPE);
+ mLatinIME.onUpdateSelection(0, 0, typedLength, typedLength, -1, -1);
+ mInputConnection.setSelection(CURSOR_POS, CURSOR_POS);
+ mLatinIME.onUpdateSelection(typedLength, typedLength,
+ CURSOR_POS, CURSOR_POS, -1, -1);
+ sleep(DELAY_TO_WAIT_FOR_PREDICTIONS);
+ runMessages();
+ assertEquals("start composing inside text", -1,
+ BaseInputConnection.getComposingSpanStart(mEditText.getText()));
+ assertEquals("start composing inside text", -1,
+ BaseInputConnection.getComposingSpanEnd(mEditText.getText()));
+ type("xxxx");
+ assertEquals("start composing inside text", 4,
+ BaseInputConnection.getComposingSpanStart(mEditText.getText()));
+ assertEquals("start composing inside text", 8,
+ BaseInputConnection.getComposingSpanEnd(mEditText.getText()));
+ }
+
+ public void testPredictions() {
+ final String WORD_TO_TYPE = "Barack ";
+ changeKeyboardLocaleAndDictLocale("th", "en_US");
+ type(WORD_TO_TYPE);
+ sleep(DELAY_TO_WAIT_FOR_PREDICTIONS);
+ runMessages();
+ // Make sure there is no space
+ assertEquals("predictions in lang without spaces", "Barack",
+ mEditText.getText().toString());
+ // Test the first prediction is displayed
+ assertEquals("predictions in lang without spaces", "Obama",
+ mLatinIME.getFirstSuggestedWord());
+ }
+}
diff --git a/tests/src/com/android/inputmethod/latin/InputTestsBase.java b/tests/src/com/android/inputmethod/latin/InputTestsBase.java
index ead134b..480570e 100644
--- a/tests/src/com/android/inputmethod/latin/InputTestsBase.java
+++ b/tests/src/com/android/inputmethod/latin/InputTestsBase.java
@@ -46,6 +46,8 @@
// The message that sets the underline is posted with a 100 ms delay
protected static final int DELAY_TO_WAIT_FOR_UNDERLINE = 200;
+ // The message that sets predictions is posted with a 100 ms delay
+ protected static final int DELAY_TO_WAIT_FOR_PREDICTIONS = 200;
protected LatinIME mLatinIME;
protected Keyboard mKeyboard;
@@ -207,8 +209,8 @@
//mLatinIME.onPressKey(codePoint, 0 /* repeatCount */, true /* isSinglePointer */);
final Key key = mKeyboard.getKey(codePoint);
if (key != null) {
- final int x = key.mX + key.mWidth / 2;
- final int y = key.mY + key.mHeight / 2;
+ final int x = key.getX() + key.getWidth() / 2;
+ final int y = key.getY() + key.getHeight() / 2;
mLatinIME.onCodeInput(codePoint, x, y);
return;
}
@@ -233,9 +235,6 @@
--remainingAttempts;
}
}
- if (!mLatinIME.hasMainDictionary()) {
- throw new RuntimeException("Can't initialize the main dictionary");
- }
}
protected void changeLanguage(final String locale) {
@@ -247,6 +246,16 @@
waitForDictionaryToBeLoaded();
}
+ protected void changeKeyboardLocaleAndDictLocale(final String keyboardLocale,
+ final String dictLocale) {
+ changeLanguage(keyboardLocale);
+ if (!keyboardLocale.equals(dictLocale)) {
+ mLatinIME.replaceMainDictionaryForTest(
+ LocaleUtils.constructLocaleFromString(dictLocale));
+ }
+ waitForDictionaryToBeLoaded();
+ }
+
protected void pickSuggestionManually(final int index, final String suggestion) {
mLatinIME.pickSuggestionManually(index, new SuggestedWordInfo(suggestion, 1,
SuggestedWordInfo.KIND_CORRECTION, "main"));
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
similarity index 87%
rename from tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
rename to tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
index 7bfd603..be468c1 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
@@ -22,10 +22,10 @@
import android.util.Log;
import android.util.SparseArray;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.CollectionUtils;
@@ -44,11 +44,11 @@
import java.util.Set;
/**
- * Unit tests for BinaryDictInputOutput
+ * Unit tests for BinaryDictDecoder and BinaryDictEncoder.
*/
@LargeTest
-public class BinaryDictIOTests extends AndroidTestCase {
- private static final String TAG = BinaryDictIOTests.class.getSimpleName();
+public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
+ private static final String TAG = BinaryDictDecoderEncoderTests.class.getSimpleName();
private static final int DEFAULT_MAX_UNIGRAMS = 100;
private static final int DEFAULT_CODE_POINT_SET_SIZE = 50;
private static final int UNIGRAM_FREQ = 10;
@@ -71,11 +71,13 @@
private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */);
- public BinaryDictIOTests() {
+ private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
+
+ public BinaryDictDecoderEncoderTests() {
this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS);
}
- public BinaryDictIOTests(final long seed, final int maxUnigrams) {
+ public BinaryDictDecoderEncoderTests(final long seed, final int maxUnigrams) {
super();
Log.e(TAG, "Testing dictionary: seed is " + seed);
final Random random = new Random(seed);
@@ -206,7 +208,7 @@
// If you need to dump the dict to a textual file, uncomment the line below and the
// function above
// dumpToCombinedFileForDebug(file, "/tmp/foo");
- BinaryDictInputOutput.writeDictionaryBinary(out, dict, formatOptions);
+ BinaryDictEncoder.writeDictionaryBinary(out, dict, formatOptions);
diff = System.currentTimeMillis() - now;
out.flush();
@@ -226,7 +228,7 @@
// check unigram
for (final String word : words) {
- final CharGroup cg = FusionDictionary.findWordInTree(dict.mRoot, word);
+ final CharGroup cg = FusionDictionary.findWordInTree(dict.mRootNodeArray, word);
assertNotNull(cg);
}
@@ -234,7 +236,8 @@
for (int i = 0; i < bigrams.size(); ++i) {
final int w1 = bigrams.keyAt(i);
for (final int w2 : bigrams.valueAt(i)) {
- final CharGroup cg = FusionDictionary.findWordInTree(dict.mRoot, words.get(w1));
+ final CharGroup cg = FusionDictionary.findWordInTree(dict.mRootNodeArray,
+ words.get(w1));
assertNotNull(words.get(w1) + "," + words.get(w2), cg.getBigram(words.get(w2)));
}
}
@@ -242,7 +245,8 @@
// check shortcut
if (shortcutMap != null) {
for (final Map.Entry<String, List<String>> entry : shortcutMap.entrySet()) {
- final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, entry.getKey());
+ final CharGroup group = FusionDictionary.findWordInTree(dict.mRootNodeArray,
+ entry.getKey());
for (final String word : entry.getValue()) {
assertNotNull("shortcut not found: " + entry.getKey() + ", " + word,
group.getShortcut(word));
@@ -272,7 +276,7 @@
getBuffer(reader, bufferType);
assertNotNull(reader.getBuffer());
now = System.currentTimeMillis();
- dict = BinaryDictInputOutput.readDictionaryBinary(reader, null);
+ dict = BinaryDictDecoder.readDictionaryBinary(reader, null);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {
Log.e(TAG, "IOException while reading dictionary", e);
@@ -291,13 +295,14 @@
final String message) {
File file = null;
try {
- file = File.createTempFile("runReadAndWrite", ".dict", getContext().getCacheDir());
+ file = File.createTempFile("runReadAndWrite", TEST_DICT_FILE_EXTENSION,
+ getContext().getCacheDir());
} catch (IOException e) {
Log.e(TAG, "IOException", e);
}
assertNotNull(file);
- final FusionDictionary dict = new FusionDictionary(new Node(),
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false));
addUnigrams(words.size(), dict, words, shortcuts);
addBigrams(dict, words, bigrams);
@@ -383,7 +388,7 @@
}
actBigrams.get(word1).add(word2);
- final int bigramFreq = BinaryDictInputOutput.reconstructBigramFrequency(
+ final int bigramFreq = BinaryDictDecoder.reconstructBigramFrequency(
unigramFreq, attr.mFrequency);
assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ);
}
@@ -433,14 +438,15 @@
final FormatSpec.FormatOptions formatOptions, final String message) {
File file = null;
try {
- file = File.createTempFile("runReadUnigrams", ".dict", getContext().getCacheDir());
+ file = File.createTempFile("runReadUnigrams", TEST_DICT_FILE_EXTENSION,
+ getContext().getCacheDir());
} catch (IOException e) {
Log.e(TAG, "IOException", e);
}
assertNotNull(file);
// making the dictionary from lists of words.
- final FusionDictionary dict = new FusionDictionary(new Node(),
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(
new HashMap<String, String>(), false, false));
addUnigrams(words.size(), dict, words, null /* shortcutMap */);
@@ -491,31 +497,31 @@
}
// Tests for getTerminalPosition
- private String getWordFromBinary(final FusionDictionaryBufferInterface buffer,
- final int address) {
+ private String getWordFromBinary(final BinaryDictReader dictReader, final int address) {
+ final FusionDictionaryBufferInterface buffer = dictReader.getBuffer();
if (buffer.position() != 0) buffer.position(0);
- FileHeader header = null;
+ FileHeader fileHeader = null;
try {
- header = BinaryDictInputOutput.readHeader(buffer);
+ fileHeader = BinaryDictDecoder.readHeader(dictReader);
} catch (IOException e) {
return null;
} catch (UnsupportedFormatException e) {
return null;
}
- if (header == null) return null;
- return BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize,
- address - header.mHeaderSize, header.mFormatOptions).mWord;
+ if (fileHeader == null) return null;
+ return BinaryDictDecoder.getWordAtAddress(buffer, fileHeader.mHeaderSize,
+ address - fileHeader.mHeaderSize, fileHeader.mFormatOptions).mWord;
}
- private long runGetTerminalPosition(final FusionDictionaryBufferInterface buffer,
- final String word, int index, boolean contained) {
+ private long runGetTerminalPosition(final BinaryDictReader reader, final String word, int index,
+ boolean contained) {
final int expectedFrequency = (UNIGRAM_FREQ + index) % 255;
long diff = -1;
int position = -1;
try {
final long now = System.nanoTime();
- position = BinaryDictIOUtils.getTerminalPosition(buffer, word);
+ position = BinaryDictIOUtils.getTerminalPosition(reader, word);
diff = System.nanoTime() - now;
} catch (IOException e) {
Log.e(TAG, "IOException while getTerminalPosition", e);
@@ -524,50 +530,48 @@
}
assertEquals(FormatSpec.NOT_VALID_WORD != position, contained);
- if (contained) assertEquals(getWordFromBinary(buffer, position), word);
+ if (contained) assertEquals(getWordFromBinary(reader, position), word);
return diff;
}
public void testGetTerminalPosition() {
File file = null;
try {
- file = File.createTempFile("testGetTerminalPosition", ".dict",
+ file = File.createTempFile("testGetTerminalPosition", TEST_DICT_FILE_EXTENSION,
getContext().getCacheDir());
} catch (IOException e) {
// do nothing
}
assertNotNull(file);
- final FusionDictionary dict = new FusionDictionary(new Node(),
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(
new HashMap<String, String>(), false, false));
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final BinaryDictReader reader = new BinaryDictReader(file);
- FusionDictionaryBufferInterface buffer = null;
try {
- buffer = reader.openAndGetBuffer(
- new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
+ reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
- assertNotNull("Can't get the buffer", buffer);
+ assertNotNull("Can't get the buffer", reader.getBuffer());
try {
// too long word
final String longWord = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
assertEquals(FormatSpec.NOT_VALID_WORD,
- BinaryDictIOUtils.getTerminalPosition(buffer, longWord));
+ BinaryDictIOUtils.getTerminalPosition(reader, longWord));
// null
assertEquals(FormatSpec.NOT_VALID_WORD,
- BinaryDictIOUtils.getTerminalPosition(buffer, null));
+ BinaryDictIOUtils.getTerminalPosition(reader, null));
// empty string
assertEquals(FormatSpec.NOT_VALID_WORD,
- BinaryDictIOUtils.getTerminalPosition(buffer, ""));
+ BinaryDictIOUtils.getTerminalPosition(reader, ""));
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
}
@@ -575,7 +579,7 @@
// Test a word that is contained within the dictionary.
long sum = 0;
for (int i = 0; i < sWords.size(); ++i) {
- final long time = runGetTerminalPosition(buffer, sWords.get(i), i, true);
+ final long time = runGetTerminalPosition(reader, sWords.get(i), i, true);
sum += time == -1 ? 0 : time;
}
Log.d(TAG, "per a search : " + (((double)sum) / sWords.size() / 1000000));
@@ -586,48 +590,48 @@
for (int i = 0; i < 1000; ++i) {
final String word = generateWord(random, codePointSet);
if (sWords.indexOf(word) != -1) continue;
- runGetTerminalPosition(buffer, word, i, false);
+ runGetTerminalPosition(reader, word, i, false);
}
}
public void testDeleteWord() {
File file = null;
try {
- file = File.createTempFile("testDeleteWord", ".dict", getContext().getCacheDir());
+ file = File.createTempFile("testDeleteWord", TEST_DICT_FILE_EXTENSION,
+ getContext().getCacheDir());
} catch (IOException e) {
// do nothing
}
assertNotNull(file);
- final FusionDictionary dict = new FusionDictionary(new Node(),
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(
new HashMap<String, String>(), false, false));
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final BinaryDictReader reader = new BinaryDictReader(file);
- FusionDictionaryBufferInterface buffer = null;
try {
- buffer = reader.openAndGetBuffer(
+ reader.openBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
- assertNotNull("Can't get the buffer", buffer);
+ assertNotNull("Can't get the buffer", reader.getBuffer());
try {
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
- BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(0)));
- DynamicBinaryDictIOUtils.deleteWord(buffer, sWords.get(0));
+ BinaryDictIOUtils.getTerminalPosition(reader, sWords.get(0)));
+ DynamicBinaryDictIOUtils.deleteWord(reader, sWords.get(0));
assertEquals(FormatSpec.NOT_VALID_WORD,
- BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(0)));
+ BinaryDictIOUtils.getTerminalPosition(reader, sWords.get(0)));
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
- BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(5)));
- DynamicBinaryDictIOUtils.deleteWord(buffer, sWords.get(5));
+ BinaryDictIOUtils.getTerminalPosition(reader, sWords.get(5)));
+ DynamicBinaryDictIOUtils.deleteWord(reader, sWords.get(5));
assertEquals(FormatSpec.NOT_VALID_WORD,
- BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(5)));
+ BinaryDictIOUtils.getTerminalPosition(reader, sWords.get(5)));
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
}
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
index e759507..bcf2c31 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
@@ -21,10 +21,11 @@
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.ByteBufferWrapper;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictReader.
+ FusionDictionaryBufferFromWritableByteBufferFactory;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.CollectionUtils;
@@ -33,8 +34,6 @@
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Random;
@@ -49,6 +48,8 @@
public static final int DEFAULT_MAX_UNIGRAMS = 1500;
private final int mMaxUnigrams;
+ private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
+
private static final String[] CHARACTERS = {
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
@@ -114,10 +115,10 @@
private static void printNode(final FusionDictionaryBufferInterface buffer,
final FormatSpec.FormatOptions formatOptions) {
Log.d(TAG, "Node at " + buffer.position());
- final int count = BinaryDictInputOutput.readCharGroupCount(buffer);
+ final int count = BinaryDictDecoder.readCharGroupCount(buffer);
Log.d(TAG, " charGroupCount = " + count);
for (int i = 0; i < count; ++i) {
- final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer,
+ final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer,
buffer.position(), formatOptions);
printCharGroup(currentInfo);
}
@@ -127,22 +128,24 @@
}
}
- private static void printBinaryFile(final FusionDictionaryBufferInterface buffer)
+ private static void printBinaryFile(final BinaryDictReader dictReader)
throws IOException, UnsupportedFormatException {
- FileHeader header = BinaryDictInputOutput.readHeader(buffer);
+ final FileHeader fileHeader = BinaryDictDecoder.readHeader(dictReader);
+ final FusionDictionaryBufferInterface buffer = dictReader.getBuffer();
while (buffer.position() < buffer.limit()) {
- printNode(buffer, header.mFormatOptions);
+ printNode(buffer, fileHeader.mFormatOptions);
}
}
private int getWordPosition(final File file, final String word) {
int position = FormatSpec.NOT_VALID_WORD;
+ final BinaryDictReader dictReader = new BinaryDictReader(file);
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
- final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper(
- inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length()));
- position = BinaryDictIOUtils.getTerminalPosition(buffer, word);
+ dictReader.openBuffer(
+ new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
+ position = BinaryDictIOUtils.getTerminalPosition(dictReader, word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
} finally {
@@ -158,23 +161,14 @@
}
private CharGroupInfo findWordFromFile(final File file, final String word) {
- FileInputStream inStream = null;
+ final BinaryDictReader dictReader = new BinaryDictReader(file);
CharGroupInfo info = null;
try {
- inStream = new FileInputStream(file);
- final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper(
- inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length()));
- info = BinaryDictIOUtils.findWordFromBuffer(buffer, word);
+ dictReader.openBuffer(
+ new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
+ info = BinaryDictIOUtils.findWordByBinaryDictReader(dictReader, word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
- } finally {
- if (inStream != null) {
- try {
- inStream.close();
- } catch (IOException e) {
- // do nothing
- }
- }
}
return info;
}
@@ -183,42 +177,33 @@
private long insertAndCheckWord(final File file, final String word, final int frequency,
final boolean exist, final ArrayList<WeightedString> bigrams,
final ArrayList<WeightedString> shortcuts) {
- RandomAccessFile raFile = null;
+ final BinaryDictReader dictReader = new BinaryDictReader(file);
BufferedOutputStream outStream = null;
- FusionDictionaryBufferInterface buffer = null;
long amountOfTime = -1;
try {
- raFile = new RandomAccessFile(file, "rw");
- buffer = new ByteBufferWrapper(raFile.getChannel().map(
- FileChannel.MapMode.READ_WRITE, 0, file.length()));
+ dictReader.openBuffer(new FusionDictionaryBufferFromWritableByteBufferFactory());
outStream = new BufferedOutputStream(new FileOutputStream(file, true));
if (!exist) {
assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word));
}
final long now = System.nanoTime();
- DynamicBinaryDictIOUtils.insertWord(buffer, outStream, word, frequency, bigrams,
+ DynamicBinaryDictIOUtils.insertWord(dictReader, outStream, word, frequency, bigrams,
shortcuts, false, false);
amountOfTime = System.nanoTime() - now;
outStream.flush();
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word));
outStream.close();
- raFile.close();
} catch (IOException e) {
+ Log.e(TAG, "Raised an IOException while inserting a word", e);
} catch (UnsupportedFormatException e) {
+ Log.e(TAG, "Raised an UnsupportedFormatException error while inserting a word", e);
} finally {
if (outStream != null) {
try {
outStream.close();
} catch (IOException e) {
- // do nothing
- }
- }
- if (raFile != null) {
- try {
- raFile.close();
- } catch (IOException e) {
- // do nothing
+ Log.e(TAG, "Failed to close the output stream", e);
}
}
}
@@ -226,64 +211,49 @@
}
private void deleteWord(final File file, final String word) {
- RandomAccessFile raFile = null;
- FusionDictionaryBufferInterface buffer = null;
+ final BinaryDictReader dictReader = new BinaryDictReader(file);
try {
- raFile = new RandomAccessFile(file, "rw");
- buffer = new ByteBufferWrapper(raFile.getChannel().map(
- FileChannel.MapMode.READ_WRITE, 0, file.length()));
- DynamicBinaryDictIOUtils.deleteWord(buffer, word);
+ dictReader.openBuffer(new FusionDictionaryBufferFromWritableByteBufferFactory());
+ DynamicBinaryDictIOUtils.deleteWord(dictReader, word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
- } finally {
- if (raFile != null) {
- try {
- raFile.close();
- } catch (IOException e) {
- // do nothing
- }
- }
}
}
private void checkReverseLookup(final File file, final String word, final int position) {
- FileInputStream inStream = null;
+ final BinaryDictReader dictReader = new BinaryDictReader(file);
try {
- inStream = new FileInputStream(file);
- final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper(
- inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length()));
- final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
- assertEquals(word, BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize,
- position - header.mHeaderSize, header.mFormatOptions).mWord);
+ final FusionDictionaryBufferInterface buffer = dictReader.openAndGetBuffer(
+ new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
+ final FileHeader fileHeader = BinaryDictDecoder.readHeader(dictReader);
+ assertEquals(word,
+ BinaryDictDecoder.getWordAtAddress(dictReader.getBuffer(),
+ fileHeader.mHeaderSize, position - fileHeader.mHeaderSize,
+ fileHeader.mFormatOptions).mWord);
} catch (IOException e) {
+ Log.e(TAG, "Raised an IOException while looking up a word", e);
} catch (UnsupportedFormatException e) {
- } finally {
- if (inStream != null) {
- try {
- inStream.close();
- } catch (IOException e) {
- // do nothing
- }
- }
+ Log.e(TAG, "Raised an UnsupportedFormatException error while looking up a word", e);
}
}
public void testInsertWord() {
File file = null;
try {
- file = File.createTempFile("testInsertWord", ".dict", getContext().getCacheDir());
+ file = File.createTempFile("testInsertWord", TEST_DICT_FILE_EXTENSION,
+ getContext().getCacheDir());
} catch (IOException e) {
fail("IOException while creating temporary file: " + e);
}
// set an initial dictionary.
- final FusionDictionary dict = new FusionDictionary(new Node(),
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false));
dict.add("abcd", 10, null, false);
try {
final FileOutputStream out = new FileOutputStream(file);
- BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
+ BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
out.close();
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
@@ -321,21 +291,21 @@
public void testInsertWordWithBigrams() {
File file = null;
try {
- file = File.createTempFile("testInsertWordWithBigrams", ".dict",
+ file = File.createTempFile("testInsertWordWithBigrams", TEST_DICT_FILE_EXTENSION,
getContext().getCacheDir());
} catch (IOException e) {
fail("IOException while creating temporary file: " + e);
}
// set an initial dictionary.
- final FusionDictionary dict = new FusionDictionary(new Node(),
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false));
dict.add("abcd", 10, null, false);
dict.add("efgh", 15, null, false);
try {
final FileOutputStream out = new FileOutputStream(file);
- BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
+ BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
out.close();
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
@@ -359,20 +329,21 @@
public void testRandomWords() {
File file = null;
try {
- file = File.createTempFile("testRandomWord", ".dict", getContext().getCacheDir());
+ file = File.createTempFile("testRandomWord", TEST_DICT_FILE_EXTENSION,
+ getContext().getCacheDir());
} catch (IOException e) {
}
assertNotNull(file);
// set an initial dictionary.
- final FusionDictionary dict = new FusionDictionary(new Node(),
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false,
false));
dict.add("initial", 10, null, false);
try {
final FileOutputStream out = new FileOutputStream(file);
- BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
+ BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
out.close();
} catch (IOException e) {
assertTrue(false);
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java
index a46e583..1c6de50 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java
@@ -16,7 +16,7 @@
package com.android.inputmethod.latin.makedict;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory;
import com.android.inputmethod.latin.makedict.BinaryDictReader.
FusionDictionaryBufferFromByteArrayFactory;
diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
index b3e2ee0..304b792 100644
--- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
@@ -22,6 +22,7 @@
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
+import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
@@ -29,6 +30,7 @@
import java.util.List;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
/**
* Unit tests for UserHistoryDictionary
@@ -43,6 +45,8 @@
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
};
+ private static final int MIN_USER_HISTORY_DICTIONARY_FILE_SIZE = 1000;
+
@Override
public void setUp() {
mPrefs = PreferenceManager.getDefaultSharedPreferences(getContext());
@@ -78,43 +82,43 @@
}
}
+ private void addAndWriteRandomWords(final String testFilenameSuffix, final int numberOfWords,
+ final Random random) {
+ final List<String> words = generateWords(numberOfWords, random);
+ final UserHistoryPredictionDictionary dict =
+ PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary(getContext(),
+ testFilenameSuffix /* locale */, mPrefs);
+ // Add random words to the user history dictionary.
+ addToDict(dict, words);
+ // write to file.
+ dict.close();
+ }
+
public void testRandomWords() {
File dictFile = null;
+ Log.d(TAG, "This test can be used for profiling.");
+ Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
+ final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis();
+ final int numberOfWords = 1000;
+ final Random random = new Random(123456);
+
try {
- Log.d(TAG, "This test can be used for profiling.");
- Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
- final int numberOfWords = 1000;
- final Random random = new Random(123456);
- List<String> words = generateWords(numberOfWords, random);
-
- final String locale = "testRandomWords";
- final String fileName = "UserHistoryDictionary." + locale + ".dict";
- dictFile = new File(getContext().getFilesDir(), fileName);
- final UserHistoryPredictionDictionary dict =
- PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary(
- getContext(), locale, mPrefs);
- dict.mIsTest = true;
-
- addToDict(dict, words);
-
- try {
- Log.d(TAG, "waiting for adding the word ...");
- Thread.sleep(2000);
- } catch (InterruptedException e) {
- Log.d(TAG, "InterruptedException: " + e);
- }
-
- // write to file
- dict.close();
-
+ addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random);
+ } finally {
try {
Log.d(TAG, "waiting for writing ...");
- Thread.sleep(5000);
+ Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS));
} catch (InterruptedException e) {
Log.d(TAG, "InterruptedException: " + e);
}
- } finally {
+
+ final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
+ + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
+ dictFile = new File(getContext().getFilesDir(), fileName);
+
if (dictFile != null) {
+ assertTrue(dictFile.exists());
+ assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
dictFile.delete();
}
}
@@ -122,49 +126,46 @@
public void testStressTestForSwitchingLanguagesAndAddingWords() {
final int numberOfLanguages = 2;
- final int numberOfLanguageSwitching = 100;
- final int numberOfWordsIntertedForEachLanguageSwitch = 100;
+ final int numberOfLanguageSwitching = 80;
+ final int numberOfWordsInsertedForEachLanguageSwitch = 100;
final File dictFiles[] = new File[numberOfLanguages];
try {
final Random random = new Random(123456);
- // Create locales for this test.
- String locales[] = new String[numberOfLanguages];
+ // Create filename suffixes for this test.
+ String testFilenameSuffixes[] = new String[numberOfLanguages];
for (int i = 0; i < numberOfLanguages; i++) {
- locales[i] = "testSwitchingLanguages" + i;
- final String fileName = "UserHistoryDictionary." + locales[i] + ".dict";
+ testFilenameSuffixes[i] = "testSwitchingLanguages" + i;
+ final String fileName = UserHistoryPredictionDictionary.NAME + "." +
+ testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFiles[i] = new File(getContext().getFilesDir(), fileName);
}
- final long now = System.currentTimeMillis();
+ final long start = System.currentTimeMillis();
for (int i = 0; i < numberOfLanguageSwitching; i++) {
final int index = i % numberOfLanguages;
- // Switch languages to locales[index].
- final UserHistoryPredictionDictionary dict =
- PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary(
- getContext(), locales[index], mPrefs);
- final List<String> words = generateWords(
- numberOfWordsIntertedForEachLanguageSwitch, random);
- // Add random words to the user history dictionary.
- addToDict(dict, words);
- // write to file
- dict.close();
+ // Switch languages to testFilenameSuffixes[index].
+ addAndWriteRandomWords(testFilenameSuffixes[index],
+ numberOfWordsInsertedForEachLanguageSwitch, random);
}
final long end = System.currentTimeMillis();
Log.d(TAG, "testStressTestForSwitchingLanguageAndAddingWords took "
- + (end - now) + " ms");
+ + (end - start) + " ms");
+ } finally {
+ Log.d(TAG, "waiting for writing ...");
try {
Log.d(TAG, "waiting for writing ...");
- Thread.sleep(5000);
+ Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS));
} catch (InterruptedException e) {
Log.d(TAG, "InterruptedException: " + e);
}
- } finally {
for (final File file : dictFiles) {
if (file != null) {
+ assertTrue(file.exists());
+ assertTrue(file.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
file.delete();
}
}
diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
index 175e511..c6fa943 100644
--- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
+++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
@@ -256,4 +256,16 @@
// code for now True is acceptable.
assertTrue(StringUtils.lastPartLooksLikeURL(".abc/def"));
}
+
+ public void testHexStringUtils() {
+ final byte[] bytes = new byte[] { (byte)0x01, (byte)0x11, (byte)0x22, (byte)0x33,
+ (byte)0x55, (byte)0x88, (byte)0xEE };
+ final String bytesStr = StringUtils.byteArrayToHexString(bytes);
+ final byte[] bytes2 = StringUtils.hexStringToByteArray(bytesStr);
+ for (int i = 0; i < bytes.length; ++i) {
+ assertTrue(bytes[i] == bytes2[i]);
+ }
+ final String bytesStr2 = StringUtils.byteArrayToHexString(bytes2);
+ assertTrue(bytesStr.equals(bytesStr2));
+ }
}
diff --git a/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java
index baebda2..856b2db 100644
--- a/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java
+++ b/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java
@@ -214,7 +214,7 @@
SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR_CA));
assertEquals("de ", "Allemand",
SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(DE));
- assertEquals("zz ", "Alphabet (QWERTY)",
+ assertEquals("zz ", "Alphabet latin (QWERTY)",
SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ));
return null;
}
@@ -236,7 +236,7 @@
SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(EN_UK_DVORAK));
assertEquals("es_US colemak","Espagnol (États-Unis) (Colemak)",
SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ES_US_COLEMAK));
- assertEquals("zz pc", "Alphabet (PC)",
+ assertEquals("zz pc", "Alphabet latin (PC)",
SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ_PC));
return null;
}
diff --git a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
index ce62bf2..8f5bec8 100644
--- a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
+++ b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
@@ -49,6 +49,7 @@
private static final int BIGRAM_FREQUENCY = 100;
private static final ArrayList<String> NOT_HAVE_BIGRAM = new ArrayList<String>();
private static final FormatSpec.FormatOptions FORMAT_OPTIONS = new FormatSpec.FormatOptions(2);
+ private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
/**
* Return same frequency for all words and bigrams
@@ -86,7 +87,7 @@
private void checkWordInFusionDict(final FusionDictionary dict, final String word,
final ArrayList<String> expectedBigrams) {
- final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, word);
+ final CharGroup group = FusionDictionary.findWordInTree(dict.mRootNodeArray, word);
assertNotNull(group);
assertTrue(group.isTerminal());
@@ -177,7 +178,8 @@
File file = null;
try {
- file = File.createTempFile("testReadAndWrite", ".dict", getContext().getCacheDir());
+ file = File.createTempFile("testReadAndWrite", TEST_DICT_FILE_EXTENSION,
+ getContext().getCacheDir());
} catch (IOException e) {
Log.d(TAG, "IOException while creating a temporary file", e);
}
diff --git a/tools/dicttool/NativeLib.mk b/tools/dicttool/NativeLib.mk
index eeae003..84e54a9 100644
--- a/tools/dicttool/NativeLib.mk
+++ b/tools/dicttool/NativeLib.mk
@@ -34,7 +34,7 @@
# Used in jni_common.cpp to avoid registering useless methods.
LATIN_IME_JNI_SRC_FILES := \
- com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp \
+ com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp \
jni_common.cpp
LATIN_IME_CORE_SRC_FILES :=
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
index d0b460a..4ddfb40 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
@@ -16,7 +16,7 @@
package com.android.inputmethod.latin.dicttool;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
@@ -31,8 +31,6 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
import java.util.ArrayList;
import javax.xml.parsers.ParserConfigurationException;
@@ -99,7 +97,7 @@
// over and over, ending in a stack overflow. Hence we limit the depth at which we try
// decoding the file.
if (depth > MAX_DECODE_DEPTH) return null;
- if (BinaryDictInputOutput.isBinaryDictionary(src)) {
+ if (BinaryDictDecoder.isBinaryDictionary(src)) {
spec.mFile = src;
return spec;
}
@@ -194,7 +192,7 @@
System.out.println("Packaging : " + decodedSpec.describeChain());
System.out.println("Uncompressed size : " + decodedSpec.mFile.length());
}
- return BinaryDictInputOutput.readDictionaryBinary(reader, null);
+ return BinaryDictDecoder.readDictionaryBinary(reader, null);
}
}
} catch (IOException e) {
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
index 092ee76..4b67169 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
@@ -19,7 +19,7 @@
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.Word;
@@ -117,7 +117,7 @@
final boolean processLigatures =
FRENCH_LIGATURE_PROCESSING_OPTION.equals(attributes.get(OPTIONS_TAG));
attributes.remove(OPTIONS_TAG);
- final FusionDictionary dict = new FusionDictionary(new Node(), new DictionaryOptions(
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions(
attributes, processUmlauts, processLigatures));
String line;
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
index 9bce988..09052c9 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
@@ -16,7 +16,8 @@
package com.android.inputmethod.latin.dicttool;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
+import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
@@ -30,8 +31,6 @@
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
import java.util.Arrays;
import java.util.LinkedList;
@@ -177,7 +176,7 @@
inputUnigramXml = filename;
} else if (CombinedInputOutput.isCombinedDictionary(filename)) {
inputCombined = filename;
- } else if (BinaryDictInputOutput.isBinaryDictionary(filename)) {
+ } else if (BinaryDictDecoder.isBinaryDictionary(filename)) {
inputBinary = filename;
} else {
throw new IllegalArgumentException(
@@ -199,7 +198,7 @@
}
} else {
if (null == inputBinary && null == inputUnigramXml) {
- if (BinaryDictInputOutput.isBinaryDictionary(arg)) {
+ if (BinaryDictDecoder.isBinaryDictionary(arg)) {
inputBinary = arg;
} else if (CombinedInputOutput.isCombinedDictionary(arg)) {
inputCombined = arg;
@@ -269,7 +268,7 @@
final File file = new File(binaryFilename);
final BinaryDictReader reader = new BinaryDictReader(file);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
- return BinaryDictInputOutput.readDictionaryBinary(reader, null);
+ return BinaryDictDecoder.readDictionaryBinary(reader, null);
}
/**
@@ -358,7 +357,7 @@
throws FileNotFoundException, IOException, UnsupportedFormatException {
final File outputFile = new File(outputFilename);
final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version);
- BinaryDictInputOutput.writeDictionaryBinary(new FileOutputStream(outputFilename), dict,
+ BinaryDictEncoder.writeDictionaryBinary(new FileOutputStream(outputFilename), dict,
formatOptions);
}
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
index 5c3e87e..d790d06 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
@@ -121,7 +121,8 @@
private static void diffWords(final FusionDictionary dict0, final FusionDictionary dict1) {
boolean hasDifferences = false;
for (final Word word0 : dict0) {
- final CharGroup word1 = FusionDictionary.findWordInTree(dict1.mRoot, word0.mWord);
+ final CharGroup word1 = FusionDictionary.findWordInTree(dict1.mRootNodeArray,
+ word0.mWord);
if (null == word1) {
// This word is not in dict1
System.out.println("Deleted: " + word0.mWord + " " + word0.mFrequency);
@@ -150,7 +151,8 @@
}
}
for (final Word word1 : dict1) {
- final CharGroup word0 = FusionDictionary.findWordInTree(dict0.mRoot, word1.mWord);
+ final CharGroup word0 = FusionDictionary.findWordInTree(dict0.mRootNodeArray,
+ word1.mWord);
if (null == word0) {
// This word is not in dict0
System.out.println("Added: " + word1.mWord + " " + word1.mFrequency);
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
index f289454..fa8c5f7 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
@@ -65,7 +65,7 @@
private static void showWordInfo(final FusionDictionary dict, final String word,
final boolean plumbing) {
- final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, word);
+ final CharGroup group = FusionDictionary.findWordInTree(dict.mRootNodeArray, word);
if (null == group) {
System.out.println(word + " is not in the dictionary");
return;
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java
index 972b6e7..9174238 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java
@@ -16,9 +16,9 @@
package com.android.inputmethod.latin.dicttool;
-import com.android.inputmethod.latin.makedict.BinaryDictIOTests;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoderEncoderTests;
+import com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtilsTests;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest;
import com.android.inputmethod.latin.makedict.FusionDictionaryTest;
import java.lang.reflect.Constructor;
@@ -37,9 +37,9 @@
private static final Class<?>[] sClassesToTest = {
BinaryDictOffdeviceUtilsTests.class,
FusionDictionaryTest.class,
- BinaryDictInputOutputTest.class,
- BinaryDictIOUtilsTests.class,
- BinaryDictIOTests.class
+ BinaryDictDecoderEncoderTests.class,
+ BinaryDictEncoderFlattenTreeTests.class,
+ BinaryDictIOUtilsTests.class
};
private ArrayList<Method> mAllTestMethods = new ArrayList<Method>();
private ArrayList<String> mUsedTestMethods = new ArrayList<String>();
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
index 1fd2cba..4e99bf9 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
@@ -18,7 +18,7 @@
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.Word;
@@ -124,8 +124,8 @@
GERMAN_UMLAUT_PROCESSING_OPTION.equals(optionsString);
final boolean processLigatures =
FRENCH_LIGATURE_PROCESSING_OPTION.equals(optionsString);
- mDictionary = new FusionDictionary(new Node(), new DictionaryOptions(attributes,
- processUmlauts, processLigatures));
+ mDictionary = new FusionDictionary(new PtNodeArray(),
+ new DictionaryOptions(attributes, processUmlauts, processLigatures));
} else {
mState = UNKNOWN;
}
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
index fb1cc84..3bda77f 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
@@ -16,36 +16,33 @@
package com.android.inputmethod.latin.dicttool;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
+import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import junit.framework.TestCase;
import java.io.File;
import java.io.BufferedOutputStream;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.ArrayList;
import java.util.HashMap;
/**
- * Unit tests for BinaryDictOffdeviceUtilsTests
+ * Unit tests for BinaryDictOffdeviceUtils
*/
public class BinaryDictOffdeviceUtilsTests extends TestCase {
private static final int TEST_FREQ = 37; // Some arbitrary value unlikely to happen by chance
public void testGetRawDictWorks() throws IOException, UnsupportedFormatException {
// Create a thrice-compressed dictionary file.
- final FusionDictionary dict = new FusionDictionary(new Node(),
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new DictionaryOptions(new HashMap<String, String>(),
false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */));
dict.add("foo", TEST_FREQ, null, false /* isNotAWord */);
@@ -61,7 +58,7 @@
Compress.getCompressedStream(
new BufferedOutputStream(new FileOutputStream(dst)))));
- BinaryDictInputOutput.writeDictionaryBinary(out, dict, new FormatOptions(2, false));
+ BinaryDictEncoder.writeDictionaryBinary(out, dict, new FormatOptions(2, false));
// Test for an actually compressed dictionary and its contents
final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec =
@@ -72,10 +69,11 @@
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final BinaryDictReader reader = new BinaryDictReader(decodeSpec.mFile);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
- final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(reader,
+ final FusionDictionary resultDict = BinaryDictDecoder.readDictionaryBinary(reader,
null /* dict : an optional dictionary to add words to, or null */);
assertEquals("Dictionary can't be read back correctly",
- resultDict.findWordInTree(resultDict.mRoot, "foo").getFrequency(), TEST_FREQ);
+ FusionDictionary.findWordInTree(resultDict.mRootNodeArray, "foo").getFrequency(),
+ TEST_FREQ);
}
public void testGetRawDictFails() throws IOException {
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictInputOutputTest.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java
similarity index 79%
rename from tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictInputOutputTest.java
rename to tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java
index 0969028..fe67383 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictInputOutputTest.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java
@@ -17,7 +17,7 @@
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import junit.framework.TestCase;
@@ -25,13 +25,13 @@
import java.util.HashMap;
/**
- * Unit tests for BinaryDictInputOutput.
+ * Unit tests for BinaryDictEncoder.flattenTree().
*/
-public class BinaryDictInputOutputTest extends TestCase {
+public class BinaryDictEncoderFlattenTreeTests extends TestCase {
// Test the flattened array contains the expected number of nodes, and
// that it does not contain any duplicates.
public void testFlattenNodes() {
- final FusionDictionary dict = new FusionDictionary(new Node(),
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new DictionaryOptions(new HashMap<String, String>(),
false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */));
dict.add("foo", 1, null, false /* isNotAWord */);
@@ -39,10 +39,10 @@
dict.add("ftb", 1, null, false /* isNotAWord */);
dict.add("bar", 1, null, false /* isNotAWord */);
dict.add("fool", 1, null, false /* isNotAWord */);
- final ArrayList<Node> result = BinaryDictInputOutput.flattenTree(dict.mRoot);
+ final ArrayList<PtNodeArray> result = BinaryDictEncoder.flattenTree(dict.mRootNodeArray);
assertEquals(4, result.size());
while (!result.isEmpty()) {
- final Node n = result.remove(0);
+ final PtNodeArray n = result.remove(0);
assertFalse("Flattened array contained the same node twice", result.contains(n));
}
}
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
index 7607113..22c0ceb 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
@@ -19,7 +19,7 @@
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.Word;
import junit.framework.TestCase;
@@ -29,7 +29,7 @@
import java.util.Random;
/**
- * Unit tests for BinaryDictInputOutput.
+ * Unit tests for FusionDictionary.
*/
public class FusionDictionaryTest extends TestCase {
private static final ArrayList<String> sWords = new ArrayList<String>();
@@ -72,7 +72,7 @@
assertNotNull(dict);
for (final String word : words) {
if (--limit < 0) return;
- final CharGroup cg = FusionDictionary.findWordInTree(dict.mRoot, word);
+ final CharGroup cg = FusionDictionary.findWordInTree(dict.mRootNodeArray, word);
assertNotNull(cg);
}
}
@@ -95,7 +95,7 @@
// Test the flattened array contains the expected number of nodes, and
// that it does not contain any duplicates.
public void testFusion() {
- final FusionDictionary dict = new FusionDictionary(new Node(),
+ final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new DictionaryOptions(new HashMap<String, String>(),
false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */));
final long time = System.currentTimeMillis();
diff --git a/tools/dicttool/tests/etc/test-dicttool.sh b/tools/dicttool/tests/etc/test-dicttool.sh
index 0921207..5eb44fc 100755
--- a/tools/dicttool/tests/etc/test-dicttool.sh
+++ b/tools/dicttool/tests/etc/test-dicttool.sh
@@ -24,5 +24,5 @@
find out -name "dicttool_aosp*" -exec rm -rf {} \; > /dev/null 2>&1
mmm -j8 external/junit
DICTTOOL_UNITTEST=true mmm -j8 packages/inputmethods/LatinIME/tools/dicttool
-java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest
+java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests
java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.dicttool.BinaryDictOffdeviceUtilsTests