[Refactor] Divide BinaryDictInputOutput into BinaryDictEncoder and BinaryDictDecoder.
Change-Id: I7c3269d77e3e3b567e459dcaa1bc029903941744
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
index d0a4afd..7e497e9 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
@@ -21,7 +21,7 @@
import android.content.res.AssetFileDescriptor;
import android.util.Log;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.DictionaryInfoUtils;
@@ -231,8 +231,8 @@
try {
// Read the version of the file
inStream = new FileInputStream(f);
- final BinaryDictInputOutput.ByteBufferWrapper buffer =
- new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map(
+ final BinaryDictDecoder.ByteBufferWrapper buffer =
+ new BinaryDictDecoder.ByteBufferWrapper(inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, f.length()));
final int magic = buffer.readInt();
if (magic != FormatSpec.MAGIC_NUMBER) {
@@ -241,7 +241,7 @@
final int formatVersion = buffer.readInt();
final int headerSize = buffer.readInt();
final HashMap<String, String> options = CollectionUtils.newHashMap();
- BinaryDictInputOutput.populateOptions(buffer, headerSize, options);
+ BinaryDictDecoder.populateOptions(buffer, headerSize, options);
final String version = options.get(VERSION_KEY);
if (null == version) {
diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
index 47151bf..3b300a3 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
@@ -20,7 +20,7 @@
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
@@ -87,7 +87,7 @@
@Override
protected void writeBinaryDictionary(final FileOutputStream out)
throws IOException, UnsupportedFormatException {
- BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
+ BinaryDictEncoder.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
}
@Override
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java
new file mode 100644
index 0000000..767f4fc
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java
@@ -0,0 +1,819 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.makedict;
+
+import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
+import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
+import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
+import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+import com.android.inputmethod.latin.utils.JniUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Decodes binary files for a FusionDictionary.
+ *
+ * All the methods in this class are static.
+ */
+public final class BinaryDictDecoder {
+
+ private static final boolean DBG = MakedictLog.DBG;
+
+ static {
+ JniUtils.loadNativeLibrary();
+ }
+
+ // TODO: implement something sensical instead of just a phony method
+ private static native int doNothing();
+
+ private BinaryDictDecoder() {
+ // This utility class is not publicly instantiable.
+ }
+
+ private static final int MAX_JUMPS = 12;
+
+ @UsedForTesting
+ public interface FusionDictionaryBufferInterface {
+ public int readUnsignedByte();
+ public int readUnsignedShort();
+ public int readUnsignedInt24();
+ public int readInt();
+ public int position();
+ public void position(int newPosition);
+ public void put(final byte b);
+ public int limit();
+ @UsedForTesting
+ public int capacity();
+ }
+
+ public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
+ private ByteBuffer mBuffer;
+
+ public ByteBufferWrapper(final ByteBuffer buffer) {
+ mBuffer = buffer;
+ }
+
+ @Override
+ public int readUnsignedByte() {
+ return mBuffer.get() & 0xFF;
+ }
+
+ @Override
+ public int readUnsignedShort() {
+ return mBuffer.getShort() & 0xFFFF;
+ }
+
+ @Override
+ public int readUnsignedInt24() {
+ final int retval = readUnsignedByte();
+ return (retval << 16) + readUnsignedShort();
+ }
+
+ @Override
+ public int readInt() {
+ return mBuffer.getInt();
+ }
+
+ @Override
+ public int position() {
+ return mBuffer.position();
+ }
+
+ @Override
+ public void position(int newPos) {
+ mBuffer.position(newPos);
+ }
+
+ @Override
+ public void put(final byte b) {
+ mBuffer.put(b);
+ }
+
+ @Override
+ public int limit() {
+ return mBuffer.limit();
+ }
+
+ @Override
+ public int capacity() {
+ return mBuffer.capacity();
+ }
+ }
+
+ /**
+ * A class grouping utility function for our specific character encoding.
+ */
+ static final class CharEncoding {
+ private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
+ private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF;
+
+ /**
+ * Helper method to find out whether this code fits on one byte
+ */
+ private static boolean fitsOnOneByte(final int character) {
+ return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE
+ && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE;
+ }
+
+ /**
+ * Compute the size of a character given its character code.
+ *
+ * Char format is:
+ * 1 byte = bbbbbbbb match
+ * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
+ * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
+ * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
+ * 00011111 would be outside unicode.
+ * else: iso-latin-1 code
+ * This allows for the whole unicode range to be encoded, including chars outside of
+ * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
+ * characters which should never happen anyway (and still work, but take 3 bytes).
+ *
+ * @param character the character code.
+ * @return the size in binary encoded-form, either 1 or 3 bytes.
+ */
+ static int getCharSize(final int character) {
+ // See char encoding in FusionDictionary.java
+ if (fitsOnOneByte(character)) return 1;
+ if (FormatSpec.INVALID_CHARACTER == character) return 1;
+ return 3;
+ }
+
+ /**
+ * Compute the byte size of a character array.
+ */
+ static int getCharArraySize(final int[] chars) {
+ int size = 0;
+ for (int character : chars) size += getCharSize(character);
+ return size;
+ }
+
+ /**
+ * Writes a char array to a byte buffer.
+ *
+ * @param codePoints the code point array to write.
+ * @param buffer the byte buffer to write to.
+ * @param index the index in buffer to write the character array to.
+ * @return the index after the last character.
+ */
+ static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) {
+ for (int codePoint : codePoints) {
+ if (1 == getCharSize(codePoint)) {
+ buffer[index++] = (byte)codePoint;
+ } else {
+ buffer[index++] = (byte)(0xFF & (codePoint >> 16));
+ buffer[index++] = (byte)(0xFF & (codePoint >> 8));
+ buffer[index++] = (byte)(0xFF & codePoint);
+ }
+ }
+ return index;
+ }
+
+ /**
+ * Writes a string with our character format to a byte buffer.
+ *
+ * This will also write the terminator byte.
+ *
+ * @param buffer the byte buffer to write to.
+ * @param origin the offset to write from.
+ * @param word the string to write.
+ * @return the size written, in bytes.
+ */
+ static int writeString(final byte[] buffer, final int origin,
+ final String word) {
+ final int length = word.length();
+ int index = origin;
+ for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
+ final int codePoint = word.codePointAt(i);
+ if (1 == getCharSize(codePoint)) {
+ buffer[index++] = (byte)codePoint;
+ } else {
+ buffer[index++] = (byte)(0xFF & (codePoint >> 16));
+ buffer[index++] = (byte)(0xFF & (codePoint >> 8));
+ buffer[index++] = (byte)(0xFF & codePoint);
+ }
+ }
+ buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
+ return index - origin;
+ }
+
+ /**
+ * Writes a string with our character format to a ByteArrayOutputStream.
+ *
+ * This will also write the terminator byte.
+ *
+ * @param buffer the ByteArrayOutputStream to write to.
+ * @param word the string to write.
+ */
+ static void writeString(final ByteArrayOutputStream buffer, final String word) {
+ final int length = word.length();
+ for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
+ final int codePoint = word.codePointAt(i);
+ if (1 == getCharSize(codePoint)) {
+ buffer.write((byte) codePoint);
+ } else {
+ buffer.write((byte) (0xFF & (codePoint >> 16)));
+ buffer.write((byte) (0xFF & (codePoint >> 8)));
+ buffer.write((byte) (0xFF & codePoint));
+ }
+ }
+ buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR);
+ }
+
+ /**
+ * Reads a string from a buffer. This is the converse of the above method.
+ */
+ private static String readString(final FusionDictionaryBufferInterface buffer) {
+ final StringBuilder s = new StringBuilder();
+ int character = readChar(buffer);
+ while (character != FormatSpec.INVALID_CHARACTER) {
+ s.appendCodePoint(character);
+ character = readChar(buffer);
+ }
+ return s.toString();
+ }
+
+ /**
+ * Reads a character from the buffer.
+ *
+ * This follows the character format documented earlier in this source file.
+ *
+ * @param buffer the buffer, positioned over an encoded character.
+ * @return the character code.
+ */
+ static int readChar(final FusionDictionaryBufferInterface buffer) {
+ int character = buffer.readUnsignedByte();
+ if (!fitsOnOneByte(character)) {
+ if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) {
+ return FormatSpec.INVALID_CHARACTER;
+ }
+ character <<= 16;
+ character += buffer.readUnsignedShort();
+ }
+ return character;
+ }
+ }
+
+ // Input methods: Read a binary dictionary to memory.
+ // readDictionaryBinary is the public entry point for them.
+
+ static int readChildrenAddress(final FusionDictionaryBufferInterface buffer,
+ final int optionFlags, final FormatOptions options) {
+ if (options.mSupportsDynamicUpdate) {
+ final int address = buffer.readUnsignedInt24();
+ if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS;
+ if ((address & FormatSpec.MSB24) != 0) {
+ return -(address & FormatSpec.SINT24_MAX);
+ } else {
+ return address;
+ }
+ }
+ int address;
+ switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
+ return buffer.readUnsignedByte();
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
+ return buffer.readUnsignedShort();
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
+ return buffer.readUnsignedInt24();
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
+ default:
+ return FormatSpec.NO_CHILDREN_ADDRESS;
+ }
+ }
+
+ static int readParentAddress(final FusionDictionaryBufferInterface buffer,
+ final FormatOptions formatOptions) {
+ if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
+ final int parentAddress = buffer.readUnsignedInt24();
+ final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1;
+ return sign * (parentAddress & FormatSpec.SINT24_MAX);
+ } else {
+ return FormatSpec.NO_PARENT_ADDRESS;
+ }
+ }
+
+ private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH];
+ public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
+ final int originalGroupAddress, final FormatOptions options) {
+ int addressPointer = originalGroupAddress;
+ final int flags = buffer.readUnsignedByte();
+ ++addressPointer;
+
+ final int parentAddress = readParentAddress(buffer, options);
+ if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
+ addressPointer += 3;
+ }
+
+ final int characters[];
+ if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) {
+ int index = 0;
+ int character = CharEncoding.readChar(buffer);
+ addressPointer += CharEncoding.getCharSize(character);
+ while (-1 != character) {
+ // FusionDictionary is making sure that the length of the word is smaller than
+ // MAX_WORD_LENGTH.
+ // So we'll never write past the end of CHARACTER_BUFFER.
+ CHARACTER_BUFFER[index++] = character;
+ character = CharEncoding.readChar(buffer);
+ addressPointer += CharEncoding.getCharSize(character);
+ }
+ characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index);
+ } else {
+ final int character = CharEncoding.readChar(buffer);
+ addressPointer += CharEncoding.getCharSize(character);
+ characters = new int[] { character };
+ }
+ final int frequency;
+ if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) {
+ ++addressPointer;
+ frequency = buffer.readUnsignedByte();
+ } else {
+ frequency = CharGroup.NOT_A_TERMINAL;
+ }
+ int childrenAddress = readChildrenAddress(buffer, flags, options);
+ if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
+ childrenAddress += addressPointer;
+ }
+ addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options);
+ ArrayList<WeightedString> shortcutTargets = null;
+ if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) {
+ final int pointerBefore = buffer.position();
+ shortcutTargets = new ArrayList<WeightedString>();
+ buffer.readUnsignedShort(); // Skip the size
+ while (true) {
+ final int targetFlags = buffer.readUnsignedByte();
+ final String word = CharEncoding.readString(buffer);
+ shortcutTargets.add(new WeightedString(word,
+ targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY));
+ if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
+ }
+ addressPointer += buffer.position() - pointerBefore;
+ }
+ ArrayList<PendingAttribute> bigrams = null;
+ if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
+ bigrams = new ArrayList<PendingAttribute>();
+ int bigramCount = 0;
+ while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
+ final int bigramFlags = buffer.readUnsignedByte();
+ ++addressPointer;
+ final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE)
+ ? 1 : -1;
+ int bigramAddress = addressPointer;
+ switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) {
+ case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
+ bigramAddress += sign * buffer.readUnsignedByte();
+ addressPointer += 1;
+ break;
+ case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
+ bigramAddress += sign * buffer.readUnsignedShort();
+ addressPointer += 2;
+ break;
+ case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
+ final int offset = (buffer.readUnsignedByte() << 16)
+ + buffer.readUnsignedShort();
+ bigramAddress += sign * offset;
+ addressPointer += 3;
+ break;
+ default:
+ throw new RuntimeException("Has bigrams with no address");
+ }
+ bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY,
+ bigramAddress));
+ if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
+ }
+ if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
+ MakedictLog.d("too many bigrams in a group.");
+ }
+ }
+ return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency,
+ parentAddress, childrenAddress, shortcutTargets, bigrams);
+ }
+
+ /**
+ * Reads and returns the char group count out of a buffer and forwards the pointer.
+ */
+ public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
+ final int msb = buffer.readUnsignedByte();
+ if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) {
+ return msb;
+ } else {
+ return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8)
+ + buffer.readUnsignedByte();
+ }
+ }
+
+ // The word cache here is a stopgap bandaid to help the catastrophic performance
+ // of this method. Since it performs direct, unbuffered random access to the file and
+ // may be called hundreds of thousands of times, the resulting performance is not
+ // reasonable without some kind of cache. Thus:
+ private static TreeMap<Integer, WeightedString> wordCache =
+ new TreeMap<Integer, WeightedString>();
+ /**
+ * Finds, as a string, the word at the address passed as an argument.
+ *
+ * @param buffer the buffer to read from.
+ * @param headerSize the size of the header.
+ * @param address the address to seek.
+ * @param formatOptions file format options.
+ * @return the word with its frequency, as a weighted string.
+ */
+ /* package for tests */ static WeightedString getWordAtAddress(
+ final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
+ final FormatOptions formatOptions) {
+ final WeightedString cachedString = wordCache.get(address);
+ if (null != cachedString) return cachedString;
+
+ final WeightedString result;
+ final int originalPointer = buffer.position();
+ buffer.position(address);
+
+ if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
+ result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions);
+ } else {
+ result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address,
+ formatOptions);
+ }
+
+ wordCache.put(address, result);
+ buffer.position(originalPointer);
+ return result;
+ }
+
+ // TODO: static!? This will behave erratically when used in multi-threaded code.
+ // We need to fix this
+ private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH];
+ @SuppressWarnings("unused")
+ private static WeightedString getWordAtAddressWithParentAddress(
+ final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
+ final FormatOptions options) {
+ int currentAddress = address;
+ int index = FormatSpec.MAX_WORD_LENGTH - 1;
+ int frequency = Integer.MIN_VALUE;
+ // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH
+ for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) {
+ CharGroupInfo currentInfo;
+ int loopCounter = 0;
+ do {
+ buffer.position(currentAddress + headerSize);
+ currentInfo = readCharGroup(buffer, currentAddress, options);
+ if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) {
+ currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
+ }
+ if (DBG && loopCounter++ > MAX_JUMPS) {
+ MakedictLog.d("Too many jumps - probably a bug");
+ }
+ } while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options));
+ if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency;
+ for (int i = 0; i < currentInfo.mCharacters.length; ++i) {
+ sGetWordBuffer[index--] =
+ currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1];
+ }
+ if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break;
+ currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
+ }
+
+ return new WeightedString(
+ new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1),
+ frequency);
+ }
+
+ private static WeightedString getWordAtAddressWithoutParentAddress(
+ final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
+ final FormatOptions options) {
+ buffer.position(headerSize);
+ final int count = readCharGroupCount(buffer);
+ int groupOffset = BinaryDictIOUtils.getGroupCountSize(count);
+ final StringBuilder builder = new StringBuilder();
+ WeightedString result = null;
+
+ CharGroupInfo last = null;
+ for (int i = count - 1; i >= 0; --i) {
+ CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
+ groupOffset = info.mEndAddress;
+ if (info.mOriginalAddress == address) {
+ builder.append(new String(info.mCharacters, 0, info.mCharacters.length));
+ result = new WeightedString(builder.toString(), info.mFrequency);
+ break; // and return
+ }
+ if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
+ if (info.mChildrenAddress > address) {
+ if (null == last) continue;
+ builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
+ buffer.position(last.mChildrenAddress + headerSize);
+ i = readCharGroupCount(buffer);
+ groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
+ last = null;
+ continue;
+ }
+ last = info;
+ }
+ if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) {
+ builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
+ buffer.position(last.mChildrenAddress + headerSize);
+ i = readCharGroupCount(buffer);
+ groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
+ last = null;
+ continue;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Reads a single node from a buffer.
+ *
+ * This methods reads the file at the current position. A node is fully expected to start at
+ * the current position.
+ * This will recursively read other nodes into the structure, populating the reverse
+ * maps on the fly and using them to keep track of already read nodes.
+ *
+ * @param buffer the buffer, correctly positioned at the start of a node.
+ * @param headerSize the size, in bytes, of the file header.
+ * @param reverseNodeMap a mapping from addresses to already read nodes.
+ * @param reverseGroupMap a mapping from addresses to already read character groups.
+ * @param options file format options.
+ * @return the read node with all his children already read.
+ */
+ private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize,
+ final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap,
+ final FormatOptions options)
+ throws IOException {
+ final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>();
+ final int nodeOrigin = buffer.position() - headerSize;
+
+ do { // Scan the linked-list node.
+ final int nodeHeadPosition = buffer.position() - headerSize;
+ final int count = readCharGroupCount(buffer);
+ int groupOffset = nodeHeadPosition + BinaryDictIOUtils.getGroupCountSize(count);
+ for (int i = count; i > 0; --i) { // Scan the array of CharGroup.
+ CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
+ if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue;
+ ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets;
+ ArrayList<WeightedString> bigrams = null;
+ if (null != info.mBigrams) {
+ bigrams = new ArrayList<WeightedString>();
+ for (PendingAttribute bigram : info.mBigrams) {
+ final WeightedString word = getWordAtAddress(
+ buffer, headerSize, bigram.mAddress, options);
+ final int reconstructedFrequency =
+ reconstructBigramFrequency(word.mFrequency, bigram.mFrequency);
+ bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
+ }
+ }
+ if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
+ Node children = reverseNodeMap.get(info.mChildrenAddress);
+ if (null == children) {
+ final int currentPosition = buffer.position();
+ buffer.position(info.mChildrenAddress + headerSize);
+ children = readNode(
+ buffer, headerSize, reverseNodeMap, reverseGroupMap, options);
+ buffer.position(currentPosition);
+ }
+ nodeContents.add(
+ new CharGroup(info.mCharacters, shortcutTargets, bigrams,
+ info.mFrequency,
+ 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
+ 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children));
+ } else {
+ nodeContents.add(
+ new CharGroup(info.mCharacters, shortcutTargets, bigrams,
+ info.mFrequency,
+ 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
+ 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED)));
+ }
+ groupOffset = info.mEndAddress;
+ }
+
+ // reach the end of the array.
+ if (options.mSupportsDynamicUpdate) {
+ final int nextAddress = buffer.readUnsignedInt24();
+ if (nextAddress >= 0 && nextAddress < buffer.limit()) {
+ buffer.position(nextAddress);
+ } else {
+ break;
+ }
+ }
+ } while (options.mSupportsDynamicUpdate &&
+ buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS);
+
+ final Node node = new Node(nodeContents);
+ node.mCachedAddressBeforeUpdate = nodeOrigin;
+ node.mCachedAddressAfterUpdate = nodeOrigin;
+ reverseNodeMap.put(node.mCachedAddressAfterUpdate, node);
+ return node;
+ }
+
+ /**
+ * Helper function to get the binary format version from the header.
+ * @throws IOException
+ */
+ private static int getFormatVersion(final FusionDictionaryBufferInterface buffer)
+ throws IOException {
+ final int magic = buffer.readInt();
+ if (FormatSpec.MAGIC_NUMBER == magic) return buffer.readUnsignedShort();
+ return FormatSpec.NOT_A_VERSION_NUMBER;
+ }
+
+ /**
+ * Helper function to get and validate the binary format version.
+ * @throws UnsupportedFormatException
+ * @throws IOException
+ */
+ private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer)
+ throws IOException, UnsupportedFormatException {
+ final int version = getFormatVersion(buffer);
+ if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
+ || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
+ throw new UnsupportedFormatException("This file has version " + version
+ + ", but this implementation does not support versions above "
+ + FormatSpec.MAXIMUM_SUPPORTED_VERSION);
+ }
+ return version;
+ }
+
+ /**
+ * Reads a header from a buffer.
+ * @param buffer the buffer to read.
+ * @throws IOException
+ * @throws UnsupportedFormatException
+ */
+ public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer)
+ throws IOException, UnsupportedFormatException {
+ final int version = checkFormatVersion(buffer);
+ final int optionsFlags = buffer.readUnsignedShort();
+
+ final HashMap<String, String> attributes = new HashMap<String, String>();
+ final int headerSize;
+ headerSize = buffer.readInt();
+
+ if (headerSize < 0) {
+ throw new UnsupportedFormatException("header size can't be negative.");
+ }
+
+ populateOptions(buffer, headerSize, attributes);
+ buffer.position(headerSize);
+
+ final FileHeader header = new FileHeader(headerSize,
+ new FusionDictionary.DictionaryOptions(attributes,
+ 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG),
+ 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)),
+ new FormatOptions(version,
+ 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE)));
+ return header;
+ }
+
+ /**
+ * Reads options from a buffer and populate a map with their contents.
+ *
+ * The buffer is read at the current position, so the caller must take care the pointer
+ * is in the right place before calling this.
+ */
+ public static void populateOptions(final FusionDictionaryBufferInterface buffer,
+ final int headerSize, final HashMap<String, String> options) {
+ while (buffer.position() < headerSize) {
+ final String key = CharEncoding.readString(buffer);
+ final String value = CharEncoding.readString(buffer);
+ options.put(key, value);
+ }
+ }
+
+ /**
+ * Reads a buffer and returns the memory representation of the dictionary.
+ *
+ * This high-level method takes a buffer and reads its contents, populating a
+ * FusionDictionary structure. The optional dict argument is an existing dictionary to
+ * which words from the buffer should be added. If it is null, a new dictionary is created.
+ *
+ * @param reader the reader.
+ * @param dict an optional dictionary to add words to, or null.
+ * @return the created (or merged) dictionary.
+ */
+ @UsedForTesting
+ public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
+ final FusionDictionary dict) throws FileNotFoundException, IOException,
+ UnsupportedFormatException {
+ // clear cache
+ wordCache.clear();
+
+ // if the buffer has not been opened, open the buffer with bytebuffer.
+ if (reader.getBuffer() == null) reader.openBuffer(
+ new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
+ if (reader.getBuffer() == null) {
+ MakedictLog.e("Cannot open the buffer");
+ }
+
+ // Read header
+ final FileHeader header = readHeader(reader.getBuffer());
+
+ Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
+ Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
+ final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
+ reverseGroupMapping, header.mFormatOptions);
+
+ FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);
+ if (null != dict) {
+ for (final Word w : dict) {
+ if (w.mIsBlacklistEntry) {
+ newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord);
+ } else {
+ newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord);
+ }
+ }
+ for (final Word w : dict) {
+ // By construction a binary dictionary may not have bigrams pointing to
+ // words that are not also registered as unigrams so we don't have to avoid
+ // them explicitly here.
+ for (final WeightedString bigram : w.mBigrams) {
+ newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency);
+ }
+ }
+ }
+
+ return newDict;
+ }
+
+ /**
+ * Helper method to pass a file name instead of a File object to isBinaryDictionary.
+ */
+ public static boolean isBinaryDictionary(final String filename) {
+ final File file = new File(filename);
+ return isBinaryDictionary(file);
+ }
+
+ /**
+ * Basic test to find out whether the file is a binary dictionary or not.
+ *
+ * Concretely this only tests the magic number.
+ *
+ * @param file The file to test.
+ * @return true if it's a binary dictionary, false otherwise
+ */
+ public static boolean isBinaryDictionary(final File file) {
+ FileInputStream inStream = null;
+ try {
+ inStream = new FileInputStream(file);
+ final ByteBuffer buffer = inStream.getChannel().map(
+ FileChannel.MapMode.READ_ONLY, 0, file.length());
+ final int version = getFormatVersion(new ByteBufferWrapper(buffer));
+ return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION
+ && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION);
+ } catch (FileNotFoundException e) {
+ return false;
+ } catch (IOException e) {
+ return false;
+ } finally {
+ if (inStream != null) {
+ try {
+ inStream.close();
+ } catch (IOException e) {
+ // do nothing
+ }
+ }
+ }
+ }
+
+ /**
+ * Calculate bigram frequency from compressed value
+ *
+ * @see #makeBigramFlags
+ *
+ * @param unigramFrequency
+ * @param bigramFrequency compressed frequency
+ * @return approximate bigram frequency
+ */
+ public static int reconstructBigramFrequency(final int unigramFrequency,
+ final int bigramFrequency) {
+ final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
+ / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
+ final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f);
+ return (int)resultFreqFloat;
+ }
+}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java
new file mode 100644
index 0000000..85219e4
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java
@@ -0,0 +1,993 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.makedict;
+
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.CharEncoding;
+import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
+import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
+import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
+import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+/**
+ * Encodes binary files for a FusionDictionary.
+ *
+ * All the methods in this class are static.
+ */
+public class BinaryDictEncoder {
+
+ private static final boolean DBG = MakedictLog.DBG;
+
+ private BinaryDictEncoder() {
+ // This utility class is not publicly instantiable.
+ }
+
+ // Arbitrary limit to how much passes we consider address size compression should
+ // terminate in. At the time of this writing, our largest dictionary completes
+ // compression in five passes.
+ // If the number of passes exceeds this number, makedict bails with an exception on
+ // suspicion that a bug might be causing an infinite loop.
+ private static final int MAX_PASSES = 24;
+
+ /**
+ * Compute the binary size of the character array.
+ *
+ * If only one character, this is the size of this character. If many, it's the sum of their
+ * sizes + 1 byte for the terminator.
+ *
+ * @param characters the character array
+ * @return the size of the char array, including the terminator if any
+ */
+ static int getGroupCharactersSize(final int[] characters) {
+ int size = CharEncoding.getCharArraySize(characters);
+ if (characters.length > 1) size += FormatSpec.GROUP_TERMINATOR_SIZE;
+ return size;
+ }
+
+ /**
+ * Compute the binary size of the character array in a group
+ *
+ * If only one character, this is the size of this character. If many, it's the sum of their
+ * sizes + 1 byte for the terminator.
+ *
+ * @param group the group
+ * @return the size of the char array, including the terminator if any
+ */
+ private static int getGroupCharactersSize(final CharGroup group) {
+ return getGroupCharactersSize(group.mChars);
+ }
+
+ /**
+ * Compute the binary size of the group count for a node
+ * @param node the node
+ * @return the size of the group count, either 1 or 2 bytes.
+ */
+ private static int getGroupCountSize(final Node node) {
+ return BinaryDictIOUtils.getGroupCountSize(node.mData.size());
+ }
+
+ /**
+ * Compute the size of a shortcut in bytes.
+ */
+ private static int getShortcutSize(final WeightedString shortcut) {
+ int size = FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE;
+ final String word = shortcut.mWord;
+ final int length = word.length();
+ for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
+ final int codePoint = word.codePointAt(i);
+ size += CharEncoding.getCharSize(codePoint);
+ }
+ size += FormatSpec.GROUP_TERMINATOR_SIZE;
+ return size;
+ }
+
+ /**
+ * Compute the size of a shortcut list in bytes.
+ *
+ * This is known in advance and does not change according to position in the file
+ * like address lists do.
+ */
+ static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) {
+ if (null == shortcutList) return 0;
+ int size = FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
+ for (final WeightedString shortcut : shortcutList) {
+ size += getShortcutSize(shortcut);
+ }
+ return size;
+ }
+
+ /**
+ * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything.
+ *
+ * @param group the CharGroup to compute the size of.
+ * @param options file format options.
+ * @return the maximum size of the group.
+ */
+ private static int getCharGroupMaximumSize(final CharGroup group, final FormatOptions options) {
+ int size = getGroupHeaderSize(group, options);
+ // If terminal, one byte for the frequency
+ if (group.isTerminal()) size += FormatSpec.GROUP_FREQUENCY_SIZE;
+ size += FormatSpec.GROUP_MAX_ADDRESS_SIZE; // For children address
+ size += getShortcutListSize(group.mShortcutTargets);
+ if (null != group.mBigrams) {
+ size += (FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE
+ + FormatSpec.GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE)
+ * group.mBigrams.size();
+ }
+ return size;
+ }
+
+ /**
+ * Compute the maximum size of a node, assuming 3-byte addresses for everything, and caches
+ * it in the 'actualSize' member of the node.
+ *
+ * @param node the node to compute the maximum size of.
+ * @param options file format options.
+ */
+ private static void calculateNodeMaximumSize(final Node node, final FormatOptions options) {
+ int size = getGroupCountSize(node);
+ for (CharGroup g : node.mData) {
+ final int groupSize = getCharGroupMaximumSize(g, options);
+ g.mCachedSize = groupSize;
+ size += groupSize;
+ }
+ if (options.mSupportsDynamicUpdate) {
+ size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
+ }
+ node.mCachedSize = size;
+ }
+
+ /**
+ * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup.
+ *
+ * @param group the group of which to compute the size of the header
+ * @param options file format options.
+ */
+ private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) {
+ if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
+ return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
+ + getGroupCharactersSize(group);
+ } else {
+ return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group);
+ }
+ }
+
+ /**
+ * Compute the size, in bytes, that an address will occupy.
+ *
+ * This can be used either for children addresses (which are always positive) or for
+ * attribute, which may be positive or negative but
+ * store their sign bit separately.
+ *
+ * @param address the address
+ * @return the byte size.
+ */
+ static int getByteSize(final int address) {
+ assert(address <= FormatSpec.UINT24_MAX);
+ if (!BinaryDictIOUtils.hasChildrenAddress(address)) {
+ return 0;
+ } else if (Math.abs(address) <= FormatSpec.UINT8_MAX) {
+ return 1;
+ } else if (Math.abs(address) <= FormatSpec.UINT16_MAX) {
+ return 2;
+ } else {
+ return 3;
+ }
+ }
+
+ // End utility methods
+
+ // This method is responsible for finding a nice ordering of the nodes that favors run-time
+ // cache performance and dictionary size.
+ /* package for tests */ static ArrayList<Node> flattenTree(final Node root) {
+ final int treeSize = FusionDictionary.countCharGroups(root);
+ MakedictLog.i("Counted nodes : " + treeSize);
+ final ArrayList<Node> flatTree = new ArrayList<Node>(treeSize);
+ return flattenTreeInner(flatTree, root);
+ }
+
+ private static ArrayList<Node> flattenTreeInner(final ArrayList<Node> list, final Node node) {
+ // Removing the node is necessary if the tails are merged, because we would then
+ // add the same node several times when we only want it once. A number of places in
+ // the code also depends on any node being only once in the list.
+ // Merging tails can only be done if there are no attributes. Searching for attributes
+ // in LatinIME code depends on a total breadth-first ordering, which merging tails
+ // breaks. If there are no attributes, it should be fine (and reduce the file size)
+ // to merge tails, and removing the node from the list would be necessary. However,
+ // we don't merge tails because breaking the breadth-first ordering would result in
+ // extreme overhead at bigram lookup time (it would make the search function O(n) instead
+ // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty
+ // high).
+ // If no nodes are ever merged, we can't have the same node twice in the list, hence
+ // searching for duplicates in unnecessary. It is also very performance consuming,
+ // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making
+ // this simple list.remove operation O(n*n) overall. On Android this overhead is very
+ // high.
+ // For future reference, the code to remove duplicate is a simple : list.remove(node);
+ list.add(node);
+ final ArrayList<CharGroup> branches = node.mData;
+ final int nodeSize = branches.size();
+ for (CharGroup group : branches) {
+ if (null != group.mChildren) flattenTreeInner(list, group.mChildren);
+ }
+ return list;
+ }
+
+ /**
+ * Get the offset from a position inside a current node to a target node, during update.
+ *
+ * If the current node is before the target node, the target node has not been updated yet,
+ * so we should return the offset from the old position of the current node to the old position
+ * of the target node. If on the other hand the target is before the current node, it already
+ * has been updated, so we should return the offset from the new position in the current node
+ * to the new position in the target node.
+ * @param currentNode the node containing the CharGroup where the offset will be written
+ * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode
+ * @param targetNode the target node to get the offset to
+ * @return the offset to the target node
+ */
+ private static int getOffsetToTargetNodeDuringUpdate(final Node currentNode,
+ final int offsetFromStartOfCurrentNode, final Node targetNode) {
+ final boolean isTargetBeforeCurrent = (targetNode.mCachedAddressBeforeUpdate
+ < currentNode.mCachedAddressBeforeUpdate);
+ if (isTargetBeforeCurrent) {
+ return targetNode.mCachedAddressAfterUpdate
+ - (currentNode.mCachedAddressAfterUpdate + offsetFromStartOfCurrentNode);
+ } else {
+ return targetNode.mCachedAddressBeforeUpdate
+ - (currentNode.mCachedAddressBeforeUpdate + offsetFromStartOfCurrentNode);
+ }
+ }
+
+ /**
+ * Get the offset from a position inside a current node to a target CharGroup, during update.
+ * @param currentNode the node containing the CharGroup where the offset will be written
+ * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode
+ * @param targetCharGroup the target CharGroup to get the offset to
+ * @return the offset to the target CharGroup
+ */
+ // TODO: is there any way to factorize this method with the one above?
+ private static int getOffsetToTargetCharGroupDuringUpdate(final Node currentNode,
+ final int offsetFromStartOfCurrentNode, final CharGroup targetCharGroup) {
+ final int oldOffsetBasePoint = currentNode.mCachedAddressBeforeUpdate
+ + offsetFromStartOfCurrentNode;
+ final boolean isTargetBeforeCurrent = (targetCharGroup.mCachedAddressBeforeUpdate
+ < oldOffsetBasePoint);
+ // If the target is before the current node, then its address has already been updated.
+ // We can use the AfterUpdate member, and compare it to our own member after update.
+ // Otherwise, the AfterUpdate member is not updated yet, so we need to use the BeforeUpdate
+ // member, and of course we have to compare this to our own address before update.
+ if (isTargetBeforeCurrent) {
+ final int newOffsetBasePoint = currentNode.mCachedAddressAfterUpdate
+ + offsetFromStartOfCurrentNode;
+ return targetCharGroup.mCachedAddressAfterUpdate - newOffsetBasePoint;
+ } else {
+ return targetCharGroup.mCachedAddressBeforeUpdate - oldOffsetBasePoint;
+ }
+ }
+
+ /**
+ * Computes the actual node size, based on the cached addresses of the children nodes.
+ *
+ * Each node stores its tentative address. During dictionary address computing, these
+ * are not final, but they can be used to compute the node size (the node size depends
+ * on the address of the children because the number of bytes necessary to store an
+ * address depends on its numeric value. The return value indicates whether the node
+ * contents (as in, any of the addresses stored in the cache fields) have changed with
+ * respect to their previous value.
+ *
+ * @param node the node to compute the size of.
+ * @param dict the dictionary in which the word/attributes are to be found.
+ * @param formatOptions file format options.
+ * @return false if none of the cached addresses inside the node changed, true otherwise.
+ */
+ private static boolean computeActualNodeSize(final Node node, final FusionDictionary dict,
+ final FormatOptions formatOptions) {
+ boolean changed = false;
+ int size = getGroupCountSize(node);
+ for (CharGroup group : node.mData) {
+ group.mCachedAddressAfterUpdate = node.mCachedAddressAfterUpdate + size;
+ if (group.mCachedAddressAfterUpdate != group.mCachedAddressBeforeUpdate) {
+ changed = true;
+ }
+ int groupSize = getGroupHeaderSize(group, formatOptions);
+ if (group.isTerminal()) groupSize += FormatSpec.GROUP_FREQUENCY_SIZE;
+ if (null == group.mChildren && formatOptions.mSupportsDynamicUpdate) {
+ groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
+ } else if (null != group.mChildren) {
+ if (formatOptions.mSupportsDynamicUpdate) {
+ groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
+ } else {
+ groupSize += getByteSize(getOffsetToTargetNodeDuringUpdate(node,
+ groupSize + size, group.mChildren));
+ }
+ }
+ groupSize += getShortcutListSize(group.mShortcutTargets);
+ if (null != group.mBigrams) {
+ for (WeightedString bigram : group.mBigrams) {
+ final int offset = getOffsetToTargetCharGroupDuringUpdate(node,
+ groupSize + size + FormatSpec.GROUP_FLAGS_SIZE,
+ FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord));
+ groupSize += getByteSize(offset) + FormatSpec.GROUP_FLAGS_SIZE;
+ }
+ }
+ group.mCachedSize = groupSize;
+ size += groupSize;
+ }
+ if (formatOptions.mSupportsDynamicUpdate) {
+ size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
+ }
+ if (node.mCachedSize != size) {
+ node.mCachedSize = size;
+ changed = true;
+ }
+ return changed;
+ }
+
+ /**
+ * Initializes the cached addresses of nodes from their size.
+ *
+ * @param flatNodes the array of nodes.
+ * @param formatOptions file format options.
+ * @return the byte size of the entire stack.
+ */
+ private static int initializeNodesCachedAddresses(final ArrayList<Node> flatNodes,
+ final FormatOptions formatOptions) {
+ int nodeOffset = 0;
+ for (final Node n : flatNodes) {
+ n.mCachedAddressBeforeUpdate = nodeOffset;
+ int groupCountSize = getGroupCountSize(n);
+ int groupOffset = 0;
+ for (final CharGroup g : n.mData) {
+ g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate =
+ groupCountSize + nodeOffset + groupOffset;
+ groupOffset += g.mCachedSize;
+ }
+ final int nodeSize = groupCountSize + groupOffset
+ + (formatOptions.mSupportsDynamicUpdate
+ ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0);
+ nodeOffset += n.mCachedSize;
+ }
+ return nodeOffset;
+ }
+
+ /**
+ * Updates the cached addresses of nodes after recomputing their new positions.
+ *
+ * @param flatNodes the array of nodes.
+ */
+ private static void updateNodeCachedAddresses(final ArrayList<Node> flatNodes) {
+ for (final Node n : flatNodes) {
+ n.mCachedAddressBeforeUpdate = n.mCachedAddressAfterUpdate;
+ for (final CharGroup g : n.mData) {
+ g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate;
+ }
+ }
+ }
+
+ /**
+ * Compute the cached parent addresses after all has been updated.
+ *
+ * The parent addresses are used by some binary formats at write-to-disk time. Not all formats
+ * need them. In particular, version 2 does not need them, and version 3 does.
+ *
+ * @param flatNodes the flat array of nodes to fill in
+ */
+ private static void computeParentAddresses(final ArrayList<Node> flatNodes) {
+ for (final Node node : flatNodes) {
+ for (final CharGroup group : node.mData) {
+ if (null != group.mChildren) {
+ // Assign my address to children's parent address
+ // Here BeforeUpdate and AfterUpdate addresses have the same value, so it
+ // does not matter which we use.
+ group.mChildren.mCachedParentAddress = group.mCachedAddressAfterUpdate
+ - group.mChildren.mCachedAddressAfterUpdate;
+ }
+ }
+ }
+ }
+
+ /**
+ * Compute the addresses and sizes of an ordered node array.
+ *
+ * This method takes a node array and will update its cached address and size values
+ * so that they can be written into a file. It determines the smallest size each of the
+ * nodes can be given the addresses of its children and attributes, and store that into
+ * each node.
+ * The order of the node is given by the order of the array. This method makes no effort
+ * to find a good order; it only mechanically computes the size this order results in.
+ *
+ * @param dict the dictionary
+ * @param flatNodes the ordered array of nodes
+ * @param formatOptions file format options.
+ * @return the same array it was passed. The nodes have been updated for address and size.
+ */
+ private static ArrayList<Node> computeAddresses(final FusionDictionary dict,
+ final ArrayList<Node> flatNodes, final FormatOptions formatOptions) {
+ // First get the worst possible sizes and offsets
+ for (final Node n : flatNodes) calculateNodeMaximumSize(n, formatOptions);
+ final int offset = initializeNodesCachedAddresses(flatNodes, formatOptions);
+
+ MakedictLog.i("Compressing the array addresses. Original size : " + offset);
+ MakedictLog.i("(Recursively seen size : " + offset + ")");
+
+ int passes = 0;
+ boolean changesDone = false;
+ do {
+ changesDone = false;
+ int nodeStartOffset = 0;
+ for (final Node n : flatNodes) {
+ n.mCachedAddressAfterUpdate = nodeStartOffset;
+ final int oldNodeSize = n.mCachedSize;
+ final boolean changed = computeActualNodeSize(n, dict, formatOptions);
+ final int newNodeSize = n.mCachedSize;
+ if (oldNodeSize < newNodeSize) throw new RuntimeException("Increased size ?!");
+ nodeStartOffset += newNodeSize;
+ changesDone |= changed;
+ }
+ updateNodeCachedAddresses(flatNodes);
+ ++passes;
+ if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug");
+ } while (changesDone);
+
+ if (formatOptions.mSupportsDynamicUpdate) {
+ computeParentAddresses(flatNodes);
+ }
+ final Node lastNode = flatNodes.get(flatNodes.size() - 1);
+ MakedictLog.i("Compression complete in " + passes + " passes.");
+ MakedictLog.i("After address compression : "
+ + (lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize));
+
+ return flatNodes;
+ }
+
+ /**
+ * Sanity-checking method.
+ *
+ * This method checks an array of node for juxtaposition, that is, it will do
+ * nothing if each node's cached address is actually the previous node's address
+ * plus the previous node's size.
+ * If this is not the case, it will throw an exception.
+ *
+ * @param array the array node to check
+ */
+ private static void checkFlatNodeArray(final ArrayList<Node> array) {
+ int offset = 0;
+ int index = 0;
+ for (final Node n : array) {
+ // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter
+ // which we use.
+ if (n.mCachedAddressAfterUpdate != offset) {
+ throw new RuntimeException("Wrong address for node " + index
+ + " : expected " + offset + ", got " + n.mCachedAddressAfterUpdate);
+ }
+ ++index;
+ offset += n.mCachedSize;
+ }
+ }
+
+ /**
+ * Helper method to write a variable-size address to a file.
+ *
+ * @param buffer the buffer to write to.
+ * @param index the index in the buffer to write the address to.
+ * @param address the address to write.
+ * @return the size in bytes the address actually took.
+ */
+ private static int writeVariableAddress(final byte[] buffer, int index, final int address) {
+ switch (getByteSize(address)) {
+ case 1:
+ buffer[index++] = (byte)address;
+ return 1;
+ case 2:
+ buffer[index++] = (byte)(0xFF & (address >> 8));
+ buffer[index++] = (byte)(0xFF & address);
+ return 2;
+ case 3:
+ buffer[index++] = (byte)(0xFF & (address >> 16));
+ buffer[index++] = (byte)(0xFF & (address >> 8));
+ buffer[index++] = (byte)(0xFF & address);
+ return 3;
+ case 0:
+ return 0;
+ default:
+ throw new RuntimeException("Address " + address + " has a strange size");
+ }
+ }
+
+ /**
+ * Helper method to write a variable-size signed address to a file.
+ *
+ * @param buffer the buffer to write to.
+ * @param index the index in the buffer to write the address to.
+ * @param address the address to write.
+ * @return the size in bytes the address actually took.
+ */
+ private static int writeVariableSignedAddress(final byte[] buffer, int index,
+ final int address) {
+ if (!BinaryDictIOUtils.hasChildrenAddress(address)) {
+ buffer[index] = buffer[index + 1] = buffer[index + 2] = 0;
+ } else {
+ final int absAddress = Math.abs(address);
+ buffer[index++] =
+ (byte)((address < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absAddress >> 16)));
+ buffer[index++] = (byte)(0xFF & (absAddress >> 8));
+ buffer[index++] = (byte)(0xFF & absAddress);
+ }
+ return 3;
+ }
+
+ /**
+ * Makes the flag value for a char group.
+ *
+ * @param hasMultipleChars whether the group has multiple chars.
+ * @param isTerminal whether the group is terminal.
+ * @param childrenAddressSize the size of a children address.
+ * @param hasShortcuts whether the group has shortcuts.
+ * @param hasBigrams whether the group has bigrams.
+ * @param isNotAWord whether the group is not a word.
+ * @param isBlackListEntry whether the group is a blacklist entry.
+ * @param formatOptions file format options.
+ * @return the flags
+ */
+ static int makeCharGroupFlags(final boolean hasMultipleChars, final boolean isTerminal,
+ final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams,
+ final boolean isNotAWord, final boolean isBlackListEntry,
+ final FormatOptions formatOptions) {
+ byte flags = 0;
+ if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS;
+ if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL;
+ if (formatOptions.mSupportsDynamicUpdate) {
+ flags |= FormatSpec.FLAG_IS_NOT_MOVED;
+ } else if (true) {
+ switch (childrenAddressSize) {
+ case 1:
+ flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE;
+ break;
+ case 2:
+ flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES;
+ break;
+ case 3:
+ flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES;
+ break;
+ case 0:
+ flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS;
+ break;
+ default:
+ throw new RuntimeException("Node with a strange address");
+ }
+ }
+ if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS;
+ if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS;
+ if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD;
+ if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED;
+ return flags;
+ }
+
+ private static byte makeCharGroupFlags(final CharGroup group, final int groupAddress,
+ final int childrenOffset, final FormatOptions formatOptions) {
+ return (byte) makeCharGroupFlags(group.mChars.length > 1, group.mFrequency >= 0,
+ getByteSize(childrenOffset), group.mShortcutTargets != null, group.mBigrams != null,
+ group.mIsNotAWord, group.mIsBlacklistEntry, formatOptions);
+ }
+
+ /**
+ * Makes the flag value for a bigram.
+ *
+ * @param more whether there are more bigrams after this one.
+ * @param offset the offset of the bigram.
+ * @param bigramFrequency the frequency of the bigram, 0..255.
+ * @param unigramFrequency the unigram frequency of the same word, 0..255.
+ * @param word the second bigram, for debugging purposes
+ * @return the flags
+ */
+ private static final int makeBigramFlags(final boolean more, final int offset,
+ int bigramFrequency, final int unigramFrequency, final String word) {
+ int bigramFlags = (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0)
+ + (offset < 0 ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0);
+ switch (getByteSize(offset)) {
+ case 1:
+ bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
+ break;
+ case 2:
+ bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES;
+ break;
+ case 3:
+ bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES;
+ break;
+ default:
+ throw new RuntimeException("Strange offset size");
+ }
+ if (unigramFrequency > bigramFrequency) {
+ MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word
+ + "\". Bigram freq is " + bigramFrequency + ", unigram freq for "
+ + word + " is " + unigramFrequency);
+ bigramFrequency = unigramFrequency;
+ }
+ // We compute the difference between 255 (which means probability = 1) and the
+ // unigram score. We split this into a number of discrete steps.
+ // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15
+ // represents an increase of 16 steps: a value of 15 will be interpreted as the median
+ // value of the 16th step. In all justice, if the bigram frequency is low enough to be
+ // rounded below the first step (which means it is less than half a step higher than the
+ // unigram frequency) then the unigram frequency itself is the best approximation of the
+ // bigram freq that we could possibly supply, hence we should *not* include this bigram
+ // in the file at all.
+ // until this is done, we'll write 0 and slightly overestimate this case.
+ // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step
+ // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to
+ // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the
+ // step size. Then we compute the start of the first step (the one where value 0 starts)
+ // by adding half-a-step to the unigramFrequency. From there, we compute the integer
+ // number of steps to the bigramFrequency. One last thing: we want our steps to include
+ // their lower bound and exclude their higher bound so we need to have the first step
+ // start at exactly 1 unit higher than floor(unigramFreq + half a step).
+ // Note : to reconstruct the score, the dictionary reader will need to divide
+ // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step,
+ // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best
+ // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the
+ // step pointed by the discretized frequency.
+ final float stepSize =
+ (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
+ / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
+ final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f);
+ final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize);
+ // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1
+ // here. The best approximation would be the unigram freq itself, so we should not
+ // include this bigram in the dictionary. For now, register as 0, and live with the
+ // small over-estimation that we get in this case. TODO: actually remove this bigram
+ // if discretizedFrequency < 0.
+ final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0;
+ bigramFlags += finalBigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY;
+ return bigramFlags;
+ }
+
+ /**
+ * Makes the 2-byte value for options flags.
+ */
+ private static final int makeOptionsValue(final FusionDictionary dictionary,
+ final FormatOptions formatOptions) {
+ final DictionaryOptions options = dictionary.mOptions;
+ final boolean hasBigrams = dictionary.hasBigrams();
+ return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0)
+ + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0)
+ + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0)
+ + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0);
+ }
+
+ /**
+ * Makes the flag value for a shortcut.
+ *
+ * @param more whether there are more attributes after this one.
+ * @param frequency the frequency of the attribute, 0..15
+ * @return the flags
+ */
+ static final int makeShortcutFlags(final boolean more, final int frequency) {
+ return (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0)
+ + (frequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY);
+ }
+
+ private static final int writeParentAddress(final byte[] buffer, final int index,
+ final int address, final FormatOptions formatOptions) {
+ if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
+ if (address == FormatSpec.NO_PARENT_ADDRESS) {
+ buffer[index] = buffer[index + 1] = buffer[index + 2] = 0;
+ } else {
+ final int absAddress = Math.abs(address);
+ assert(absAddress <= FormatSpec.SINT24_MAX);
+ buffer[index] = (byte)((address < 0 ? FormatSpec.MSB8 : 0)
+ | ((absAddress >> 16) & 0xFF));
+ buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF);
+ buffer[index + 2] = (byte)(absAddress & 0xFF);
+ }
+ return index + 3;
+ } else {
+ return index;
+ }
+ }
+
+ /**
+ * Write a node to memory. The node is expected to have its final position cached.
+ *
+ * This can be an empty map, but the more is inside the faster the lookups will be. It can
+ * be carried on as long as nodes do not move.
+ *
+ * @param dict the dictionary the node is a part of (for relative offsets).
+ * @param buffer the memory buffer to write to.
+ * @param node the node to write.
+ * @param formatOptions file format options.
+ * @return the address of the END of the node.
+ */
+ @SuppressWarnings("unused")
+ private static int writePlacedNode(final FusionDictionary dict, byte[] buffer,
+ final Node node, final FormatOptions formatOptions) {
+ // TODO: Make the code in common with BinaryDictIOUtils#writeCharGroup
+ int index = node.mCachedAddressAfterUpdate;
+
+ final int groupCount = node.mData.size();
+ final int countSize = getGroupCountSize(node);
+ final int parentAddress = node.mCachedParentAddress;
+ if (1 == countSize) {
+ buffer[index++] = (byte)groupCount;
+ } else if (2 == countSize) {
+ // We need to signal 2-byte size by setting the top bit of the MSB to 1, so
+ // we | 0x80 to do this.
+ buffer[index++] = (byte)((groupCount >> 8) | 0x80);
+ buffer[index++] = (byte)(groupCount & 0xFF);
+ } else {
+ throw new RuntimeException("Strange size from getGroupCountSize : " + countSize);
+ }
+ int groupAddress = index;
+ for (int i = 0; i < groupCount; ++i) {
+ final CharGroup group = node.mData.get(i);
+ if (index != group.mCachedAddressAfterUpdate) {
+ throw new RuntimeException("Bug: write index is not the same as the cached address "
+ + "of the group : " + index + " <> " + group.mCachedAddressAfterUpdate);
+ }
+ groupAddress += getGroupHeaderSize(group, formatOptions);
+ // Sanity checks.
+ if (DBG && group.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) {
+ throw new RuntimeException("A node has a frequency > "
+ + FormatSpec.MAX_TERMINAL_FREQUENCY
+ + " : " + group.mFrequency);
+ }
+ if (group.mFrequency >= 0) groupAddress += FormatSpec.GROUP_FREQUENCY_SIZE;
+ final int childrenOffset = null == group.mChildren
+ ? FormatSpec.NO_CHILDREN_ADDRESS
+ : group.mChildren.mCachedAddressAfterUpdate - groupAddress;
+ buffer[index++] =
+ makeCharGroupFlags(group, groupAddress, childrenOffset, formatOptions);
+
+ if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) {
+ index = writeParentAddress(buffer, index, parentAddress, formatOptions);
+ } else {
+ index = writeParentAddress(buffer, index, parentAddress
+ + (node.mCachedAddressAfterUpdate - group.mCachedAddressAfterUpdate),
+ formatOptions);
+ }
+
+ index = CharEncoding.writeCharArray(group.mChars, buffer, index);
+ if (group.hasSeveralChars()) {
+ buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
+ }
+ if (group.mFrequency >= 0) {
+ buffer[index++] = (byte) group.mFrequency;
+ }
+
+ final int shift;
+ if (formatOptions.mSupportsDynamicUpdate) {
+ shift = writeVariableSignedAddress(buffer, index, childrenOffset);
+ } else {
+ shift = writeVariableAddress(buffer, index, childrenOffset);
+ }
+ index += shift;
+ groupAddress += shift;
+
+ // Write shortcuts
+ if (null != group.mShortcutTargets) {
+ final int indexOfShortcutByteSize = index;
+ index += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
+ groupAddress += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
+ final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator();
+ while (shortcutIterator.hasNext()) {
+ final WeightedString target = shortcutIterator.next();
+ ++groupAddress;
+ int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(),
+ target.mFrequency);
+ buffer[index++] = (byte)shortcutFlags;
+ final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord);
+ index += shortcutShift;
+ groupAddress += shortcutShift;
+ }
+ final int shortcutByteSize = index - indexOfShortcutByteSize;
+ if (shortcutByteSize > 0xFFFF) {
+ throw new RuntimeException("Shortcut list too large");
+ }
+ buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8);
+ buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF);
+ }
+ // Write bigrams
+ if (null != group.mBigrams) {
+ final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator();
+ while (bigramIterator.hasNext()) {
+ final WeightedString bigram = bigramIterator.next();
+ final CharGroup target =
+ FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord);
+ final int addressOfBigram = target.mCachedAddressAfterUpdate;
+ final int unigramFrequencyForThisWord = target.mFrequency;
+ ++groupAddress;
+ final int offset = addressOfBigram - groupAddress;
+ int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset,
+ bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
+ buffer[index++] = (byte)bigramFlags;
+ final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset));
+ index += bigramShift;
+ groupAddress += bigramShift;
+ }
+ }
+
+ }
+ if (formatOptions.mSupportsDynamicUpdate) {
+ buffer[index] = buffer[index + 1] = buffer[index + 2]
+ = FormatSpec.NO_FORWARD_LINK_ADDRESS;
+ index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
+ }
+ if (index != node.mCachedAddressAfterUpdate + node.mCachedSize) throw new RuntimeException(
+ "Not the same size : written "
+ + (index - node.mCachedAddressAfterUpdate) + " bytes from a node that should have "
+ + node.mCachedSize + " bytes");
+ return index;
+ }
+
+ /**
+ * Dumps a collection of useful statistics about a node array.
+ *
+ * This prints purely informative stuff, like the total estimated file size, the
+ * number of nodes, of character groups, the repartition of each address size, etc
+ *
+ * @param nodes the node array.
+ */
+ private static void showStatistics(ArrayList<Node> nodes) {
+ int firstTerminalAddress = Integer.MAX_VALUE;
+ int lastTerminalAddress = Integer.MIN_VALUE;
+ int size = 0;
+ int charGroups = 0;
+ int maxGroups = 0;
+ int maxRuns = 0;
+ for (final Node n : nodes) {
+ if (maxGroups < n.mData.size()) maxGroups = n.mData.size();
+ for (final CharGroup cg : n.mData) {
+ ++charGroups;
+ if (cg.mChars.length > maxRuns) maxRuns = cg.mChars.length;
+ if (cg.mFrequency >= 0) {
+ if (n.mCachedAddressAfterUpdate < firstTerminalAddress)
+ firstTerminalAddress = n.mCachedAddressAfterUpdate;
+ if (n.mCachedAddressAfterUpdate > lastTerminalAddress)
+ lastTerminalAddress = n.mCachedAddressAfterUpdate;
+ }
+ }
+ if (n.mCachedAddressAfterUpdate + n.mCachedSize > size) {
+ size = n.mCachedAddressAfterUpdate + n.mCachedSize;
+ }
+ }
+ final int[] groupCounts = new int[maxGroups + 1];
+ final int[] runCounts = new int[maxRuns + 1];
+ for (final Node n : nodes) {
+ ++groupCounts[n.mData.size()];
+ for (final CharGroup cg : n.mData) {
+ ++runCounts[cg.mChars.length];
+ }
+ }
+
+ MakedictLog.i("Statistics:\n"
+ + " total file size " + size + "\n"
+ + " " + nodes.size() + " nodes\n"
+ + " " + charGroups + " groups (" + ((float)charGroups / nodes.size())
+ + " groups per node)\n"
+ + " first terminal at " + firstTerminalAddress + "\n"
+ + " last terminal at " + lastTerminalAddress + "\n"
+ + " Group stats : max = " + maxGroups);
+ for (int i = 0; i < groupCounts.length; ++i) {
+ MakedictLog.i(" " + i + " : " + groupCounts[i]);
+ }
+ MakedictLog.i(" Character run stats : max = " + maxRuns);
+ for (int i = 0; i < runCounts.length; ++i) {
+ MakedictLog.i(" " + i + " : " + runCounts[i]);
+ }
+ }
+
+ /**
+ * Dumps a FusionDictionary to a file.
+ *
+ * This is the public entry point to write a dictionary to a file.
+ *
+ * @param destination the stream to write the binary data to.
+ * @param dict the dictionary to write.
+ * @param formatOptions file format options.
+ */
+ public static void writeDictionaryBinary(final OutputStream destination,
+ final FusionDictionary dict, final FormatOptions formatOptions)
+ throws IOException, UnsupportedFormatException {
+
+ // Addresses are limited to 3 bytes, but since addresses can be relative to each node, the
+ // structure itself is not limited to 16MB. However, if it is over 16MB deciding the order
+ // of the nodes becomes a quite complicated problem, because though the dictionary itself
+ // does not have a size limit, each node must still be within 16MB of all its children and
+ // parents. As long as this is ensured, the dictionary file may grow to any size.
+
+ final int version = formatOptions.mVersion;
+ if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
+ || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
+ throw new UnsupportedFormatException("Requested file format version " + version
+ + ", but this implementation only supports versions "
+ + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through "
+ + FormatSpec.MAXIMUM_SUPPORTED_VERSION);
+ }
+
+ ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256);
+
+ // The magic number in big-endian order.
+ // Magic number for all versions.
+ headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 24)));
+ headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 16)));
+ headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 8)));
+ headerBuffer.write((byte) (0xFF & FormatSpec.MAGIC_NUMBER));
+ // Dictionary version.
+ headerBuffer.write((byte) (0xFF & (version >> 8)));
+ headerBuffer.write((byte) (0xFF & version));
+
+ // Options flags
+ final int options = makeOptionsValue(dict, formatOptions);
+ headerBuffer.write((byte) (0xFF & (options >> 8)));
+ headerBuffer.write((byte) (0xFF & options));
+ final int headerSizeOffset = headerBuffer.size();
+ // Placeholder to be written later with header size.
+ for (int i = 0; i < 4; ++i) {
+ headerBuffer.write(0);
+ }
+ // Write out the options.
+ for (final String key : dict.mOptions.mAttributes.keySet()) {
+ final String value = dict.mOptions.mAttributes.get(key);
+ CharEncoding.writeString(headerBuffer, key);
+ CharEncoding.writeString(headerBuffer, value);
+ }
+ final int size = headerBuffer.size();
+ final byte[] bytes = headerBuffer.toByteArray();
+ // Write out the header size.
+ bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24));
+ bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16));
+ bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8));
+ bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0));
+ destination.write(bytes);
+
+ headerBuffer.close();
+
+ // Leave the choice of the optimal node order to the flattenTree function.
+ MakedictLog.i("Flattening the tree...");
+ ArrayList<Node> flatNodes = flattenTree(dict.mRoot);
+
+ MakedictLog.i("Computing addresses...");
+ computeAddresses(dict, flatNodes, formatOptions);
+ MakedictLog.i("Checking array...");
+ if (DBG) checkFlatNodeArray(flatNodes);
+
+ // Create a buffer that matches the final dictionary size.
+ final Node lastNode = flatNodes.get(flatNodes.size() - 1);
+ final int bufferSize = lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize;
+ final byte[] buffer = new byte[bufferSize];
+ int index = 0;
+
+ MakedictLog.i("Writing file...");
+ int dataEndOffset = 0;
+ for (Node n : flatNodes) {
+ dataEndOffset = writePlacedNode(dict, buffer, n, formatOptions);
+ }
+
+ if (DBG) showStatistics(flatNodes);
+
+ destination.write(buffer, 0, dataEndOffset);
+
+ destination.close();
+ MakedictLog.i("Done");
+ }
+}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
index 2e6c4b2..c013013 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
@@ -18,8 +18,8 @@
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.CharEncoding;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.CharEncoding;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
@@ -86,7 +86,7 @@
if (index != p.mLength) index = p.mLength;
if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) {
- p.mNumOfCharGroup = BinaryDictInputOutput.readCharGroupCount(buffer);
+ p.mNumOfCharGroup = BinaryDictDecoder.readCharGroupCount(buffer);
p.mAddress += getGroupCountSize(p.mNumOfCharGroup);
p.mPosition = 0;
}
@@ -94,7 +94,7 @@
stack.pop();
continue;
}
- CharGroupInfo info = BinaryDictInputOutput.readCharGroup(buffer,
+ CharGroupInfo info = BinaryDictDecoder.readCharGroup(buffer,
p.mAddress - headerSize, formatOptions);
for (int i = 0; i < info.mCharacters.length; ++i) {
pushedChars[index++] = info.mCharacters[i];
@@ -153,7 +153,7 @@
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
UnsupportedFormatException {
// Read header
- final FileHeader header = BinaryDictInputOutput.readHeader(reader.getBuffer());
+ final FileHeader header = BinaryDictDecoder.readHeader(reader.getBuffer());
readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words,
frequencies, bigrams, header.mFormatOptions);
}
@@ -174,18 +174,18 @@
if (word == null) return FormatSpec.NOT_VALID_WORD;
if (buffer.position() != 0) buffer.position(0);
- final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
+ final FileHeader header = BinaryDictDecoder.readHeader(buffer);
int wordPos = 0;
final int wordLen = word.codePointCount(0, word.length());
for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) {
if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD;
do {
- final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer);
+ final int charGroupCount = BinaryDictDecoder.readCharGroupCount(buffer);
boolean foundNextCharGroup = false;
for (int i = 0; i < charGroupCount; ++i) {
final int charGroupPos = buffer.position();
- final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer,
+ final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer,
buffer.position(), header.mFormatOptions);
final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags,
header.mFormatOptions);
@@ -271,7 +271,7 @@
*/
private static int writeVariableAddress(final OutputStream destination, final int value)
throws IOException {
- switch (BinaryDictInputOutput.getByteSize(value)) {
+ switch (BinaryDictEncoder.getByteSize(value)) {
case 1:
destination.write((byte)value);
break;
@@ -285,15 +285,15 @@
destination.write((byte)(0xFF & value));
break;
}
- return BinaryDictInputOutput.getByteSize(value);
+ return BinaryDictEncoder.getByteSize(value);
}
static void skipCharGroup(final FusionDictionaryBufferInterface buffer,
final FormatOptions formatOptions) {
final int flags = buffer.readUnsignedByte();
- BinaryDictInputOutput.readParentAddress(buffer, formatOptions);
+ BinaryDictDecoder.readParentAddress(buffer, formatOptions);
skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
- BinaryDictInputOutput.readChildrenAddress(buffer, flags, formatOptions);
+ BinaryDictDecoder.readChildrenAddress(buffer, flags, formatOptions);
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte();
if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) {
final int shortcutsSize = buffer.readUnsignedShort();
@@ -411,14 +411,14 @@
if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) {
final int shortcutListSize =
- BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets);
+ BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets);
destination.write((byte)(shortcutListSize >> 8));
destination.write((byte)(shortcutListSize & 0xFF));
size += 2;
final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator();
while (shortcutIterator.hasNext()) {
final WeightedString target = shortcutIterator.next();
- destination.write((byte)BinaryDictInputOutput.makeShortcutFlags(
+ destination.write((byte)BinaryDictEncoder.makeShortcutFlags(
shortcutIterator.hasNext(), target.mFrequency));
size++;
size += writeString(destination, target.mWord);
@@ -427,7 +427,7 @@
if (info.mBigrams != null) {
// TODO: Consolidate this code with the code that computes the size of the bigram list
- // in BinaryDictionaryInputOutput#computeActualNodeSize
+ // in BinaryDictEncoder#computeActualNodeSize
for (int i = 0; i < info.mBigrams.size(); ++i) {
final int bigramFrequency = info.mBigrams.get(i).mFrequency;
@@ -437,7 +437,7 @@
final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress
+ size);
bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0;
- switch (BinaryDictInputOutput.getByteSize(bigramOffset)) {
+ switch (BinaryDictEncoder.getByteSize(bigramOffset)) {
case 1:
bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
break;
@@ -461,18 +461,18 @@
*/
static int computeGroupSize(final CharGroupInfo info, final FormatOptions formatOptions) {
int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
- + BinaryDictInputOutput.getGroupCharactersSize(info.mCharacters)
- + BinaryDictInputOutput.getChildrenAddressSize(info.mFlags, formatOptions);
+ + BinaryDictEncoder.getGroupCharactersSize(info.mCharacters)
+ + getChildrenAddressSize(info.mFlags, formatOptions);
if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) {
size += FormatSpec.GROUP_FREQUENCY_SIZE;
}
if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) {
- size += BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets);
+ size += BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets);
}
if (info.mBigrams != null) {
for (final PendingAttribute attr : info.mBigrams) {
size += FormatSpec.GROUP_FLAGS_SIZE;
- size += BinaryDictInputOutput.getByteSize(attr.mAddress);
+ size += BinaryDictEncoder.getByteSize(attr.mAddress);
}
}
return size;
@@ -520,9 +520,9 @@
int position = getTerminalPosition(buffer, word);
if (position != FormatSpec.NOT_VALID_WORD) {
buffer.position(0);
- final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
+ final FileHeader header = BinaryDictDecoder.readHeader(buffer);
buffer.position(position);
- return BinaryDictInputOutput.readCharGroup(buffer, position, header.mFormatOptions);
+ return BinaryDictDecoder.readCharGroup(buffer, position, header.mFormatOptions);
}
return null;
}
@@ -544,10 +544,10 @@
final FileInputStream inStream = new FileInputStream(file);
try {
inStream.read(buffer);
- final BinaryDictInputOutput.ByteBufferWrapper wrapper =
- new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map(
+ final BinaryDictDecoder.ByteBufferWrapper wrapper =
+ new BinaryDictDecoder.ByteBufferWrapper(inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, offset, length));
- return BinaryDictInputOutput.readHeader(wrapper);
+ return BinaryDictDecoder.readHeader(wrapper);
} finally {
inStream.close();
}
@@ -612,4 +612,20 @@
+ ")");
}
}
+
+ static int getChildrenAddressSize(final int optionFlags,
+ final FormatOptions formatOptions) {
+ if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
+ switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
+ return 1;
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
+ return 2;
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
+ return 3;
+ case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
+ default:
+ return 0;
+ }
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
deleted file mode 100644
index a33cddb..0000000
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
+++ /dev/null
@@ -1,1783 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.inputmethod.latin.makedict;
-
-import com.android.inputmethod.annotations.UsedForTesting;
-import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
-import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
-import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
-import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
-import com.android.inputmethod.latin.utils.JniUtils;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * Reads and writes XML files for a FusionDictionary.
- *
- * All the methods in this class are static.
- */
-public final class BinaryDictInputOutput {
-
- private static final boolean DBG = MakedictLog.DBG;
-
- static {
- JniUtils.loadNativeLibrary();
- }
-
- // TODO: implement something sensical instead of just a phony method
- private static native int doNothing();
-
- // Arbitrary limit to how much passes we consider address size compression should
- // terminate in. At the time of this writing, our largest dictionary completes
- // compression in five passes.
- // If the number of passes exceeds this number, makedict bails with an exception on
- // suspicion that a bug might be causing an infinite loop.
- private static final int MAX_PASSES = 24;
- private static final int MAX_JUMPS = 12;
-
- @UsedForTesting
- public interface FusionDictionaryBufferInterface {
- public int readUnsignedByte();
- public int readUnsignedShort();
- public int readUnsignedInt24();
- public int readInt();
- public int position();
- public void position(int newPosition);
- public void put(final byte b);
- public int limit();
- @UsedForTesting
- public int capacity();
- }
-
- public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
- private ByteBuffer mBuffer;
-
- public ByteBufferWrapper(final ByteBuffer buffer) {
- mBuffer = buffer;
- }
-
- @Override
- public int readUnsignedByte() {
- return mBuffer.get() & 0xFF;
- }
-
- @Override
- public int readUnsignedShort() {
- return mBuffer.getShort() & 0xFFFF;
- }
-
- @Override
- public int readUnsignedInt24() {
- final int retval = readUnsignedByte();
- return (retval << 16) + readUnsignedShort();
- }
-
- @Override
- public int readInt() {
- return mBuffer.getInt();
- }
-
- @Override
- public int position() {
- return mBuffer.position();
- }
-
- @Override
- public void position(int newPos) {
- mBuffer.position(newPos);
- }
-
- @Override
- public void put(final byte b) {
- mBuffer.put(b);
- }
-
- @Override
- public int limit() {
- return mBuffer.limit();
- }
-
- @Override
- public int capacity() {
- return mBuffer.capacity();
- }
- }
-
- /**
- * A class grouping utility function for our specific character encoding.
- */
- static final class CharEncoding {
- private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
- private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF;
-
- /**
- * Helper method to find out whether this code fits on one byte
- */
- private static boolean fitsOnOneByte(final int character) {
- return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE
- && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE;
- }
-
- /**
- * Compute the size of a character given its character code.
- *
- * Char format is:
- * 1 byte = bbbbbbbb match
- * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
- * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
- * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
- * 00011111 would be outside unicode.
- * else: iso-latin-1 code
- * This allows for the whole unicode range to be encoded, including chars outside of
- * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
- * characters which should never happen anyway (and still work, but take 3 bytes).
- *
- * @param character the character code.
- * @return the size in binary encoded-form, either 1 or 3 bytes.
- */
- static int getCharSize(final int character) {
- // See char encoding in FusionDictionary.java
- if (fitsOnOneByte(character)) return 1;
- if (FormatSpec.INVALID_CHARACTER == character) return 1;
- return 3;
- }
-
- /**
- * Compute the byte size of a character array.
- */
- private static int getCharArraySize(final int[] chars) {
- int size = 0;
- for (int character : chars) size += getCharSize(character);
- return size;
- }
-
- /**
- * Writes a char array to a byte buffer.
- *
- * @param codePoints the code point array to write.
- * @param buffer the byte buffer to write to.
- * @param index the index in buffer to write the character array to.
- * @return the index after the last character.
- */
- private static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) {
- for (int codePoint : codePoints) {
- if (1 == getCharSize(codePoint)) {
- buffer[index++] = (byte)codePoint;
- } else {
- buffer[index++] = (byte)(0xFF & (codePoint >> 16));
- buffer[index++] = (byte)(0xFF & (codePoint >> 8));
- buffer[index++] = (byte)(0xFF & codePoint);
- }
- }
- return index;
- }
-
- /**
- * Writes a string with our character format to a byte buffer.
- *
- * This will also write the terminator byte.
- *
- * @param buffer the byte buffer to write to.
- * @param origin the offset to write from.
- * @param word the string to write.
- * @return the size written, in bytes.
- */
- private static int writeString(final byte[] buffer, final int origin,
- final String word) {
- final int length = word.length();
- int index = origin;
- for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
- final int codePoint = word.codePointAt(i);
- if (1 == getCharSize(codePoint)) {
- buffer[index++] = (byte)codePoint;
- } else {
- buffer[index++] = (byte)(0xFF & (codePoint >> 16));
- buffer[index++] = (byte)(0xFF & (codePoint >> 8));
- buffer[index++] = (byte)(0xFF & codePoint);
- }
- }
- buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
- return index - origin;
- }
-
- /**
- * Writes a string with our character format to a ByteArrayOutputStream.
- *
- * This will also write the terminator byte.
- *
- * @param buffer the ByteArrayOutputStream to write to.
- * @param word the string to write.
- */
- private static void writeString(final ByteArrayOutputStream buffer, final String word) {
- final int length = word.length();
- for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
- final int codePoint = word.codePointAt(i);
- if (1 == getCharSize(codePoint)) {
- buffer.write((byte) codePoint);
- } else {
- buffer.write((byte) (0xFF & (codePoint >> 16)));
- buffer.write((byte) (0xFF & (codePoint >> 8)));
- buffer.write((byte) (0xFF & codePoint));
- }
- }
- buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR);
- }
-
- /**
- * Reads a string from a buffer. This is the converse of the above method.
- */
- private static String readString(final FusionDictionaryBufferInterface buffer) {
- final StringBuilder s = new StringBuilder();
- int character = readChar(buffer);
- while (character != FormatSpec.INVALID_CHARACTER) {
- s.appendCodePoint(character);
- character = readChar(buffer);
- }
- return s.toString();
- }
-
- /**
- * Reads a character from the buffer.
- *
- * This follows the character format documented earlier in this source file.
- *
- * @param buffer the buffer, positioned over an encoded character.
- * @return the character code.
- */
- static int readChar(final FusionDictionaryBufferInterface buffer) {
- int character = buffer.readUnsignedByte();
- if (!fitsOnOneByte(character)) {
- if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) {
- return FormatSpec.INVALID_CHARACTER;
- }
- character <<= 16;
- character += buffer.readUnsignedShort();
- }
- return character;
- }
- }
-
- /**
- * Compute the binary size of the character array.
- *
- * If only one character, this is the size of this character. If many, it's the sum of their
- * sizes + 1 byte for the terminator.
- *
- * @param characters the character array
- * @return the size of the char array, including the terminator if any
- */
- static int getGroupCharactersSize(final int[] characters) {
- int size = CharEncoding.getCharArraySize(characters);
- if (characters.length > 1) size += FormatSpec.GROUP_TERMINATOR_SIZE;
- return size;
- }
-
- /**
- * Compute the binary size of the character array in a group
- *
- * If only one character, this is the size of this character. If many, it's the sum of their
- * sizes + 1 byte for the terminator.
- *
- * @param group the group
- * @return the size of the char array, including the terminator if any
- */
- private static int getGroupCharactersSize(final CharGroup group) {
- return getGroupCharactersSize(group.mChars);
- }
-
- /**
- * Compute the binary size of the group count for a node
- * @param node the node
- * @return the size of the group count, either 1 or 2 bytes.
- */
- private static int getGroupCountSize(final Node node) {
- return BinaryDictIOUtils.getGroupCountSize(node.mData.size());
- }
-
- /**
- * Compute the size of a shortcut in bytes.
- */
- private static int getShortcutSize(final WeightedString shortcut) {
- int size = FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE;
- final String word = shortcut.mWord;
- final int length = word.length();
- for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
- final int codePoint = word.codePointAt(i);
- size += CharEncoding.getCharSize(codePoint);
- }
- size += FormatSpec.GROUP_TERMINATOR_SIZE;
- return size;
- }
-
- /**
- * Compute the size of a shortcut list in bytes.
- *
- * This is known in advance and does not change according to position in the file
- * like address lists do.
- */
- static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) {
- if (null == shortcutList) return 0;
- int size = FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
- for (final WeightedString shortcut : shortcutList) {
- size += getShortcutSize(shortcut);
- }
- return size;
- }
-
- /**
- * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything.
- *
- * @param group the CharGroup to compute the size of.
- * @param options file format options.
- * @return the maximum size of the group.
- */
- private static int getCharGroupMaximumSize(final CharGroup group, final FormatOptions options) {
- int size = getGroupHeaderSize(group, options);
- // If terminal, one byte for the frequency
- if (group.isTerminal()) size += FormatSpec.GROUP_FREQUENCY_SIZE;
- size += FormatSpec.GROUP_MAX_ADDRESS_SIZE; // For children address
- size += getShortcutListSize(group.mShortcutTargets);
- if (null != group.mBigrams) {
- size += (FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE
- + FormatSpec.GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE)
- * group.mBigrams.size();
- }
- return size;
- }
-
- /**
- * Compute the maximum size of a node, assuming 3-byte addresses for everything, and caches
- * it in the 'actualSize' member of the node.
- *
- * @param node the node to compute the maximum size of.
- * @param options file format options.
- */
- private static void calculateNodeMaximumSize(final Node node, final FormatOptions options) {
- int size = getGroupCountSize(node);
- for (CharGroup g : node.mData) {
- final int groupSize = getCharGroupMaximumSize(g, options);
- g.mCachedSize = groupSize;
- size += groupSize;
- }
- if (options.mSupportsDynamicUpdate) {
- size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
- }
- node.mCachedSize = size;
- }
-
- /**
- * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup.
- *
- * @param group the group of which to compute the size of the header
- * @param options file format options.
- */
- private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) {
- if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
- return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
- + getGroupCharactersSize(group);
- } else {
- return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group);
- }
- }
-
- /**
- * Compute the size, in bytes, that an address will occupy.
- *
- * This can be used either for children addresses (which are always positive) or for
- * attribute, which may be positive or negative but
- * store their sign bit separately.
- *
- * @param address the address
- * @return the byte size.
- */
- static int getByteSize(final int address) {
- assert(address <= FormatSpec.UINT24_MAX);
- if (!BinaryDictIOUtils.hasChildrenAddress(address)) {
- return 0;
- } else if (Math.abs(address) <= FormatSpec.UINT8_MAX) {
- return 1;
- } else if (Math.abs(address) <= FormatSpec.UINT16_MAX) {
- return 2;
- } else {
- return 3;
- }
- }
-
- // End utility methods.
-
- // This method is responsible for finding a nice ordering of the nodes that favors run-time
- // cache performance and dictionary size.
- /* package for tests */ static ArrayList<Node> flattenTree(final Node root) {
- final int treeSize = FusionDictionary.countCharGroups(root);
- MakedictLog.i("Counted nodes : " + treeSize);
- final ArrayList<Node> flatTree = new ArrayList<Node>(treeSize);
- return flattenTreeInner(flatTree, root);
- }
-
- private static ArrayList<Node> flattenTreeInner(final ArrayList<Node> list, final Node node) {
- // Removing the node is necessary if the tails are merged, because we would then
- // add the same node several times when we only want it once. A number of places in
- // the code also depends on any node being only once in the list.
- // Merging tails can only be done if there are no attributes. Searching for attributes
- // in LatinIME code depends on a total breadth-first ordering, which merging tails
- // breaks. If there are no attributes, it should be fine (and reduce the file size)
- // to merge tails, and removing the node from the list would be necessary. However,
- // we don't merge tails because breaking the breadth-first ordering would result in
- // extreme overhead at bigram lookup time (it would make the search function O(n) instead
- // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty
- // high).
- // If no nodes are ever merged, we can't have the same node twice in the list, hence
- // searching for duplicates in unnecessary. It is also very performance consuming,
- // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making
- // this simple list.remove operation O(n*n) overall. On Android this overhead is very
- // high.
- // For future reference, the code to remove duplicate is a simple : list.remove(node);
- list.add(node);
- final ArrayList<CharGroup> branches = node.mData;
- final int nodeSize = branches.size();
- for (CharGroup group : branches) {
- if (null != group.mChildren) flattenTreeInner(list, group.mChildren);
- }
- return list;
- }
-
- /**
- * Get the offset from a position inside a current node to a target node, during update.
- *
- * If the current node is before the target node, the target node has not been updated yet,
- * so we should return the offset from the old position of the current node to the old position
- * of the target node. If on the other hand the target is before the current node, it already
- * has been updated, so we should return the offset from the new position in the current node
- * to the new position in the target node.
- * @param currentNode the node containing the CharGroup where the offset will be written
- * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode
- * @param targetNode the target node to get the offset to
- * @return the offset to the target node
- */
- private static int getOffsetToTargetNodeDuringUpdate(final Node currentNode,
- final int offsetFromStartOfCurrentNode, final Node targetNode) {
- final boolean isTargetBeforeCurrent = (targetNode.mCachedAddressBeforeUpdate
- < currentNode.mCachedAddressBeforeUpdate);
- if (isTargetBeforeCurrent) {
- return targetNode.mCachedAddressAfterUpdate
- - (currentNode.mCachedAddressAfterUpdate + offsetFromStartOfCurrentNode);
- } else {
- return targetNode.mCachedAddressBeforeUpdate
- - (currentNode.mCachedAddressBeforeUpdate + offsetFromStartOfCurrentNode);
- }
- }
-
- /**
- * Get the offset from a position inside a current node to a target CharGroup, during update.
- * @param currentNode the node containing the CharGroup where the offset will be written
- * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode
- * @param targetCharGroup the target CharGroup to get the offset to
- * @return the offset to the target CharGroup
- */
- // TODO: is there any way to factorize this method with the one above?
- private static int getOffsetToTargetCharGroupDuringUpdate(final Node currentNode,
- final int offsetFromStartOfCurrentNode, final CharGroup targetCharGroup) {
- final int oldOffsetBasePoint = currentNode.mCachedAddressBeforeUpdate
- + offsetFromStartOfCurrentNode;
- final boolean isTargetBeforeCurrent = (targetCharGroup.mCachedAddressBeforeUpdate
- < oldOffsetBasePoint);
- // If the target is before the current node, then its address has already been updated.
- // We can use the AfterUpdate member, and compare it to our own member after update.
- // Otherwise, the AfterUpdate member is not updated yet, so we need to use the BeforeUpdate
- // member, and of course we have to compare this to our own address before update.
- if (isTargetBeforeCurrent) {
- final int newOffsetBasePoint = currentNode.mCachedAddressAfterUpdate
- + offsetFromStartOfCurrentNode;
- return targetCharGroup.mCachedAddressAfterUpdate - newOffsetBasePoint;
- } else {
- return targetCharGroup.mCachedAddressBeforeUpdate - oldOffsetBasePoint;
- }
- }
-
- /**
- * Computes the actual node size, based on the cached addresses of the children nodes.
- *
- * Each node stores its tentative address. During dictionary address computing, these
- * are not final, but they can be used to compute the node size (the node size depends
- * on the address of the children because the number of bytes necessary to store an
- * address depends on its numeric value. The return value indicates whether the node
- * contents (as in, any of the addresses stored in the cache fields) have changed with
- * respect to their previous value.
- *
- * @param node the node to compute the size of.
- * @param dict the dictionary in which the word/attributes are to be found.
- * @param formatOptions file format options.
- * @return false if none of the cached addresses inside the node changed, true otherwise.
- */
- private static boolean computeActualNodeSize(final Node node, final FusionDictionary dict,
- final FormatOptions formatOptions) {
- boolean changed = false;
- int size = getGroupCountSize(node);
- for (CharGroup group : node.mData) {
- group.mCachedAddressAfterUpdate = node.mCachedAddressAfterUpdate + size;
- if (group.mCachedAddressAfterUpdate != group.mCachedAddressBeforeUpdate) {
- changed = true;
- }
- int groupSize = getGroupHeaderSize(group, formatOptions);
- if (group.isTerminal()) groupSize += FormatSpec.GROUP_FREQUENCY_SIZE;
- if (null == group.mChildren && formatOptions.mSupportsDynamicUpdate) {
- groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
- } else if (null != group.mChildren) {
- if (formatOptions.mSupportsDynamicUpdate) {
- groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
- } else {
- groupSize += getByteSize(getOffsetToTargetNodeDuringUpdate(node,
- groupSize + size, group.mChildren));
- }
- }
- groupSize += getShortcutListSize(group.mShortcutTargets);
- if (null != group.mBigrams) {
- for (WeightedString bigram : group.mBigrams) {
- final int offset = getOffsetToTargetCharGroupDuringUpdate(node,
- groupSize + size + FormatSpec.GROUP_FLAGS_SIZE,
- FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord));
- groupSize += getByteSize(offset) + FormatSpec.GROUP_FLAGS_SIZE;
- }
- }
- group.mCachedSize = groupSize;
- size += groupSize;
- }
- if (formatOptions.mSupportsDynamicUpdate) {
- size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
- }
- if (node.mCachedSize != size) {
- node.mCachedSize = size;
- changed = true;
- }
- return changed;
- }
-
- /**
- * Initializes the cached addresses of nodes from their size.
- *
- * @param flatNodes the array of nodes.
- * @param formatOptions file format options.
- * @return the byte size of the entire stack.
- */
- private static int initializeNodesCachedAddresses(final ArrayList<Node> flatNodes,
- final FormatOptions formatOptions) {
- int nodeOffset = 0;
- for (final Node n : flatNodes) {
- n.mCachedAddressBeforeUpdate = nodeOffset;
- int groupCountSize = getGroupCountSize(n);
- int groupOffset = 0;
- for (final CharGroup g : n.mData) {
- g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate =
- groupCountSize + nodeOffset + groupOffset;
- groupOffset += g.mCachedSize;
- }
- final int nodeSize = groupCountSize + groupOffset
- + (formatOptions.mSupportsDynamicUpdate
- ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0);
- nodeOffset += n.mCachedSize;
- }
- return nodeOffset;
- }
-
- /**
- * Updates the cached addresses of nodes after recomputing their new positions.
- *
- * @param flatNodes the array of nodes.
- */
- private static void updateNodeCachedAddresses(final ArrayList<Node> flatNodes) {
- for (final Node n : flatNodes) {
- n.mCachedAddressBeforeUpdate = n.mCachedAddressAfterUpdate;
- for (final CharGroup g : n.mData) {
- g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate;
- }
- }
- }
-
- /**
- * Compute the cached parent addresses after all has been updated.
- *
- * The parent addresses are used by some binary formats at write-to-disk time. Not all formats
- * need them. In particular, version 2 does not need them, and version 3 does.
- *
- * @param flatNodes the flat array of nodes to fill in
- */
- private static void computeParentAddresses(final ArrayList<Node> flatNodes) {
- for (final Node node : flatNodes) {
- for (final CharGroup group : node.mData) {
- if (null != group.mChildren) {
- // Assign my address to children's parent address
- // Here BeforeUpdate and AfterUpdate addresses have the same value, so it
- // does not matter which we use.
- group.mChildren.mCachedParentAddress = group.mCachedAddressAfterUpdate
- - group.mChildren.mCachedAddressAfterUpdate;
- }
- }
- }
- }
-
- /**
- * Compute the addresses and sizes of an ordered node array.
- *
- * This method takes a node array and will update its cached address and size values
- * so that they can be written into a file. It determines the smallest size each of the
- * nodes can be given the addresses of its children and attributes, and store that into
- * each node.
- * The order of the node is given by the order of the array. This method makes no effort
- * to find a good order; it only mechanically computes the size this order results in.
- *
- * @param dict the dictionary
- * @param flatNodes the ordered array of nodes
- * @param formatOptions file format options.
- * @return the same array it was passed. The nodes have been updated for address and size.
- */
- private static ArrayList<Node> computeAddresses(final FusionDictionary dict,
- final ArrayList<Node> flatNodes, final FormatOptions formatOptions) {
- // First get the worst possible sizes and offsets
- for (final Node n : flatNodes) calculateNodeMaximumSize(n, formatOptions);
- final int offset = initializeNodesCachedAddresses(flatNodes, formatOptions);
-
- MakedictLog.i("Compressing the array addresses. Original size : " + offset);
- MakedictLog.i("(Recursively seen size : " + offset + ")");
-
- int passes = 0;
- boolean changesDone = false;
- do {
- changesDone = false;
- int nodeStartOffset = 0;
- for (final Node n : flatNodes) {
- n.mCachedAddressAfterUpdate = nodeStartOffset;
- final int oldNodeSize = n.mCachedSize;
- final boolean changed = computeActualNodeSize(n, dict, formatOptions);
- final int newNodeSize = n.mCachedSize;
- if (oldNodeSize < newNodeSize) throw new RuntimeException("Increased size ?!");
- nodeStartOffset += newNodeSize;
- changesDone |= changed;
- }
- updateNodeCachedAddresses(flatNodes);
- ++passes;
- if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug");
- } while (changesDone);
-
- if (formatOptions.mSupportsDynamicUpdate) {
- computeParentAddresses(flatNodes);
- }
- final Node lastNode = flatNodes.get(flatNodes.size() - 1);
- MakedictLog.i("Compression complete in " + passes + " passes.");
- MakedictLog.i("After address compression : "
- + (lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize));
-
- return flatNodes;
- }
-
- /**
- * Sanity-checking method.
- *
- * This method checks an array of node for juxtaposition, that is, it will do
- * nothing if each node's cached address is actually the previous node's address
- * plus the previous node's size.
- * If this is not the case, it will throw an exception.
- *
- * @param array the array node to check
- */
- private static void checkFlatNodeArray(final ArrayList<Node> array) {
- int offset = 0;
- int index = 0;
- for (final Node n : array) {
- // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter
- // which we use.
- if (n.mCachedAddressAfterUpdate != offset) {
- throw new RuntimeException("Wrong address for node " + index
- + " : expected " + offset + ", got " + n.mCachedAddressAfterUpdate);
- }
- ++index;
- offset += n.mCachedSize;
- }
- }
-
- /**
- * Helper method to write a variable-size address to a file.
- *
- * @param buffer the buffer to write to.
- * @param index the index in the buffer to write the address to.
- * @param address the address to write.
- * @return the size in bytes the address actually took.
- */
- private static int writeVariableAddress(final byte[] buffer, int index, final int address) {
- switch (getByteSize(address)) {
- case 1:
- buffer[index++] = (byte)address;
- return 1;
- case 2:
- buffer[index++] = (byte)(0xFF & (address >> 8));
- buffer[index++] = (byte)(0xFF & address);
- return 2;
- case 3:
- buffer[index++] = (byte)(0xFF & (address >> 16));
- buffer[index++] = (byte)(0xFF & (address >> 8));
- buffer[index++] = (byte)(0xFF & address);
- return 3;
- case 0:
- return 0;
- default:
- throw new RuntimeException("Address " + address + " has a strange size");
- }
- }
-
- /**
- * Helper method to write a variable-size signed address to a file.
- *
- * @param buffer the buffer to write to.
- * @param index the index in the buffer to write the address to.
- * @param address the address to write.
- * @return the size in bytes the address actually took.
- */
- private static int writeVariableSignedAddress(final byte[] buffer, int index,
- final int address) {
- if (!BinaryDictIOUtils.hasChildrenAddress(address)) {
- buffer[index] = buffer[index + 1] = buffer[index + 2] = 0;
- } else {
- final int absAddress = Math.abs(address);
- buffer[index++] =
- (byte)((address < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absAddress >> 16)));
- buffer[index++] = (byte)(0xFF & (absAddress >> 8));
- buffer[index++] = (byte)(0xFF & absAddress);
- }
- return 3;
- }
-
- /**
- * Makes the flag value for a char group.
- *
- * @param hasMultipleChars whether the group has multiple chars.
- * @param isTerminal whether the group is terminal.
- * @param childrenAddressSize the size of a children address.
- * @param hasShortcuts whether the group has shortcuts.
- * @param hasBigrams whether the group has bigrams.
- * @param isNotAWord whether the group is not a word.
- * @param isBlackListEntry whether the group is a blacklist entry.
- * @param formatOptions file format options.
- * @return the flags
- */
- static int makeCharGroupFlags(final boolean hasMultipleChars, final boolean isTerminal,
- final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams,
- final boolean isNotAWord, final boolean isBlackListEntry,
- final FormatOptions formatOptions) {
- byte flags = 0;
- if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS;
- if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL;
- if (formatOptions.mSupportsDynamicUpdate) {
- flags |= FormatSpec.FLAG_IS_NOT_MOVED;
- } else if (true) {
- switch (childrenAddressSize) {
- case 1:
- flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE;
- break;
- case 2:
- flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES;
- break;
- case 3:
- flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES;
- break;
- case 0:
- flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS;
- break;
- default:
- throw new RuntimeException("Node with a strange address");
- }
- }
- if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS;
- if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS;
- if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD;
- if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED;
- return flags;
- }
-
- private static byte makeCharGroupFlags(final CharGroup group, final int groupAddress,
- final int childrenOffset, final FormatOptions formatOptions) {
- return (byte) makeCharGroupFlags(group.mChars.length > 1, group.mFrequency >= 0,
- getByteSize(childrenOffset), group.mShortcutTargets != null, group.mBigrams != null,
- group.mIsNotAWord, group.mIsBlacklistEntry, formatOptions);
- }
-
- /**
- * Makes the flag value for a bigram.
- *
- * @param more whether there are more bigrams after this one.
- * @param offset the offset of the bigram.
- * @param bigramFrequency the frequency of the bigram, 0..255.
- * @param unigramFrequency the unigram frequency of the same word, 0..255.
- * @param word the second bigram, for debugging purposes
- * @return the flags
- */
- private static final int makeBigramFlags(final boolean more, final int offset,
- int bigramFrequency, final int unigramFrequency, final String word) {
- int bigramFlags = (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0)
- + (offset < 0 ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0);
- switch (getByteSize(offset)) {
- case 1:
- bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
- break;
- case 2:
- bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES;
- break;
- case 3:
- bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES;
- break;
- default:
- throw new RuntimeException("Strange offset size");
- }
- if (unigramFrequency > bigramFrequency) {
- MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word
- + "\". Bigram freq is " + bigramFrequency + ", unigram freq for "
- + word + " is " + unigramFrequency);
- bigramFrequency = unigramFrequency;
- }
- // We compute the difference between 255 (which means probability = 1) and the
- // unigram score. We split this into a number of discrete steps.
- // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15
- // represents an increase of 16 steps: a value of 15 will be interpreted as the median
- // value of the 16th step. In all justice, if the bigram frequency is low enough to be
- // rounded below the first step (which means it is less than half a step higher than the
- // unigram frequency) then the unigram frequency itself is the best approximation of the
- // bigram freq that we could possibly supply, hence we should *not* include this bigram
- // in the file at all.
- // until this is done, we'll write 0 and slightly overestimate this case.
- // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step
- // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to
- // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the
- // step size. Then we compute the start of the first step (the one where value 0 starts)
- // by adding half-a-step to the unigramFrequency. From there, we compute the integer
- // number of steps to the bigramFrequency. One last thing: we want our steps to include
- // their lower bound and exclude their higher bound so we need to have the first step
- // start at exactly 1 unit higher than floor(unigramFreq + half a step).
- // Note : to reconstruct the score, the dictionary reader will need to divide
- // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step,
- // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best
- // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the
- // step pointed by the discretized frequency.
- final float stepSize =
- (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
- / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
- final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f);
- final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize);
- // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1
- // here. The best approximation would be the unigram freq itself, so we should not
- // include this bigram in the dictionary. For now, register as 0, and live with the
- // small over-estimation that we get in this case. TODO: actually remove this bigram
- // if discretizedFrequency < 0.
- final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0;
- bigramFlags += finalBigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY;
- return bigramFlags;
- }
-
- /**
- * Makes the 2-byte value for options flags.
- */
- private static final int makeOptionsValue(final FusionDictionary dictionary,
- final FormatOptions formatOptions) {
- final DictionaryOptions options = dictionary.mOptions;
- final boolean hasBigrams = dictionary.hasBigrams();
- return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0)
- + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0)
- + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0)
- + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0);
- }
-
- /**
- * Makes the flag value for a shortcut.
- *
- * @param more whether there are more attributes after this one.
- * @param frequency the frequency of the attribute, 0..15
- * @return the flags
- */
- static final int makeShortcutFlags(final boolean more, final int frequency) {
- return (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0)
- + (frequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY);
- }
-
- private static final int writeParentAddress(final byte[] buffer, final int index,
- final int address, final FormatOptions formatOptions) {
- if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
- if (address == FormatSpec.NO_PARENT_ADDRESS) {
- buffer[index] = buffer[index + 1] = buffer[index + 2] = 0;
- } else {
- final int absAddress = Math.abs(address);
- assert(absAddress <= FormatSpec.SINT24_MAX);
- buffer[index] = (byte)((address < 0 ? FormatSpec.MSB8 : 0)
- | ((absAddress >> 16) & 0xFF));
- buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF);
- buffer[index + 2] = (byte)(absAddress & 0xFF);
- }
- return index + 3;
- } else {
- return index;
- }
- }
-
- /**
- * Write a node to memory. The node is expected to have its final position cached.
- *
- * This can be an empty map, but the more is inside the faster the lookups will be. It can
- * be carried on as long as nodes do not move.
- *
- * @param dict the dictionary the node is a part of (for relative offsets).
- * @param buffer the memory buffer to write to.
- * @param node the node to write.
- * @param formatOptions file format options.
- * @return the address of the END of the node.
- */
- @SuppressWarnings("unused")
- private static int writePlacedNode(final FusionDictionary dict, byte[] buffer,
- final Node node, final FormatOptions formatOptions) {
- // TODO: Make the code in common with BinaryDictIOUtils#writeCharGroup
- int index = node.mCachedAddressAfterUpdate;
-
- final int groupCount = node.mData.size();
- final int countSize = getGroupCountSize(node);
- final int parentAddress = node.mCachedParentAddress;
- if (1 == countSize) {
- buffer[index++] = (byte)groupCount;
- } else if (2 == countSize) {
- // We need to signal 2-byte size by setting the top bit of the MSB to 1, so
- // we | 0x80 to do this.
- buffer[index++] = (byte)((groupCount >> 8) | 0x80);
- buffer[index++] = (byte)(groupCount & 0xFF);
- } else {
- throw new RuntimeException("Strange size from getGroupCountSize : " + countSize);
- }
- int groupAddress = index;
- for (int i = 0; i < groupCount; ++i) {
- final CharGroup group = node.mData.get(i);
- if (index != group.mCachedAddressAfterUpdate) {
- throw new RuntimeException("Bug: write index is not the same as the cached address "
- + "of the group : " + index + " <> " + group.mCachedAddressAfterUpdate);
- }
- groupAddress += getGroupHeaderSize(group, formatOptions);
- // Sanity checks.
- if (DBG && group.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) {
- throw new RuntimeException("A node has a frequency > "
- + FormatSpec.MAX_TERMINAL_FREQUENCY
- + " : " + group.mFrequency);
- }
- if (group.mFrequency >= 0) groupAddress += FormatSpec.GROUP_FREQUENCY_SIZE;
- final int childrenOffset = null == group.mChildren
- ? FormatSpec.NO_CHILDREN_ADDRESS
- : group.mChildren.mCachedAddressAfterUpdate - groupAddress;
- buffer[index++] =
- makeCharGroupFlags(group, groupAddress, childrenOffset, formatOptions);
-
- if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) {
- index = writeParentAddress(buffer, index, parentAddress, formatOptions);
- } else {
- index = writeParentAddress(buffer, index, parentAddress
- + (node.mCachedAddressAfterUpdate - group.mCachedAddressAfterUpdate),
- formatOptions);
- }
-
- index = CharEncoding.writeCharArray(group.mChars, buffer, index);
- if (group.hasSeveralChars()) {
- buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
- }
- if (group.mFrequency >= 0) {
- buffer[index++] = (byte) group.mFrequency;
- }
-
- final int shift;
- if (formatOptions.mSupportsDynamicUpdate) {
- shift = writeVariableSignedAddress(buffer, index, childrenOffset);
- } else {
- shift = writeVariableAddress(buffer, index, childrenOffset);
- }
- index += shift;
- groupAddress += shift;
-
- // Write shortcuts
- if (null != group.mShortcutTargets) {
- final int indexOfShortcutByteSize = index;
- index += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
- groupAddress += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE;
- final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator();
- while (shortcutIterator.hasNext()) {
- final WeightedString target = shortcutIterator.next();
- ++groupAddress;
- int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(),
- target.mFrequency);
- buffer[index++] = (byte)shortcutFlags;
- final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord);
- index += shortcutShift;
- groupAddress += shortcutShift;
- }
- final int shortcutByteSize = index - indexOfShortcutByteSize;
- if (shortcutByteSize > 0xFFFF) {
- throw new RuntimeException("Shortcut list too large");
- }
- buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8);
- buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF);
- }
- // Write bigrams
- if (null != group.mBigrams) {
- final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator();
- while (bigramIterator.hasNext()) {
- final WeightedString bigram = bigramIterator.next();
- final CharGroup target =
- FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord);
- final int addressOfBigram = target.mCachedAddressAfterUpdate;
- final int unigramFrequencyForThisWord = target.mFrequency;
- ++groupAddress;
- final int offset = addressOfBigram - groupAddress;
- int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset,
- bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
- buffer[index++] = (byte)bigramFlags;
- final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset));
- index += bigramShift;
- groupAddress += bigramShift;
- }
- }
-
- }
- if (formatOptions.mSupportsDynamicUpdate) {
- buffer[index] = buffer[index + 1] = buffer[index + 2]
- = FormatSpec.NO_FORWARD_LINK_ADDRESS;
- index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
- }
- if (index != node.mCachedAddressAfterUpdate + node.mCachedSize) throw new RuntimeException(
- "Not the same size : written "
- + (index - node.mCachedAddressAfterUpdate) + " bytes from a node that should have "
- + node.mCachedSize + " bytes");
- return index;
- }
-
- /**
- * Dumps a collection of useful statistics about a node array.
- *
- * This prints purely informative stuff, like the total estimated file size, the
- * number of nodes, of character groups, the repartition of each address size, etc
- *
- * @param nodes the node array.
- */
- private static void showStatistics(ArrayList<Node> nodes) {
- int firstTerminalAddress = Integer.MAX_VALUE;
- int lastTerminalAddress = Integer.MIN_VALUE;
- int size = 0;
- int charGroups = 0;
- int maxGroups = 0;
- int maxRuns = 0;
- for (final Node n : nodes) {
- if (maxGroups < n.mData.size()) maxGroups = n.mData.size();
- for (final CharGroup cg : n.mData) {
- ++charGroups;
- if (cg.mChars.length > maxRuns) maxRuns = cg.mChars.length;
- if (cg.mFrequency >= 0) {
- if (n.mCachedAddressAfterUpdate < firstTerminalAddress)
- firstTerminalAddress = n.mCachedAddressAfterUpdate;
- if (n.mCachedAddressAfterUpdate > lastTerminalAddress)
- lastTerminalAddress = n.mCachedAddressAfterUpdate;
- }
- }
- if (n.mCachedAddressAfterUpdate + n.mCachedSize > size) {
- size = n.mCachedAddressAfterUpdate + n.mCachedSize;
- }
- }
- final int[] groupCounts = new int[maxGroups + 1];
- final int[] runCounts = new int[maxRuns + 1];
- for (final Node n : nodes) {
- ++groupCounts[n.mData.size()];
- for (final CharGroup cg : n.mData) {
- ++runCounts[cg.mChars.length];
- }
- }
-
- MakedictLog.i("Statistics:\n"
- + " total file size " + size + "\n"
- + " " + nodes.size() + " nodes\n"
- + " " + charGroups + " groups (" + ((float)charGroups / nodes.size())
- + " groups per node)\n"
- + " first terminal at " + firstTerminalAddress + "\n"
- + " last terminal at " + lastTerminalAddress + "\n"
- + " Group stats : max = " + maxGroups);
- for (int i = 0; i < groupCounts.length; ++i) {
- MakedictLog.i(" " + i + " : " + groupCounts[i]);
- }
- MakedictLog.i(" Character run stats : max = " + maxRuns);
- for (int i = 0; i < runCounts.length; ++i) {
- MakedictLog.i(" " + i + " : " + runCounts[i]);
- }
- }
-
- /**
- * Dumps a FusionDictionary to a file.
- *
- * This is the public entry point to write a dictionary to a file.
- *
- * @param destination the stream to write the binary data to.
- * @param dict the dictionary to write.
- * @param formatOptions file format options.
- */
- public static void writeDictionaryBinary(final OutputStream destination,
- final FusionDictionary dict, final FormatOptions formatOptions)
- throws IOException, UnsupportedFormatException {
-
- // Addresses are limited to 3 bytes, but since addresses can be relative to each node, the
- // structure itself is not limited to 16MB. However, if it is over 16MB deciding the order
- // of the nodes becomes a quite complicated problem, because though the dictionary itself
- // does not have a size limit, each node must still be within 16MB of all its children and
- // parents. As long as this is ensured, the dictionary file may grow to any size.
-
- final int version = formatOptions.mVersion;
- if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
- || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
- throw new UnsupportedFormatException("Requested file format version " + version
- + ", but this implementation only supports versions "
- + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through "
- + FormatSpec.MAXIMUM_SUPPORTED_VERSION);
- }
-
- ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256);
-
- // The magic number in big-endian order.
- // Magic number for all versions.
- headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 24)));
- headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 16)));
- headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 8)));
- headerBuffer.write((byte) (0xFF & FormatSpec.MAGIC_NUMBER));
- // Dictionary version.
- headerBuffer.write((byte) (0xFF & (version >> 8)));
- headerBuffer.write((byte) (0xFF & version));
-
- // Options flags
- final int options = makeOptionsValue(dict, formatOptions);
- headerBuffer.write((byte) (0xFF & (options >> 8)));
- headerBuffer.write((byte) (0xFF & options));
- final int headerSizeOffset = headerBuffer.size();
- // Placeholder to be written later with header size.
- for (int i = 0; i < 4; ++i) {
- headerBuffer.write(0);
- }
- // Write out the options.
- for (final String key : dict.mOptions.mAttributes.keySet()) {
- final String value = dict.mOptions.mAttributes.get(key);
- CharEncoding.writeString(headerBuffer, key);
- CharEncoding.writeString(headerBuffer, value);
- }
- final int size = headerBuffer.size();
- final byte[] bytes = headerBuffer.toByteArray();
- // Write out the header size.
- bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24));
- bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16));
- bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8));
- bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0));
- destination.write(bytes);
-
- headerBuffer.close();
-
- // Leave the choice of the optimal node order to the flattenTree function.
- MakedictLog.i("Flattening the tree...");
- ArrayList<Node> flatNodes = flattenTree(dict.mRoot);
-
- MakedictLog.i("Computing addresses...");
- computeAddresses(dict, flatNodes, formatOptions);
- MakedictLog.i("Checking array...");
- if (DBG) checkFlatNodeArray(flatNodes);
-
- // Create a buffer that matches the final dictionary size.
- final Node lastNode = flatNodes.get(flatNodes.size() - 1);
- final int bufferSize = lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize;
- final byte[] buffer = new byte[bufferSize];
- int index = 0;
-
- MakedictLog.i("Writing file...");
- int dataEndOffset = 0;
- for (Node n : flatNodes) {
- dataEndOffset = writePlacedNode(dict, buffer, n, formatOptions);
- }
-
- if (DBG) showStatistics(flatNodes);
-
- destination.write(buffer, 0, dataEndOffset);
-
- destination.close();
- MakedictLog.i("Done");
- }
-
-
- // Input methods: Read a binary dictionary to memory.
- // readDictionaryBinary is the public entry point for them.
-
- static int getChildrenAddressSize(final int optionFlags,
- final FormatOptions formatOptions) {
- if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
- switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
- return 1;
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
- return 2;
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
- return 3;
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
- default:
- return 0;
- }
- }
-
- static int readChildrenAddress(final FusionDictionaryBufferInterface buffer,
- final int optionFlags, final FormatOptions options) {
- if (options.mSupportsDynamicUpdate) {
- final int address = buffer.readUnsignedInt24();
- if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS;
- if ((address & FormatSpec.MSB24) != 0) {
- return -(address & FormatSpec.SINT24_MAX);
- } else {
- return address;
- }
- }
- int address;
- switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
- return buffer.readUnsignedByte();
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
- return buffer.readUnsignedShort();
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
- return buffer.readUnsignedInt24();
- case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
- default:
- return FormatSpec.NO_CHILDREN_ADDRESS;
- }
- }
-
- static int readParentAddress(final FusionDictionaryBufferInterface buffer,
- final FormatOptions formatOptions) {
- if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
- final int parentAddress = buffer.readUnsignedInt24();
- final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1;
- return sign * (parentAddress & FormatSpec.SINT24_MAX);
- } else {
- return FormatSpec.NO_PARENT_ADDRESS;
- }
- }
-
- private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH];
- public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
- final int originalGroupAddress, final FormatOptions options) {
- int addressPointer = originalGroupAddress;
- final int flags = buffer.readUnsignedByte();
- ++addressPointer;
-
- final int parentAddress = readParentAddress(buffer, options);
- if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
- addressPointer += 3;
- }
-
- final int characters[];
- if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) {
- int index = 0;
- int character = CharEncoding.readChar(buffer);
- addressPointer += CharEncoding.getCharSize(character);
- while (-1 != character) {
- // FusionDictionary is making sure that the length of the word is smaller than
- // MAX_WORD_LENGTH.
- // So we'll never write past the end of CHARACTER_BUFFER.
- CHARACTER_BUFFER[index++] = character;
- character = CharEncoding.readChar(buffer);
- addressPointer += CharEncoding.getCharSize(character);
- }
- characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index);
- } else {
- final int character = CharEncoding.readChar(buffer);
- addressPointer += CharEncoding.getCharSize(character);
- characters = new int[] { character };
- }
- final int frequency;
- if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) {
- ++addressPointer;
- frequency = buffer.readUnsignedByte();
- } else {
- frequency = CharGroup.NOT_A_TERMINAL;
- }
- int childrenAddress = readChildrenAddress(buffer, flags, options);
- if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
- childrenAddress += addressPointer;
- }
- addressPointer += getChildrenAddressSize(flags, options);
- ArrayList<WeightedString> shortcutTargets = null;
- if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) {
- final int pointerBefore = buffer.position();
- shortcutTargets = new ArrayList<WeightedString>();
- buffer.readUnsignedShort(); // Skip the size
- while (true) {
- final int targetFlags = buffer.readUnsignedByte();
- final String word = CharEncoding.readString(buffer);
- shortcutTargets.add(new WeightedString(word,
- targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY));
- if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
- }
- addressPointer += buffer.position() - pointerBefore;
- }
- ArrayList<PendingAttribute> bigrams = null;
- if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
- bigrams = new ArrayList<PendingAttribute>();
- int bigramCount = 0;
- while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
- final int bigramFlags = buffer.readUnsignedByte();
- ++addressPointer;
- final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE)
- ? 1 : -1;
- int bigramAddress = addressPointer;
- switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) {
- case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
- bigramAddress += sign * buffer.readUnsignedByte();
- addressPointer += 1;
- break;
- case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
- bigramAddress += sign * buffer.readUnsignedShort();
- addressPointer += 2;
- break;
- case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
- final int offset = (buffer.readUnsignedByte() << 16)
- + buffer.readUnsignedShort();
- bigramAddress += sign * offset;
- addressPointer += 3;
- break;
- default:
- throw new RuntimeException("Has bigrams with no address");
- }
- bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY,
- bigramAddress));
- if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
- }
- if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
- MakedictLog.d("too many bigrams in a group.");
- }
- }
- return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency,
- parentAddress, childrenAddress, shortcutTargets, bigrams);
- }
-
- /**
- * Reads and returns the char group count out of a buffer and forwards the pointer.
- */
- public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
- final int msb = buffer.readUnsignedByte();
- if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) {
- return msb;
- } else {
- return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8)
- + buffer.readUnsignedByte();
- }
- }
-
- // The word cache here is a stopgap bandaid to help the catastrophic performance
- // of this method. Since it performs direct, unbuffered random access to the file and
- // may be called hundreds of thousands of times, the resulting performance is not
- // reasonable without some kind of cache. Thus:
- private static TreeMap<Integer, WeightedString> wordCache =
- new TreeMap<Integer, WeightedString>();
- /**
- * Finds, as a string, the word at the address passed as an argument.
- *
- * @param buffer the buffer to read from.
- * @param headerSize the size of the header.
- * @param address the address to seek.
- * @param formatOptions file format options.
- * @return the word with its frequency, as a weighted string.
- */
- /* package for tests */ static WeightedString getWordAtAddress(
- final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
- final FormatOptions formatOptions) {
- final WeightedString cachedString = wordCache.get(address);
- if (null != cachedString) return cachedString;
-
- final WeightedString result;
- final int originalPointer = buffer.position();
- buffer.position(address);
-
- if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
- result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions);
- } else {
- result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address,
- formatOptions);
- }
-
- wordCache.put(address, result);
- buffer.position(originalPointer);
- return result;
- }
-
- // TODO: static!? This will behave erratically when used in multi-threaded code.
- // We need to fix this
- private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH];
- @SuppressWarnings("unused")
- private static WeightedString getWordAtAddressWithParentAddress(
- final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
- final FormatOptions options) {
- int currentAddress = address;
- int index = FormatSpec.MAX_WORD_LENGTH - 1;
- int frequency = Integer.MIN_VALUE;
- // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH
- for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) {
- CharGroupInfo currentInfo;
- int loopCounter = 0;
- do {
- buffer.position(currentAddress + headerSize);
- currentInfo = readCharGroup(buffer, currentAddress, options);
- if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) {
- currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
- }
- if (DBG && loopCounter++ > MAX_JUMPS) {
- MakedictLog.d("Too many jumps - probably a bug");
- }
- } while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options));
- if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency;
- for (int i = 0; i < currentInfo.mCharacters.length; ++i) {
- sGetWordBuffer[index--] =
- currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1];
- }
- if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break;
- currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
- }
-
- return new WeightedString(
- new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1),
- frequency);
- }
-
- private static WeightedString getWordAtAddressWithoutParentAddress(
- final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
- final FormatOptions options) {
- buffer.position(headerSize);
- final int count = readCharGroupCount(buffer);
- int groupOffset = BinaryDictIOUtils.getGroupCountSize(count);
- final StringBuilder builder = new StringBuilder();
- WeightedString result = null;
-
- CharGroupInfo last = null;
- for (int i = count - 1; i >= 0; --i) {
- CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
- groupOffset = info.mEndAddress;
- if (info.mOriginalAddress == address) {
- builder.append(new String(info.mCharacters, 0, info.mCharacters.length));
- result = new WeightedString(builder.toString(), info.mFrequency);
- break; // and return
- }
- if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
- if (info.mChildrenAddress > address) {
- if (null == last) continue;
- builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
- buffer.position(last.mChildrenAddress + headerSize);
- i = readCharGroupCount(buffer);
- groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
- last = null;
- continue;
- }
- last = info;
- }
- if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) {
- builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
- buffer.position(last.mChildrenAddress + headerSize);
- i = readCharGroupCount(buffer);
- groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
- last = null;
- continue;
- }
- }
- return result;
- }
-
- /**
- * Reads a single node from a buffer.
- *
- * This methods reads the file at the current position. A node is fully expected to start at
- * the current position.
- * This will recursively read other nodes into the structure, populating the reverse
- * maps on the fly and using them to keep track of already read nodes.
- *
- * @param buffer the buffer, correctly positioned at the start of a node.
- * @param headerSize the size, in bytes, of the file header.
- * @param reverseNodeMap a mapping from addresses to already read nodes.
- * @param reverseGroupMap a mapping from addresses to already read character groups.
- * @param options file format options.
- * @return the read node with all his children already read.
- */
- private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize,
- final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap,
- final FormatOptions options)
- throws IOException {
- final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>();
- final int nodeOrigin = buffer.position() - headerSize;
-
- do { // Scan the linked-list node.
- final int nodeHeadPosition = buffer.position() - headerSize;
- final int count = readCharGroupCount(buffer);
- int groupOffset = nodeHeadPosition + BinaryDictIOUtils.getGroupCountSize(count);
- for (int i = count; i > 0; --i) { // Scan the array of CharGroup.
- CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
- if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue;
- ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets;
- ArrayList<WeightedString> bigrams = null;
- if (null != info.mBigrams) {
- bigrams = new ArrayList<WeightedString>();
- for (PendingAttribute bigram : info.mBigrams) {
- final WeightedString word = getWordAtAddress(
- buffer, headerSize, bigram.mAddress, options);
- final int reconstructedFrequency =
- reconstructBigramFrequency(word.mFrequency, bigram.mFrequency);
- bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
- }
- }
- if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
- Node children = reverseNodeMap.get(info.mChildrenAddress);
- if (null == children) {
- final int currentPosition = buffer.position();
- buffer.position(info.mChildrenAddress + headerSize);
- children = readNode(
- buffer, headerSize, reverseNodeMap, reverseGroupMap, options);
- buffer.position(currentPosition);
- }
- nodeContents.add(
- new CharGroup(info.mCharacters, shortcutTargets, bigrams,
- info.mFrequency,
- 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
- 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children));
- } else {
- nodeContents.add(
- new CharGroup(info.mCharacters, shortcutTargets, bigrams,
- info.mFrequency,
- 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
- 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED)));
- }
- groupOffset = info.mEndAddress;
- }
-
- // reach the end of the array.
- if (options.mSupportsDynamicUpdate) {
- final int nextAddress = buffer.readUnsignedInt24();
- if (nextAddress >= 0 && nextAddress < buffer.limit()) {
- buffer.position(nextAddress);
- } else {
- break;
- }
- }
- } while (options.mSupportsDynamicUpdate &&
- buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS);
-
- final Node node = new Node(nodeContents);
- node.mCachedAddressBeforeUpdate = nodeOrigin;
- node.mCachedAddressAfterUpdate = nodeOrigin;
- reverseNodeMap.put(node.mCachedAddressAfterUpdate, node);
- return node;
- }
-
- /**
- * Helper function to get the binary format version from the header.
- * @throws IOException
- */
- private static int getFormatVersion(final FusionDictionaryBufferInterface buffer)
- throws IOException {
- final int magic = buffer.readInt();
- if (FormatSpec.MAGIC_NUMBER == magic) return buffer.readUnsignedShort();
- return FormatSpec.NOT_A_VERSION_NUMBER;
- }
-
- /**
- * Helper function to get and validate the binary format version.
- * @throws UnsupportedFormatException
- * @throws IOException
- */
- private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer)
- throws IOException, UnsupportedFormatException {
- final int version = getFormatVersion(buffer);
- if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
- || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
- throw new UnsupportedFormatException("This file has version " + version
- + ", but this implementation does not support versions above "
- + FormatSpec.MAXIMUM_SUPPORTED_VERSION);
- }
- return version;
- }
-
- /**
- * Reads a header from a buffer.
- * @param buffer the buffer to read.
- * @throws IOException
- * @throws UnsupportedFormatException
- */
- public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer)
- throws IOException, UnsupportedFormatException {
- final int version = checkFormatVersion(buffer);
- final int optionsFlags = buffer.readUnsignedShort();
-
- final HashMap<String, String> attributes = new HashMap<String, String>();
- final int headerSize;
- headerSize = buffer.readInt();
-
- if (headerSize < 0) {
- throw new UnsupportedFormatException("header size can't be negative.");
- }
-
- populateOptions(buffer, headerSize, attributes);
- buffer.position(headerSize);
-
- final FileHeader header = new FileHeader(headerSize,
- new FusionDictionary.DictionaryOptions(attributes,
- 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG),
- 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)),
- new FormatOptions(version,
- 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE)));
- return header;
- }
-
- /**
- * Reads options from a buffer and populate a map with their contents.
- *
- * The buffer is read at the current position, so the caller must take care the pointer
- * is in the right place before calling this.
- */
- public static void populateOptions(final FusionDictionaryBufferInterface buffer,
- final int headerSize, final HashMap<String, String> options) {
- while (buffer.position() < headerSize) {
- final String key = CharEncoding.readString(buffer);
- final String value = CharEncoding.readString(buffer);
- options.put(key, value);
- }
- }
-
- /**
- * Reads a buffer and returns the memory representation of the dictionary.
- *
- * This high-level method takes a buffer and reads its contents, populating a
- * FusionDictionary structure. The optional dict argument is an existing dictionary to
- * which words from the buffer should be added. If it is null, a new dictionary is created.
- *
- * @param reader the reader.
- * @param dict an optional dictionary to add words to, or null.
- * @return the created (or merged) dictionary.
- */
- @UsedForTesting
- public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
- final FusionDictionary dict) throws FileNotFoundException, IOException,
- UnsupportedFormatException {
- // clear cache
- wordCache.clear();
-
- // if the buffer has not been opened, open the buffer with bytebuffer.
- if (reader.getBuffer() == null) reader.openBuffer(
- new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
- if (reader.getBuffer() == null) {
- MakedictLog.e("Cannot open the buffer");
- }
-
- // Read header
- final FileHeader header = readHeader(reader.getBuffer());
-
- Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
- Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
- final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
- reverseGroupMapping, header.mFormatOptions);
-
- FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);
- if (null != dict) {
- for (final Word w : dict) {
- if (w.mIsBlacklistEntry) {
- newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord);
- } else {
- newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord);
- }
- }
- for (final Word w : dict) {
- // By construction a binary dictionary may not have bigrams pointing to
- // words that are not also registered as unigrams so we don't have to avoid
- // them explicitly here.
- for (final WeightedString bigram : w.mBigrams) {
- newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency);
- }
- }
- }
-
- return newDict;
- }
-
- /**
- * Helper method to pass a file name instead of a File object to isBinaryDictionary.
- */
- public static boolean isBinaryDictionary(final String filename) {
- final File file = new File(filename);
- return isBinaryDictionary(file);
- }
-
- /**
- * Basic test to find out whether the file is a binary dictionary or not.
- *
- * Concretely this only tests the magic number.
- *
- * @param file The file to test.
- * @return true if it's a binary dictionary, false otherwise
- */
- public static boolean isBinaryDictionary(final File file) {
- FileInputStream inStream = null;
- try {
- inStream = new FileInputStream(file);
- final ByteBuffer buffer = inStream.getChannel().map(
- FileChannel.MapMode.READ_ONLY, 0, file.length());
- final int version = getFormatVersion(new ByteBufferWrapper(buffer));
- return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION
- && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION);
- } catch (FileNotFoundException e) {
- return false;
- } catch (IOException e) {
- return false;
- } finally {
- if (inStream != null) {
- try {
- inStream.close();
- } catch (IOException e) {
- // do nothing
- }
- }
- }
- }
-
- /**
- * Calculate bigram frequency from compressed value
- *
- * @see #makeBigramFlags
- *
- * @param unigramFrequency
- * @param bigramFrequency compressed frequency
- * @return approximate bigram frequency
- */
- public static int reconstructBigramFrequency(final int unigramFrequency,
- final int bigramFrequency) {
- final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
- / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
- final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f);
- return (int)resultFreqFloat;
- }
-}
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java
index a4a7ce4..f2f3c46 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java
@@ -17,7 +17,7 @@
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import java.io.File;
@@ -55,7 +55,7 @@
}
}
if (buffer != null) {
- return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
+ return new BinaryDictDecoder.ByteBufferWrapper(buffer);
}
return null;
}
@@ -103,7 +103,7 @@
}
}
if (buffer != null) {
- return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
+ return new BinaryDictDecoder.ByteBufferWrapper(buffer);
}
return null;
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
index 5b10912..4dbfcb7 100644
--- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
@@ -18,7 +18,7 @@
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
@@ -58,7 +58,7 @@
public static void deleteWord(final FusionDictionaryBufferInterface buffer,
final String word) throws IOException, UnsupportedFormatException {
buffer.position(0);
- final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
+ final FileHeader header = BinaryDictDecoder.readHeader(buffer);
final int wordPosition = BinaryDictIOUtils.getTerminalPosition(buffer, word);
if (wordPosition == FormatSpec.NOT_VALID_WORD) return;
@@ -114,7 +114,7 @@
final int originalPosition = buffer.position();
buffer.position(nodeOriginAddress);
do {
- final int count = BinaryDictInputOutput.readCharGroupCount(buffer);
+ final int count = BinaryDictDecoder.readCharGroupCount(buffer);
for (int i = 0; i < count; ++i) {
updateParentAddress(buffer, buffer.position(), newParentAddress, formatOptions);
BinaryDictIOUtils.skipCharGroup(buffer, formatOptions);
@@ -140,7 +140,7 @@
final int originalPosition = buffer.position();
buffer.position(groupOriginAddress);
final int flags = buffer.readUnsignedByte();
- final int parentAddress = BinaryDictInputOutput.readParentAddress(buffer, formatOptions);
+ final int parentAddress = BinaryDictDecoder.readParentAddress(buffer, formatOptions);
BinaryDictIOUtils.skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte();
final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS
@@ -174,7 +174,7 @@
buffer.position(nodeOriginAddress);
int jumpCount = 0;
while (jumpCount++ < MAX_JUMPS) {
- final int count = BinaryDictInputOutput.readCharGroupCount(buffer);
+ final int count = BinaryDictDecoder.readCharGroupCount(buffer);
for (int i = 0; i < count; ++i) BinaryDictIOUtils.skipCharGroup(buffer, formatOptions);
final int forwardLinkAddress = buffer.readUnsignedInt24();
if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) {
@@ -269,7 +269,7 @@
// find the insert position of the word.
if (buffer.position() != 0) buffer.position(0);
- final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
+ final FileHeader header = BinaryDictDecoder.readHeader(buffer);
int wordPos = 0, address = buffer.position(), nodeOriginAddress = buffer.position();
final int[] codePoints = FusionDictionary.getCodePoints(word);
@@ -279,12 +279,12 @@
if (wordPos >= wordLen) break;
nodeOriginAddress = buffer.position();
int nodeParentAddress = -1;
- final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer);
+ final int charGroupCount = BinaryDictDecoder.readCharGroupCount(buffer);
boolean foundNextGroup = false;
for (int i = 0; i < charGroupCount; ++i) {
address = buffer.position();
- final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer,
+ final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer,
buffer.position(), header.mFormatOptions);
final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags,
header.mFormatOptions);
@@ -305,7 +305,7 @@
* abc - d - ef
*/
final int newNodeAddress = buffer.limit();
- final int flags = BinaryDictInputOutput.makeCharGroupFlags(p > 1,
+ final int flags = BinaryDictEncoder.makeCharGroupFlags(p > 1,
isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */,
false /* isBlackListEntry */, header.mFormatOptions);
int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags,
@@ -344,7 +344,7 @@
final int childrenAddress = currentInfo.mChildrenAddress;
// move prefix
- final int prefixFlags = BinaryDictInputOutput.makeCharGroupFlags(p > 1,
+ final int prefixFlags = BinaryDictEncoder.makeCharGroupFlags(p > 1,
false /* isTerminal */, 0 /* childrenAddressSize*/,
false /* hasShortcut */, false /* hasBigrams */,
false /* isNotAWord */, false /* isBlackListEntry */,
@@ -360,7 +360,7 @@
updateParentAddresses(buffer, currentInfo.mChildrenAddress,
newNodeAddress + written + 1, header.mFormatOptions);
}
- final int suffixFlags = BinaryDictInputOutput.makeCharGroupFlags(
+ final int suffixFlags = BinaryDictEncoder.makeCharGroupFlags(
suffixCharacters.length > 1,
(currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0,
0 /* childrenAddressSize */,
@@ -378,7 +378,7 @@
final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p,
codePoints.length);
- final int flags = BinaryDictInputOutput.makeCharGroupFlags(
+ final int flags = BinaryDictEncoder.makeCharGroupFlags(
newCharacters.length > 1, isTerminal,
0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
isNotAWord, isBlackListEntry, header.mFormatOptions);
@@ -401,7 +401,7 @@
// only update group.
final int newNodeAddress = buffer.limit();
final boolean hasMultipleChars = currentInfo.mCharacters.length > 1;
- final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars,
+ final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars,
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
isNotAWord, isBlackListEntry, header.mFormatOptions);
final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1,
@@ -431,7 +431,7 @@
header.mFormatOptions);
final int newGroupAddress = newNodeAddress + 1;
final boolean hasMultipleChars = (wordLen - wordPos) > 1;
- final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars,
+ final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars,
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
isNotAWord, isBlackListEntry, header.mFormatOptions);
final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen);
@@ -476,7 +476,7 @@
BinaryDictIOUtils.writeSInt24ToBuffer(buffer, newNodeAddress);
final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen);
- final int flags = BinaryDictInputOutput.makeCharGroupFlags(characters.length > 1,
+ final int flags = BinaryDictEncoder.makeCharGroupFlags(characters.length > 1,
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
isNotAWord, isBlackListEntry, header.mFormatOptions);
final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1,
diff --git a/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java b/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java
index 1bb27aa..d93b614 100644
--- a/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java
+++ b/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java
@@ -16,7 +16,7 @@
package com.android.inputmethod.latin.utils;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
/**
* This class provides an implementation for the FusionDictionary buffer interface that is backed
diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
index a0ad27c..aed07fd 100644
--- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
@@ -19,8 +19,9 @@
import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
+import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
@@ -62,7 +63,7 @@
final FormatOptions formatOptions) {
final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams);
try {
- BinaryDictInputOutput.writeDictionaryBinary(destination, fusionDict, formatOptions);
+ BinaryDictEncoder.writeDictionaryBinary(destination, fusionDict, formatOptions);
Log.d(TAG, "end writing");
} catch (IOException e) {
Log.e(TAG, "IO exception while writing file", e);
@@ -156,7 +157,7 @@
continue;
}
to.setBigram(word1, word2,
- BinaryDictInputOutput.reconstructBigramFrequency(unigramFrequency,
+ BinaryDictDecoder.reconstructBigramFrequency(unigramFrequency,
attr.mFrequency));
}
}
diff --git a/native/jni/Android.mk b/native/jni/Android.mk
index 8d51a2f..9e7407a 100644
--- a/native/jni/Android.mk
+++ b/native/jni/Android.mk
@@ -43,7 +43,7 @@
com_android_inputmethod_keyboard_ProximityInfo.cpp \
com_android_inputmethod_latin_BinaryDictionary.cpp \
com_android_inputmethod_latin_DicTraverseSession.cpp \
- com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp \
+ com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp \
jni_common.cpp
LATIN_IME_CORE_SRC_FILES := \
diff --git a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp
similarity index 75%
rename from native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp
rename to native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp
index f78883c..457b226 100644
--- a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp
+++ b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp
@@ -14,16 +14,16 @@
* limitations under the License.
*/
-#define LOG_TAG "LatinIME: jni: BinaryDictInputOutput"
+#define LOG_TAG "LatinIME: jni: BinaryDictDecoder"
-#include "com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h"
+#include "com_android_inputmethod_latin_makedict_BinaryDictDecoder.h"
#include "defines.h"
#include "jni.h"
#include "jni_common.h"
namespace latinime {
-static int latinime_BinaryDictInputOutput_doNothing(JNIEnv *env, jclass clazz) {
+static int latinime_BinaryDictDecoder_doNothing(JNIEnv *env, jclass clazz) {
// This is a phony method for test - it does nothing. It just returns some value
// unlikely to be in memory by chance for testing purposes.
// TODO: remove this method.
@@ -35,13 +35,13 @@
// TODO: remove this entry when we have one useful method in here
const_cast<char *>("doNothing"),
const_cast<char *>("()I"),
- reinterpret_cast<void *>(latinime_BinaryDictInputOutput_doNothing)
+ reinterpret_cast<void *>(latinime_BinaryDictDecoder_doNothing)
},
};
-int register_BinaryDictInputOutput(JNIEnv *env) {
+int register_BinaryDictDecoder(JNIEnv *env) {
const char *const kClassPathName =
- "com/android/inputmethod/latin/makedict/BinaryDictInputOutput";
+ "com/android/inputmethod/latin/makedict/BinaryDictDecoder";
return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods));
}
} // namespace latinime
diff --git a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.h
similarity index 71%
rename from native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h
rename to native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.h
index e622ed4..7f3cb67 100644
--- a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h
+++ b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.h
@@ -14,12 +14,12 @@
* limitations under the License.
*/
-#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H
-#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H
+#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H
+#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H
#include "jni.h"
namespace latinime {
-int register_BinaryDictInputOutput(JNIEnv *env);
+int register_BinaryDictDecoder(JNIEnv *env);
} // namespace latinime
-#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H
+#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H
diff --git a/native/jni/jni_common.cpp b/native/jni/jni_common.cpp
index 733e15f..d44be67 100644
--- a/native/jni/jni_common.cpp
+++ b/native/jni/jni_common.cpp
@@ -23,7 +23,7 @@
#include "com_android_inputmethod_latin_BinaryDictionary.h"
#include "com_android_inputmethod_latin_DicTraverseSession.h"
#endif
-#include "com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h"
+#include "com_android_inputmethod_latin_makedict_BinaryDictDecoder.h"
#include "defines.h"
/*
@@ -55,8 +55,8 @@
return -1;
}
#endif
- if (!latinime::register_BinaryDictInputOutput(env)) {
- AKLOGE("ERROR: BinaryDictInputOutput native registration failed");
+ if (!latinime::register_BinaryDictDecoder(env)) {
+ AKLOGE("ERROR: BinaryDictDecoder native registration failed");
return -1;
}
/* success -- return valid version number */
diff --git a/native/jni/src/suggest/core/dictionary/probability_utils.h b/native/jni/src/suggest/core/dictionary/probability_utils.h
index f450087..2192135 100644
--- a/native/jni/src/suggest/core/dictionary/probability_utils.h
+++ b/native/jni/src/suggest/core/dictionary/probability_utils.h
@@ -41,7 +41,7 @@
// the unigram probability to be the median value of the 17th step from the top. A value of
// 0 for the bigram probability represents the middle of the 16th step from the top,
// while a value of 15 represents the middle of the top step.
- // See makedict.BinaryDictInputOutput for details.
+ // See makedict.BinaryDictDecoder for details.
const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability)
/ (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY);
return unigramProbability
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
similarity index 96%
rename from tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
rename to tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
index 7bfd603..9664779 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
@@ -22,7 +22,7 @@
import android.util.Log;
import android.util.SparseArray;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
@@ -44,11 +44,11 @@
import java.util.Set;
/**
- * Unit tests for BinaryDictInputOutput
+ * Unit tests for BinaryDictDecoder and BinaryDictEncoder.
*/
@LargeTest
-public class BinaryDictIOTests extends AndroidTestCase {
- private static final String TAG = BinaryDictIOTests.class.getSimpleName();
+public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
+ private static final String TAG = BinaryDictDecoderEncoderTests.class.getSimpleName();
private static final int DEFAULT_MAX_UNIGRAMS = 100;
private static final int DEFAULT_CODE_POINT_SET_SIZE = 50;
private static final int UNIGRAM_FREQ = 10;
@@ -71,11 +71,11 @@
private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */);
- public BinaryDictIOTests() {
+ public BinaryDictDecoderEncoderTests() {
this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS);
}
- public BinaryDictIOTests(final long seed, final int maxUnigrams) {
+ public BinaryDictDecoderEncoderTests(final long seed, final int maxUnigrams) {
super();
Log.e(TAG, "Testing dictionary: seed is " + seed);
final Random random = new Random(seed);
@@ -206,7 +206,7 @@
// If you need to dump the dict to a textual file, uncomment the line below and the
// function above
// dumpToCombinedFileForDebug(file, "/tmp/foo");
- BinaryDictInputOutput.writeDictionaryBinary(out, dict, formatOptions);
+ BinaryDictEncoder.writeDictionaryBinary(out, dict, formatOptions);
diff = System.currentTimeMillis() - now;
out.flush();
@@ -272,7 +272,7 @@
getBuffer(reader, bufferType);
assertNotNull(reader.getBuffer());
now = System.currentTimeMillis();
- dict = BinaryDictInputOutput.readDictionaryBinary(reader, null);
+ dict = BinaryDictDecoder.readDictionaryBinary(reader, null);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {
Log.e(TAG, "IOException while reading dictionary", e);
@@ -383,7 +383,7 @@
}
actBigrams.get(word1).add(word2);
- final int bigramFreq = BinaryDictInputOutput.reconstructBigramFrequency(
+ final int bigramFreq = BinaryDictDecoder.reconstructBigramFrequency(
unigramFreq, attr.mFrequency);
assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ);
}
@@ -497,14 +497,14 @@
FileHeader header = null;
try {
- header = BinaryDictInputOutput.readHeader(buffer);
+ header = BinaryDictDecoder.readHeader(buffer);
} catch (IOException e) {
return null;
} catch (UnsupportedFormatException e) {
return null;
}
if (header == null) return null;
- return BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize,
+ return BinaryDictDecoder.getWordAtAddress(buffer, header.mHeaderSize,
address - header.mHeaderSize, header.mFormatOptions).mWord;
}
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
index e759507..f2476b2 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
@@ -21,8 +21,8 @@
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.ByteBufferWrapper;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.ByteBufferWrapper;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
@@ -114,10 +114,10 @@
private static void printNode(final FusionDictionaryBufferInterface buffer,
final FormatSpec.FormatOptions formatOptions) {
Log.d(TAG, "Node at " + buffer.position());
- final int count = BinaryDictInputOutput.readCharGroupCount(buffer);
+ final int count = BinaryDictDecoder.readCharGroupCount(buffer);
Log.d(TAG, " charGroupCount = " + count);
for (int i = 0; i < count; ++i) {
- final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer,
+ final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer,
buffer.position(), formatOptions);
printCharGroup(currentInfo);
}
@@ -129,7 +129,7 @@
private static void printBinaryFile(final FusionDictionaryBufferInterface buffer)
throws IOException, UnsupportedFormatException {
- FileHeader header = BinaryDictInputOutput.readHeader(buffer);
+ FileHeader header = BinaryDictDecoder.readHeader(buffer);
while (buffer.position() < buffer.limit()) {
printNode(buffer, header.mFormatOptions);
}
@@ -252,8 +252,8 @@
inStream = new FileInputStream(file);
final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper(
inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length()));
- final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
- assertEquals(word, BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize,
+ final FileHeader header = BinaryDictDecoder.readHeader(buffer);
+ assertEquals(word, BinaryDictDecoder.getWordAtAddress(buffer, header.mHeaderSize,
position - header.mHeaderSize, header.mFormatOptions).mWord);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
@@ -283,7 +283,7 @@
try {
final FileOutputStream out = new FileOutputStream(file);
- BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
+ BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
out.close();
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
@@ -335,7 +335,7 @@
try {
final FileOutputStream out = new FileOutputStream(file);
- BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
+ BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
out.close();
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
@@ -372,7 +372,7 @@
try {
final FileOutputStream out = new FileOutputStream(file);
- BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
+ BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
out.close();
} catch (IOException e) {
assertTrue(false);
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java
index a46e583..1c6de50 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java
@@ -16,7 +16,7 @@
package com.android.inputmethod.latin.makedict;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory;
import com.android.inputmethod.latin.makedict.BinaryDictReader.
FusionDictionaryBufferFromByteArrayFactory;
diff --git a/tools/dicttool/NativeLib.mk b/tools/dicttool/NativeLib.mk
index eeae003..84e54a9 100644
--- a/tools/dicttool/NativeLib.mk
+++ b/tools/dicttool/NativeLib.mk
@@ -34,7 +34,7 @@
# Used in jni_common.cpp to avoid registering useless methods.
LATIN_IME_JNI_SRC_FILES := \
- com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp \
+ com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp \
jni_common.cpp
LATIN_IME_CORE_SRC_FILES :=
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
index d0b460a..f9a599c 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
@@ -16,7 +16,7 @@
package com.android.inputmethod.latin.dicttool;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
@@ -99,7 +99,7 @@
// over and over, ending in a stack overflow. Hence we limit the depth at which we try
// decoding the file.
if (depth > MAX_DECODE_DEPTH) return null;
- if (BinaryDictInputOutput.isBinaryDictionary(src)) {
+ if (BinaryDictDecoder.isBinaryDictionary(src)) {
spec.mFile = src;
return spec;
}
@@ -194,7 +194,7 @@
System.out.println("Packaging : " + decodedSpec.describeChain());
System.out.println("Uncompressed size : " + decodedSpec.mFile.length());
}
- return BinaryDictInputOutput.readDictionaryBinary(reader, null);
+ return BinaryDictDecoder.readDictionaryBinary(reader, null);
}
}
} catch (IOException e) {
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
index 9bce988..fa267f1 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
@@ -16,7 +16,8 @@
package com.android.inputmethod.latin.dicttool;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
+import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
@@ -177,7 +178,7 @@
inputUnigramXml = filename;
} else if (CombinedInputOutput.isCombinedDictionary(filename)) {
inputCombined = filename;
- } else if (BinaryDictInputOutput.isBinaryDictionary(filename)) {
+ } else if (BinaryDictDecoder.isBinaryDictionary(filename)) {
inputBinary = filename;
} else {
throw new IllegalArgumentException(
@@ -199,7 +200,7 @@
}
} else {
if (null == inputBinary && null == inputUnigramXml) {
- if (BinaryDictInputOutput.isBinaryDictionary(arg)) {
+ if (BinaryDictDecoder.isBinaryDictionary(arg)) {
inputBinary = arg;
} else if (CombinedInputOutput.isCombinedDictionary(arg)) {
inputCombined = arg;
@@ -269,7 +270,7 @@
final File file = new File(binaryFilename);
final BinaryDictReader reader = new BinaryDictReader(file);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
- return BinaryDictInputOutput.readDictionaryBinary(reader, null);
+ return BinaryDictDecoder.readDictionaryBinary(reader, null);
}
/**
@@ -358,7 +359,7 @@
throws FileNotFoundException, IOException, UnsupportedFormatException {
final File outputFile = new File(outputFilename);
final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version);
- BinaryDictInputOutput.writeDictionaryBinary(new FileOutputStream(outputFilename), dict,
+ BinaryDictEncoder.writeDictionaryBinary(new FileOutputStream(outputFilename), dict,
formatOptions);
}
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java
index 972b6e7..967e21e 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java
@@ -16,9 +16,9 @@
package com.android.inputmethod.latin.dicttool;
-import com.android.inputmethod.latin.makedict.BinaryDictIOTests;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoderEncoderTests;
+import com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtilsTests;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest;
import com.android.inputmethod.latin.makedict.FusionDictionaryTest;
import java.lang.reflect.Constructor;
@@ -37,9 +37,9 @@
private static final Class<?>[] sClassesToTest = {
BinaryDictOffdeviceUtilsTests.class,
FusionDictionaryTest.class,
- BinaryDictInputOutputTest.class,
+ BinaryDictDecoderEncoderTests.class
+ BinaryDictEncoderFlattenTreeTests.class,
BinaryDictIOUtilsTests.class,
- BinaryDictIOTests.class
};
private ArrayList<Method> mAllTestMethods = new ArrayList<Method>();
private ArrayList<String> mUsedTestMethods = new ArrayList<String>();
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
index fb1cc84..5490aff 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
@@ -16,7 +16,8 @@
package com.android.inputmethod.latin.dicttool;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
+import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
@@ -38,7 +39,7 @@
import java.util.HashMap;
/**
- * Unit tests for BinaryDictOffdeviceUtilsTests
+ * Unit tests for BinaryDictOffdeviceUtils
*/
public class BinaryDictOffdeviceUtilsTests extends TestCase {
private static final int TEST_FREQ = 37; // Some arbitrary value unlikely to happen by chance
@@ -61,7 +62,7 @@
Compress.getCompressedStream(
new BufferedOutputStream(new FileOutputStream(dst)))));
- BinaryDictInputOutput.writeDictionaryBinary(out, dict, new FormatOptions(2, false));
+ BinaryDictEncoder.writeDictionaryBinary(out, dict, new FormatOptions(2, false));
// Test for an actually compressed dictionary and its contents
final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec =
@@ -72,7 +73,7 @@
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final BinaryDictReader reader = new BinaryDictReader(decodeSpec.mFile);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
- final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(reader,
+ final FusionDictionary resultDict = BinaryDictDecoder.readDictionaryBinary(reader,
null /* dict : an optional dictionary to add words to, or null */);
assertEquals("Dictionary can't be read back correctly",
resultDict.findWordInTree(resultDict.mRoot, "foo").getFrequency(), TEST_FREQ);
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictInputOutputTest.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java
similarity index 89%
rename from tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictInputOutputTest.java
rename to tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java
index 0969028..2fcfb5e 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictInputOutputTest.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java
@@ -25,9 +25,9 @@
import java.util.HashMap;
/**
- * Unit tests for BinaryDictInputOutput.
+ * Unit tests for BinaryDictEncoder.flattenTree().
*/
-public class BinaryDictInputOutputTest extends TestCase {
+public class BinaryDictEncoderFlattenTreeTests extends TestCase {
// Test the flattened array contains the expected number of nodes, and
// that it does not contain any duplicates.
public void testFlattenNodes() {
@@ -39,7 +39,7 @@
dict.add("ftb", 1, null, false /* isNotAWord */);
dict.add("bar", 1, null, false /* isNotAWord */);
dict.add("fool", 1, null, false /* isNotAWord */);
- final ArrayList<Node> result = BinaryDictInputOutput.flattenTree(dict.mRoot);
+ final ArrayList<Node> result = BinaryDictEncoder.flattenTree(dict.mRoot);
assertEquals(4, result.size());
while (!result.isEmpty()) {
final Node n = result.remove(0);
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
index 7607113..8efb4a4 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
@@ -29,7 +29,7 @@
import java.util.Random;
/**
- * Unit tests for BinaryDictInputOutput.
+ * Unit tests for FusionDictionary.
*/
public class FusionDictionaryTest extends TestCase {
private static final ArrayList<String> sWords = new ArrayList<String>();
diff --git a/tools/dicttool/tests/etc/test-dicttool.sh b/tools/dicttool/tests/etc/test-dicttool.sh
index 0921207..5eb44fc 100755
--- a/tools/dicttool/tests/etc/test-dicttool.sh
+++ b/tools/dicttool/tests/etc/test-dicttool.sh
@@ -24,5 +24,5 @@
find out -name "dicttool_aosp*" -exec rm -rf {} \; > /dev/null 2>&1
mmm -j8 external/junit
DICTTOOL_UNITTEST=true mmm -j8 packages/inputmethods/LatinIME/tools/dicttool
-java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest
+java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests
java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.dicttool.BinaryDictOffdeviceUtilsTests