diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
index a755f90..858aa32 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
@@ -48,6 +48,8 @@
 
     public abstract void onDictionaryReady();
 
+    public abstract void onDictionaryClosed();
+
     public void setPredictionDictionary(String locale, DynamicPredictionDictionaryBase dictionary) {
         mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary);
         mLocale = locale;
@@ -68,6 +70,7 @@
 
     public void closeSession() {
         unsetPredictionDictionary();
+        onDictionaryClosed();
     }
 
     public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,
diff --git a/java/src/com/android/inputmethod/latin/settings/Settings.java b/java/src/com/android/inputmethod/latin/settings/Settings.java
index d432087..20b6752 100644
--- a/java/src/com/android/inputmethod/latin/settings/Settings.java
+++ b/java/src/com/android/inputmethod/latin/settings/Settings.java
@@ -27,10 +27,10 @@
 import com.android.inputmethod.latin.InputAttributes;
 import com.android.inputmethod.latin.R;
 import com.android.inputmethod.latin.utils.AdditionalSubtypeUtils;
-import com.android.inputmethod.latin.utils.DebugLogUtils;
 import com.android.inputmethod.latin.utils.LocaleUtils;
 import com.android.inputmethod.latin.utils.ResourceUtils;
 import com.android.inputmethod.latin.utils.RunInLocale;
+import com.android.inputmethod.latin.utils.StringUtils;
 
 import java.util.HashMap;
 import java.util.Locale;
@@ -90,6 +90,8 @@
     private static final String PREF_SUPPRESS_LANGUAGE_SWITCH_KEY =
             "pref_suppress_language_switch_key";
 
+    private static final String PREF_LAST_USED_PERSONALIZATION_TOKEN =
+            "pref_last_used_personalization_token";
     public static final String PREF_SEND_FEEDBACK = "send_feedback";
     public static final String PREF_ABOUT_KEYBOARD = "about_keyboard";
 
@@ -343,4 +345,14 @@
         return prefs.getBoolean(
                 DebugSettings.PREF_USE_ONLY_PERSONALIZATION_DICTIONARY_FOR_DEBUG, false);
     }
+
+    public void writeLastUsedPersonalizationToken(byte[] token) {
+        final String tokenStr = StringUtils.byteArrayToHexString(token);
+        mPrefs.edit().putString(PREF_LAST_USED_PERSONALIZATION_TOKEN, tokenStr).apply();
+    }
+
+    public byte[] readLastUsedPersonalizationToken() {
+        final String tokenStr = mPrefs.getString(PREF_LAST_USED_PERSONALIZATION_TOKEN, null);
+        return StringUtils.hexStringToByteArray(tokenStr);
+    }
 }
diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
index 0b4838c..20cc5d8 100644
--- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
@@ -357,4 +357,30 @@
         }
         return true;
     }
+
+    @UsedForTesting
+    public static String byteArrayToHexString(byte[] bytes) {
+        if (bytes == null || bytes.length == 0) {
+            return "";
+        }
+        final StringBuilder sb = new StringBuilder();
+        for (byte b : bytes) {
+            sb.append(String.format("%02x", b & 0xff));
+        }
+        return sb.toString();
+    }
+
+    @UsedForTesting
+    public static byte[] hexStringToByteArray(String hexString) {
+        if (TextUtils.isEmpty(hexString)) {
+            return null;
+        }
+        final int N = hexString.length();
+        final byte[] bytes = new byte[N / 2];
+        for (int i = 0; i < N; i += 2) {
+            bytes[i / 2] = (byte) ((Character.digit(hexString.charAt(i), 16) << 4)
+                    + Character.digit(hexString.charAt(i + 1), 16));
+        }
+        return bytes;
+    }
 }
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index c895a86..86c2394 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -18,42 +18,24 @@
 
 #include "com_android_inputmethod_latin_BinaryDictionary.h"
 
-#include <cerrno>
 #include <cstring> // for memset()
-#include <fcntl.h>
-#include <sys/mman.h>
-#include <unistd.h>
 
 #include "defines.h"
 #include "jni.h"
 #include "jni_common.h"
-#include "suggest/core/dictionary/binary_dictionary_info.h"
 #include "suggest/core/dictionary/dictionary.h"
 #include "suggest/core/suggest_options.h"
-#include "suggest/policyimpl/dictionary/utils/format_utils.h"
+#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
 #include "utils/autocorrection_threshold_utils.h"
 
 namespace latinime {
 
 class ProximityInfo;
 
-// Helper method
-static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd) {
-    int ret = munmap(const_cast<void *>(dictBuf), length);
-    if (ret != 0) {
-        AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno);
-    }
-    ret = close(fd);
-    if (ret != 0) {
-        AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno);
-    }
-}
-
 static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
         jlong dictOffset, jlong dictSize, jboolean isUpdatable) {
     PROF_OPEN;
     PROF_START(66);
-    // TODO: Move dictionary buffer handling to policyimpl.
     const jsize sourceDirUtf8Length = env->GetStringUTFLength(sourceDir);
     if (sourceDirUtf8Length <= 0) {
         AKLOGE("DICT: Can't get sourceDir string");
@@ -62,41 +44,16 @@
     char sourceDirChars[sourceDirUtf8Length + 1];
     env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);
     sourceDirChars[sourceDirUtf8Length] = '\0';
-    int fd = 0;
-    void *dictBuf = 0;
-    int offset = 0;
-    const bool updatableMmap = (isUpdatable == JNI_TRUE);
-    const int openMode = updatableMmap ? O_RDWR : O_RDONLY;
-    fd = open(sourceDirChars, openMode);
-    if (fd < 0) {
-        AKLOGE("DICT: Can't open sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno);
+    DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy =
+            DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy(
+                    sourceDirChars, static_cast<int>(sourceDirUtf8Length),
+                    static_cast<int>(dictOffset), static_cast<int>(dictSize),
+                    isUpdatable == JNI_TRUE);
+    if (!dictionaryStructureWithBufferPolicy) {
         return 0;
     }
-    int pagesize = getpagesize();
-    offset = static_cast<int>(dictOffset) % pagesize;
-    int adjDictOffset = static_cast<int>(dictOffset) - offset;
-    int adjDictSize = static_cast<int>(dictSize) + offset;
-    const int protMode = updatableMmap ? PROT_READ | PROT_WRITE : PROT_READ;
-    dictBuf = mmap(0, adjDictSize, protMode, MAP_PRIVATE, fd, adjDictOffset);
-    if (dictBuf == MAP_FAILED) {
-        AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno);
-        return 0;
-    }
-    dictBuf = static_cast<char *>(dictBuf) + offset;
-    if (!dictBuf) {
-        AKLOGE("DICT: dictBuf is null");
-        return 0;
-    }
-    Dictionary *dictionary = 0;
-    if (FormatUtils::UNKNOWN_VERSION
-            == FormatUtils::detectFormatVersion(static_cast<uint8_t *>(dictBuf),
-                    static_cast<int>(dictSize))) {
-        AKLOGE("DICT: dictionary format is unknown, bad magic number");
-        releaseDictBuf(static_cast<const char *>(dictBuf) - offset, adjDictSize, fd);
-    } else {
-        dictionary = new Dictionary(env, dictBuf, static_cast<int>(dictSize), fd, offset,
-                updatableMmap);
-    }
+
+    Dictionary *const dictionary = new Dictionary(env, dictionaryStructureWithBufferPolicy);
     PROF_END(66);
     PROF_CLOSE;
     return reinterpret_cast<jlong>(dictionary);
@@ -105,13 +62,6 @@
 static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {
     Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
     if (!dictionary) return;
-    const BinaryDictionaryInfo *const binaryDictionaryInfo = dictionary->getBinaryDictionaryInfo();
-    const int dictBufOffset = binaryDictionaryInfo->getDictBufOffset();
-    const void *dictBuf = binaryDictionaryInfo->getDictBuf();
-    if (!dictBuf) return;
-    releaseDictBuf(static_cast<const char *>(dictBuf) - dictBufOffset,
-            binaryDictionaryInfo->getDictSize() + dictBufOffset,
-            binaryDictionaryInfo->getMmapFd());
     delete dictionary;
 }
 
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h
deleted file mode 100644
index 7f646e3..0000000
--- a/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (C) 2013, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_BINARY_DICTIONARY_INFO_H
-#define LATINIME_BINARY_DICTIONARY_INFO_H
-
-#include <stdint.h>
-
-#include "defines.h"
-
-namespace latinime {
-
-class BinaryDictionaryInfo {
- public:
-     AK_FORCE_INLINE BinaryDictionaryInfo(const uint8_t *const dictBuf,
-            const int dictSize, const int mmapFd, const int dictBufOffset, const bool isUpdatable)
-            : mDictBuf(dictBuf), mDictSize(dictSize), mMmapFd(mmapFd),
-              mDictBufOffset(dictBufOffset), mIsUpdatable(isUpdatable) {}
-
-    ~BinaryDictionaryInfo() {}
-
-    AK_FORCE_INLINE const uint8_t *getDictBuf() const {
-        return mDictBuf;
-    }
-
-    AK_FORCE_INLINE int getDictSize() const {
-        return mDictSize;
-    }
-
-    AK_FORCE_INLINE int getMmapFd() const {
-        return mMmapFd;
-    }
-
-    AK_FORCE_INLINE int getDictBufOffset() const {
-        return mDictBufOffset;
-    }
-
-    AK_FORCE_INLINE bool isDynamicallyUpdatable() const {
-        return mIsUpdatable;
-    }
-
- private:
-    DISALLOW_COPY_AND_ASSIGN(BinaryDictionaryInfo);
-
-    const uint8_t *const mDictBuf;
-    const int mDictSize;
-    const int mMmapFd;
-    const int mDictBufOffset;
-    const bool mIsUpdatable;
-};
-}
-#endif /* LATINIME_BINARY_DICTIONARY_INFO_H */
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp
index 2ffd494..8418a60 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp
@@ -23,23 +23,19 @@
 #include "defines.h"
 #include "suggest/core/dictionary/bigram_dictionary.h"
 #include "suggest/core/policy/dictionary_header_structure_policy.h"
+#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
 #include "suggest/core/session/dic_traverse_session.h"
 #include "suggest/core/suggest.h"
 #include "suggest/core/suggest_options.h"
-#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
 #include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h"
 #include "suggest/policyimpl/typing/typing_suggest_policy_factory.h"
 #include "utils/log_utils.h"
 
 namespace latinime {
 
-Dictionary::Dictionary(JNIEnv *env, void *dict, int dictSize, int mmapFd,
-        int dictBufOffset, bool isUpdatable)
-        : mBinaryDictionaryInfo(static_cast<const uint8_t *>(dict), dictSize, mmapFd,
-                dictBufOffset, isUpdatable),
-          mDictionaryStructureWithBufferPolicy(DictionaryStructureWithBufferPolicyFactory
-                  ::newDictionaryStructureWithBufferPolicy(
-                          static_cast<const uint8_t *>(dict), dictSize)),
+Dictionary::Dictionary(JNIEnv *env,
+        DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy)
+        : mDictionaryStructureWithBufferPolicy(dictionaryStructureWithBufferPolicy),
           mBigramDictionary(new BigramDictionary(mDictionaryStructureWithBufferPolicy)),
           mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())),
           mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) {
@@ -102,32 +98,17 @@
 }
 
 void Dictionary::addUnigramWord(const int *const word, const int length, const int probability) {
-    if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {
-        // This method should not be called for non-updatable dictionary.
-        AKLOGI("Warning: Dictionary::addUnigramWord() is called for non-updatable dictionary.");
-        return;
-    }
     mDictionaryStructureWithBufferPolicy->addUnigramWord(word, length, probability);
 }
 
 void Dictionary::addBigramWords(const int *const word0, const int length0, const int *const word1,
         const int length1, const int probability) {
-    if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {
-        // This method should not be called for non-updatable dictionary.
-        AKLOGI("Warning: Dictionary::addBigramWords() is called for non-updatable dictionary.");
-        return;
-    }
     mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, word1, length1,
             probability);
 }
 
 void Dictionary::removeBigramWords(const int *const word0, const int length0,
         const int *const word1, const int length1) {
-    if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {
-        // This method should not be called for non-updatable dictionary.
-        AKLOGI("Warning: Dictionary::removeBigramWords() is called for non-updatable dictionary.");
-        return;
-    }
     mDictionaryStructureWithBufferPolicy->removeBigramWords(word0, length0, word1, length1);
 }
 
@@ -155,9 +136,8 @@
             dateStringCharBuffer, BUFFER_SIZE);
 
     LogUtils::logToJava(env,
-            "Dictionary info: dictionary = %s ; version = %s ; date = %s ; filesize = %i",
-            dictionaryIdCharBuffer, versionStringCharBuffer, dateStringCharBuffer,
-            mBinaryDictionaryInfo.getDictSize());
+            "Dictionary info: dictionary = %s ; version = %s ; date = %s",
+            dictionaryIdCharBuffer, versionStringCharBuffer, dateStringCharBuffer);
 }
 
 } // namespace latinime
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h
index a54b62f..0afe5a5 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.h
+++ b/native/jni/src/suggest/core/dictionary/dictionary.h
@@ -21,7 +21,6 @@
 
 #include "defines.h"
 #include "jni.h"
-#include "suggest/core/dictionary/binary_dictionary_info.h"
 
 namespace latinime {
 
@@ -54,8 +53,8 @@
     static const int KIND_FLAG_POSSIBLY_OFFENSIVE = 0x80000000;
     static const int KIND_FLAG_EXACT_MATCH = 0x40000000;
 
-    Dictionary(JNIEnv *env, void *dict, int dictSize, int mmapFd, int dictBufOffset,
-            bool isUpdatable);
+    Dictionary(JNIEnv *env,
+            DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPoilcy);
 
     int getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,
             int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
@@ -78,11 +77,6 @@
     void removeBigramWords(const int *const word0, const int length0, const int *const word1,
             const int length1);
 
-    // TODO: Remove.
-    const BinaryDictionaryInfo *getBinaryDictionaryInfo() const {
-        return &mBinaryDictionaryInfo;
-    }
-
     const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
         return mDictionaryStructureWithBufferPolicy;
     }
@@ -92,7 +86,6 @@
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
 
-    const BinaryDictionaryInfo mBinaryDictionaryInfo;
     DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy;
     const BigramDictionary *const mBigramDictionary;
     const SuggestInterface *const mGestureSuggest;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/binary_format.h b/native/jni/src/suggest/policyimpl/dictionary/binary_format.h
deleted file mode 100644
index 23f4c7f..0000000
--- a/native/jni/src/suggest/policyimpl/dictionary/binary_format.h
+++ /dev/null
@@ -1,470 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_BINARY_FORMAT_H
-#define LATINIME_BINARY_FORMAT_H
-
-#include <stdint.h>
-
-#include "suggest/core/dictionary/probability_utils.h"
-#include "utils/char_utils.h"
-
-namespace latinime {
-
-class BinaryFormat {
- public:
-    // Mask and flags for children address type selection.
-    static const int MASK_GROUP_ADDRESS_TYPE = 0xC0;
-
-    // Flag for single/multiple char group
-    static const int FLAG_HAS_MULTIPLE_CHARS = 0x20;
-
-    // Flag for terminal groups
-    static const int FLAG_IS_TERMINAL = 0x10;
-
-    // Flag for shortcut targets presence
-    static const int FLAG_HAS_SHORTCUT_TARGETS = 0x08;
-    // Flag for bigram presence
-    static const int FLAG_HAS_BIGRAMS = 0x04;
-    // Flag for non-words (typically, shortcut only entries)
-    static const int FLAG_IS_NOT_A_WORD = 0x02;
-    // Flag for blacklist
-    static const int FLAG_IS_BLACKLISTED = 0x01;
-
-    // Attribute (bigram/shortcut) related flags:
-    // Flag for presence of more attributes
-    static const int FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
-    // Flag for sign of offset. If this flag is set, the offset value must be negated.
-    static const int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
-
-    // Mask for attribute probability, stored on 4 bits inside the flags byte.
-    static const int MASK_ATTRIBUTE_PROBABILITY = 0x0F;
-
-    // Mask and flags for attribute address type selection.
-    static const int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30;
-
-    static int getGroupCountAndForwardPointer(const uint8_t *const dict, int *pos);
-    static uint8_t getFlagsAndForwardPointer(const uint8_t *const dict, int *pos);
-    static int getCodePointAndForwardPointer(const uint8_t *const dict, int *pos);
-    static int readProbabilityWithoutMovingPointer(const uint8_t *const dict, const int pos);
-    static int skipOtherCharacters(const uint8_t *const dict, const int pos);
-    static int skipChildrenPosition(const uint8_t flags, const int pos);
-    static int skipProbability(const uint8_t flags, const int pos);
-    static int skipShortcuts(const uint8_t *const dict, const uint8_t flags, const int pos);
-    static int skipChildrenPosAndAttributes(const uint8_t *const dict, const uint8_t flags,
-            const int pos);
-    static int readChildrenPosition(const uint8_t *const dict, const uint8_t flags, const int pos);
-    static bool hasChildrenInFlags(const uint8_t flags);
-    static int getTerminalPosition(const uint8_t *const root, const int *const inWord,
-            const int length, const bool forceLowerCaseSearch);
-    static int getCodePointsAndProbabilityAndReturnCodePointCount(
-            const uint8_t *const root, const int nodePos, const int maxCodePointCount,
-            int *const outCodePoints, int *const outUnigramProbability);
-
- private:
-    DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryFormat);
-
-    static const int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00;
-    static const int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40;
-    static const int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80;
-    static const int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0;
-    static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
-    static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
-    static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
-
-    static const int CHARACTER_ARRAY_TERMINATOR_SIZE = 1;
-    static const int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
-    static const int CHARACTER_ARRAY_TERMINATOR = 0x1F;
-    static const int MULTIPLE_BYTE_CHARACTER_ADDITIONAL_SIZE = 2;
-    static const int NO_FLAGS = 0;
-    static int skipAllAttributes(const uint8_t *const dict, const uint8_t flags, const int pos);
-    static int skipBigrams(const uint8_t *const dict, const uint8_t flags, const int pos);
-};
-
-AK_FORCE_INLINE int BinaryFormat::getGroupCountAndForwardPointer(const uint8_t *const dict,
-        int *pos) {
-    const int msb = dict[(*pos)++];
-    if (msb < 0x80) return msb;
-    return ((msb & 0x7F) << 8) | dict[(*pos)++];
-}
-
-inline uint8_t BinaryFormat::getFlagsAndForwardPointer(const uint8_t *const dict, int *pos) {
-    return dict[(*pos)++];
-}
-
-AK_FORCE_INLINE int BinaryFormat::getCodePointAndForwardPointer(const uint8_t *const dict,
-        int *pos) {
-    const int origin = *pos;
-    const int codePoint = dict[origin];
-    if (codePoint < MINIMAL_ONE_BYTE_CHARACTER_VALUE) {
-        if (codePoint == CHARACTER_ARRAY_TERMINATOR) {
-            *pos = origin + 1;
-            return NOT_A_CODE_POINT;
-        } else {
-            *pos = origin + 3;
-            const int char_1 = codePoint << 16;
-            const int char_2 = char_1 + (dict[origin + 1] << 8);
-            return char_2 + dict[origin + 2];
-        }
-    } else {
-        *pos = origin + 1;
-        return codePoint;
-    }
-}
-
-inline int BinaryFormat::readProbabilityWithoutMovingPointer(const uint8_t *const dict,
-        const int pos) {
-    return dict[pos];
-}
-
-AK_FORCE_INLINE int BinaryFormat::skipOtherCharacters(const uint8_t *const dict, const int pos) {
-    int currentPos = pos;
-    int character = dict[currentPos++];
-    while (CHARACTER_ARRAY_TERMINATOR != character) {
-        if (character < MINIMAL_ONE_BYTE_CHARACTER_VALUE) {
-            currentPos += MULTIPLE_BYTE_CHARACTER_ADDITIONAL_SIZE;
-        }
-        character = dict[currentPos++];
-    }
-    return currentPos;
-}
-
-static inline int attributeAddressSize(const uint8_t flags) {
-    static const int ATTRIBUTE_ADDRESS_SHIFT = 4;
-    return (flags & BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT;
-    /* Note: this is a value-dependant optimization of what may probably be
-       more readably written this way:
-       switch (flags * BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) {
-       case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1;
-       case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2;
-       case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3;
-       default: return 0;
-       }
-    */
-}
-
-static AK_FORCE_INLINE int skipExistingBigrams(const uint8_t *const dict, const int pos) {
-    int currentPos = pos;
-    uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(dict, &currentPos);
-    while (flags & BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT) {
-        currentPos += attributeAddressSize(flags);
-        flags = BinaryFormat::getFlagsAndForwardPointer(dict, &currentPos);
-    }
-    currentPos += attributeAddressSize(flags);
-    return currentPos;
-}
-
-static inline int childrenAddressSize(const uint8_t flags) {
-    static const int CHILDREN_ADDRESS_SHIFT = 6;
-    return (BinaryFormat::MASK_GROUP_ADDRESS_TYPE & flags) >> CHILDREN_ADDRESS_SHIFT;
-    /* See the note in attributeAddressSize. The same applies here */
-}
-
-static AK_FORCE_INLINE int shortcutByteSize(const uint8_t *const dict, const int pos) {
-    return (static_cast<int>(dict[pos] << 8)) + (dict[pos + 1]);
-}
-
-inline int BinaryFormat::skipChildrenPosition(const uint8_t flags, const int pos) {
-    return pos + childrenAddressSize(flags);
-}
-
-inline int BinaryFormat::skipProbability(const uint8_t flags, const int pos) {
-    return FLAG_IS_TERMINAL & flags ? pos + 1 : pos;
-}
-
-AK_FORCE_INLINE int BinaryFormat::skipShortcuts(const uint8_t *const dict, const uint8_t flags,
-        const int pos) {
-    if (FLAG_HAS_SHORTCUT_TARGETS & flags) {
-        return pos + shortcutByteSize(dict, pos);
-    } else {
-        return pos;
-    }
-}
-
-AK_FORCE_INLINE int BinaryFormat::skipBigrams(const uint8_t *const dict, const uint8_t flags,
-        const int pos) {
-    if (FLAG_HAS_BIGRAMS & flags) {
-        return skipExistingBigrams(dict, pos);
-    } else {
-        return pos;
-    }
-}
-
-AK_FORCE_INLINE int BinaryFormat::skipAllAttributes(const uint8_t *const dict, const uint8_t flags,
-        const int pos) {
-    // This function skips all attributes: shortcuts and bigrams.
-    int newPos = pos;
-    newPos = skipShortcuts(dict, flags, newPos);
-    newPos = skipBigrams(dict, flags, newPos);
-    return newPos;
-}
-
-AK_FORCE_INLINE int BinaryFormat::skipChildrenPosAndAttributes(const uint8_t *const dict,
-        const uint8_t flags, const int pos) {
-    int currentPos = pos;
-    currentPos = skipChildrenPosition(flags, currentPos);
-    currentPos = skipAllAttributes(dict, flags, currentPos);
-    return currentPos;
-}
-
-AK_FORCE_INLINE int BinaryFormat::readChildrenPosition(const uint8_t *const dict,
-        const uint8_t flags, const int pos) {
-    int offset = 0;
-    switch (MASK_GROUP_ADDRESS_TYPE & flags) {
-        case FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
-            offset = dict[pos];
-            break;
-        case FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
-            offset = dict[pos] << 8;
-            offset += dict[pos + 1];
-            break;
-        case FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
-            offset = dict[pos] << 16;
-            offset += dict[pos + 1] << 8;
-            offset += dict[pos + 2];
-            break;
-        default:
-            // If we come here, it means we asked for the children of a word with
-            // no children.
-            return -1;
-    }
-    return pos + offset;
-}
-
-inline bool BinaryFormat::hasChildrenInFlags(const uint8_t flags) {
-    return (FLAG_GROUP_ADDRESS_TYPE_NOADDRESS != (MASK_GROUP_ADDRESS_TYPE & flags));
-}
-
-// This function gets the byte position of the last chargroup of the exact matching word in the
-// dictionary. If no match is found, it returns NOT_A_VALID_WORD_POS.
-AK_FORCE_INLINE int BinaryFormat::getTerminalPosition(const uint8_t *const root,
-        const int *const inWord, const int length, const bool forceLowerCaseSearch) {
-    int pos = 0;
-    int wordPos = 0;
-
-    while (true) {
-        // If we already traversed the tree further than the word is long, there means
-        // there was no match (or we would have found it).
-        if (wordPos >= length) return NOT_A_VALID_WORD_POS;
-        int charGroupCount = BinaryFormat::getGroupCountAndForwardPointer(root, &pos);
-        const int wChar = forceLowerCaseSearch
-                ? CharUtils::toLowerCase(inWord[wordPos]) : inWord[wordPos];
-        while (true) {
-            // If there are no more character groups in this node, it means we could not
-            // find a matching character for this depth, therefore there is no match.
-            if (0 >= charGroupCount) return NOT_A_VALID_WORD_POS;
-            const int charGroupPos = pos;
-            const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
-            int character = BinaryFormat::getCodePointAndForwardPointer(root, &pos);
-            if (character == wChar) {
-                // This is the correct node. Only one character group may start with the same
-                // char within a node, so either we found our match in this node, or there is
-                // no match and we can return NOT_A_VALID_WORD_POS. So we will check all the
-                // characters in this character group indeed does match.
-                if (FLAG_HAS_MULTIPLE_CHARS & flags) {
-                    character = BinaryFormat::getCodePointAndForwardPointer(root, &pos);
-                    while (NOT_A_CODE_POINT != character) {
-                        ++wordPos;
-                        // If we shoot the length of the word we search for, or if we find a single
-                        // character that does not match, as explained above, it means the word is
-                        // not in the dictionary (by virtue of this chargroup being the only one to
-                        // match the word on the first character, but not matching the whole word).
-                        if (wordPos >= length) return NOT_A_VALID_WORD_POS;
-                        if (inWord[wordPos] != character) return NOT_A_VALID_WORD_POS;
-                        character = BinaryFormat::getCodePointAndForwardPointer(root, &pos);
-                    }
-                }
-                // If we come here we know that so far, we do match. Either we are on a terminal
-                // and we match the length, in which case we found it, or we traverse children.
-                // If we don't match the length AND don't have children, then a word in the
-                // dictionary fully matches a prefix of the searched word but not the full word.
-                ++wordPos;
-                if (FLAG_IS_TERMINAL & flags) {
-                    if (wordPos == length) {
-                        return charGroupPos;
-                    }
-                    pos = BinaryFormat::skipProbability(FLAG_IS_TERMINAL, pos);
-                }
-                if (FLAG_GROUP_ADDRESS_TYPE_NOADDRESS == (MASK_GROUP_ADDRESS_TYPE & flags)) {
-                    return NOT_A_VALID_WORD_POS;
-                }
-                // We have children and we are still shorter than the word we are searching for, so
-                // we need to traverse children. Put the pointer on the children position, and
-                // break
-                pos = BinaryFormat::readChildrenPosition(root, flags, pos);
-                break;
-            } else {
-                // This chargroup does not match, so skip the remaining part and go to the next.
-                if (FLAG_HAS_MULTIPLE_CHARS & flags) {
-                    pos = BinaryFormat::skipOtherCharacters(root, pos);
-                }
-                pos = BinaryFormat::skipProbability(flags, pos);
-                pos = BinaryFormat::skipChildrenPosAndAttributes(root, flags, pos);
-            }
-            --charGroupCount;
-        }
-    }
-}
-
-// This function searches for a terminal in the dictionary by its address.
-// Due to the fact that words are ordered in the dictionary in a strict breadth-first order,
-// it is possible to check for this with advantageous complexity. For each node, we search
-// for groups with children and compare the children address with the address we look for.
-// When we shoot the address we look for, it means the word we look for is in the children
-// of the previous group. The only tricky part is the fact that if we arrive at the end of a
-// node with the last group's children address still less than what we are searching for, we
-// must descend the last group's children (for example, if the word we are searching for starts
-// with a z, it's the last group of the root node, so all children addresses will be smaller
-// than the address we look for, and we have to descend the z node).
-/* Parameters :
- * root: the dictionary buffer
- * address: the byte position of the last chargroup of the word we are searching for (this is
- *   what is stored as the "bigram address" in each bigram)
- * outword: an array to write the found word, with MAX_WORD_LENGTH size.
- * outUnigramProbability: a pointer to an int to write the probability into.
- * Return value : the length of the word, of 0 if the word was not found.
- */
-AK_FORCE_INLINE int BinaryFormat::getCodePointsAndProbabilityAndReturnCodePointCount(
-        const uint8_t *const root, const int nodePos, const int maxCodePointCount,
-        int *const outCodePoints, int *const outUnigramProbability) {
-    int pos = 0;
-    int wordPos = 0;
-
-    // One iteration of the outer loop iterates through nodes. As stated above, we will only
-    // traverse nodes that are actually a part of the terminal we are searching, so each time
-    // we enter this loop we are one depth level further than last time.
-    // The only reason we count nodes is because we want to reduce the probability of infinite
-    // looping in case there is a bug. Since we know there is an upper bound to the depth we are
-    // supposed to traverse, it does not hurt to count iterations.
-    for (int loopCount = maxCodePointCount; loopCount > 0; --loopCount) {
-        int lastCandidateGroupPos = 0;
-        // Let's loop through char groups in this node searching for either the terminal
-        // or one of its ascendants.
-        for (int charGroupCount = getGroupCountAndForwardPointer(root, &pos); charGroupCount > 0;
-                 --charGroupCount) {
-            const int startPos = pos;
-            const uint8_t flags = getFlagsAndForwardPointer(root, &pos);
-            const int character = getCodePointAndForwardPointer(root, &pos);
-            if (nodePos == startPos) {
-                // We found the address. Copy the rest of the word in the buffer and return
-                // the length.
-                outCodePoints[wordPos] = character;
-                if (FLAG_HAS_MULTIPLE_CHARS & flags) {
-                    int nextChar = getCodePointAndForwardPointer(root, &pos);
-                    // We count chars in order to avoid infinite loops if the file is broken or
-                    // if there is some other bug
-                    int charCount = maxCodePointCount;
-                    while (NOT_A_CODE_POINT != nextChar && --charCount > 0) {
-                        outCodePoints[++wordPos] = nextChar;
-                        nextChar = getCodePointAndForwardPointer(root, &pos);
-                    }
-                }
-                *outUnigramProbability = readProbabilityWithoutMovingPointer(root, pos);
-                return ++wordPos;
-            }
-            // We need to skip past this char group, so skip any remaining chars after the
-            // first and possibly the probability.
-            if (FLAG_HAS_MULTIPLE_CHARS & flags) {
-                pos = skipOtherCharacters(root, pos);
-            }
-            pos = skipProbability(flags, pos);
-
-            // The fact that this group has children is very important. Since we already know
-            // that this group does not match, if it has no children we know it is irrelevant
-            // to what we are searching for.
-            const bool hasChildren = (FLAG_GROUP_ADDRESS_TYPE_NOADDRESS !=
-                    (MASK_GROUP_ADDRESS_TYPE & flags));
-            // We will write in `found' whether we have passed the children address we are
-            // searching for. For example if we search for "beer", the children of b are less
-            // than the address we are searching for and the children of c are greater. When we
-            // come here for c, we realize this is too big, and that we should descend b.
-            bool found;
-            if (hasChildren) {
-                // Here comes the tricky part. First, read the children position.
-                const int childrenPos = readChildrenPosition(root, flags, pos);
-                if (childrenPos > nodePos) {
-                    // If the children pos is greater than address, it means the previous chargroup,
-                    // which address is stored in lastCandidateGroupPos, was the right one.
-                    found = true;
-                } else if (1 >= charGroupCount) {
-                    // However if we are on the LAST group of this node, and we have NOT shot the
-                    // address we should descend THIS node. So we trick the lastCandidateGroupPos
-                    // so that we will descend this node, not the previous one.
-                    lastCandidateGroupPos = startPos;
-                    found = true;
-                } else {
-                    // Else, we should continue looking.
-                    found = false;
-                }
-            } else {
-                // Even if we don't have children here, we could still be on the last group of this
-                // node. If this is the case, we should descend the last group that had children,
-                // and their address is already in lastCandidateGroup.
-                found = (1 >= charGroupCount);
-            }
-
-            if (found) {
-                // Okay, we found the group we should descend. Its address is in
-                // the lastCandidateGroupPos variable, so we just re-read it.
-                if (0 != lastCandidateGroupPos) {
-                    const uint8_t lastFlags =
-                            getFlagsAndForwardPointer(root, &lastCandidateGroupPos);
-                    const int lastChar =
-                            getCodePointAndForwardPointer(root, &lastCandidateGroupPos);
-                    // We copy all the characters in this group to the buffer
-                    outCodePoints[wordPos] = lastChar;
-                    if (FLAG_HAS_MULTIPLE_CHARS & lastFlags) {
-                        int nextChar = getCodePointAndForwardPointer(root, &lastCandidateGroupPos);
-                        int charCount = maxCodePointCount;
-                        while (-1 != nextChar && --charCount > 0) {
-                            outCodePoints[++wordPos] = nextChar;
-                            nextChar = getCodePointAndForwardPointer(root, &lastCandidateGroupPos);
-                        }
-                    }
-                    ++wordPos;
-                    // Now we only need to branch to the children address. Skip the probability if
-                    // it's there, read pos, and break to resume the search at pos.
-                    lastCandidateGroupPos = skipProbability(lastFlags, lastCandidateGroupPos);
-                    pos = readChildrenPosition(root, lastFlags, lastCandidateGroupPos);
-                    break;
-                } else {
-                    // Here is a little tricky part: we come here if we found out that all children
-                    // addresses in this group are bigger than the address we are searching for.
-                    // Should we conclude the word is not in the dictionary? No! It could still be
-                    // one of the remaining chargroups in this node, so we have to keep looking in
-                    // this node until we find it (or we realize it's not there either, in which
-                    // case it's actually not in the dictionary). Pass the end of this group, ready
-                    // to start the next one.
-                    pos = skipChildrenPosAndAttributes(root, flags, pos);
-                }
-            } else {
-                // If we did not find it, we should record the last children address for the next
-                // iteration.
-                if (hasChildren) lastCandidateGroupPos = startPos;
-                // Now skip the end of this group (children pos and the attributes if any) so that
-                // our pos is after the end of this char group, at the start of the next one.
-                pos = skipChildrenPosAndAttributes(root, flags, pos);
-            }
-
-        }
-    }
-    // If we have looked through all the chargroups and found no match, the address is
-    // not the address of a terminal in this dictionary.
-    return 0;
-}
-
-} // namespace latinime
-#endif // LATINIME_BINARY_FORMAT_H
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp
index 34f092f..cc5252c 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp
@@ -22,18 +22,29 @@
 #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h"
 #include "suggest/policyimpl/dictionary/patricia_trie_policy.h"
 #include "suggest/policyimpl/dictionary/utils/format_utils.h"
+#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h"
 
 namespace latinime {
 
 /* static */ DictionaryStructureWithBufferPolicy *DictionaryStructureWithBufferPolicyFactory
-        ::newDictionaryStructureWithBufferPolicy(const uint8_t *const dictBuf,
-                const int dictSize) {
-    switch (FormatUtils::detectFormatVersion(dictBuf, dictSize)) {
+        ::newDictionaryStructureWithBufferPolicy(const char *const path, const int pathLength,
+                const int bufOffset, const int size, const bool isUpdatable) {
+    // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
+    // impl classes of DictionaryStructureWithBufferPolicy.
+    const MmapedBuffer *const mmapedBuffer = MmapedBuffer::openBuffer(path, pathLength, bufOffset,
+            size, isUpdatable);
+    if (!mmapedBuffer) {
+        return 0;
+    }
+    switch (FormatUtils::detectFormatVersion(mmapedBuffer->getBuffer(),
+            mmapedBuffer->getBufferSize())) {
         case FormatUtils::VERSION_2:
-            return new PatriciaTriePolicy(dictBuf);
+            return new PatriciaTriePolicy(mmapedBuffer);
         case FormatUtils::VERSION_3:
-            return new DynamicPatriciaTriePolicy(dictBuf);
+            return new DynamicPatriciaTriePolicy(mmapedBuffer);
         default:
+            AKLOGE("DICT: dictionary format is unknown, bad magic number");
+            delete mmapedBuffer;
             ASSERT(false);
             return 0;
     }
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h
index 53eb8f9..1cb7a89 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h
@@ -27,7 +27,8 @@
 class DictionaryStructureWithBufferPolicyFactory {
  public:
     static DictionaryStructureWithBufferPolicy *newDictionaryStructureWithBufferPolicy(
-            const uint8_t *const dictBuf, const int dictSize);
+            const char *const path, const int pathLength, const int bufOffset, const int size,
+            const bool isUpdatable);
 
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
index 311e94d..3000860 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
@@ -233,18 +233,30 @@
 
 bool DynamicPatriciaTriePolicy::addUnigramWord(const int *const word, const int length,
         const int probability) {
+    if (!mBuffer->isUpdatable()) {
+        AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
+        return false;
+    }
     // TODO: Implement.
     return false;
 }
 
 bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int length0,
         const int *const word1, const int length1, const int probability) {
+    if (!mBuffer->isUpdatable()) {
+        AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
+        return false;
+    }
     // TODO: Implement.
     return false;
 }
 
 bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0,
         const int *const word1, const int length1) {
+    if (!mBuffer->isUpdatable()) {
+        AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary.");
+        return false;
+    }
     // TODO: Implement.
     return false;
 }
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
index 4a5192c..708967d 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
@@ -24,6 +24,7 @@
 #include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
 #include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h"
+#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h"
 
 namespace latinime {
 
@@ -32,11 +33,14 @@
 
 class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
  public:
-    DynamicPatriciaTriePolicy(const uint8_t *const dictBuf)
-            : mHeaderPolicy(dictBuf), mDictRoot(dictBuf + mHeaderPolicy.getSize()),
+    DynamicPatriciaTriePolicy(const MmapedBuffer *const buffer)
+            : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()),
+              mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()),
               mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}
 
-    ~DynamicPatriciaTriePolicy() {}
+    ~DynamicPatriciaTriePolicy() {
+        delete mBuffer;
+    }
 
     AK_FORCE_INLINE int getRootPosition() const {
         return 0;
@@ -82,6 +86,7 @@
     DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);
     static const int MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP;
 
+    const MmapedBuffer *const mBuffer;
     const HeaderPolicy mHeaderPolicy;
     // TODO: Consolidate mDictRoot.
     const uint8_t *const mDictRoot;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp
index 3e664a2..15eb067 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp
@@ -20,7 +20,6 @@
 #include "defines.h"
 #include "suggest/core/dicnode/dic_node.h"
 #include "suggest/core/dicnode/dic_node_vector.h"
-#include "suggest/policyimpl/dictionary/binary_format.h"
 #include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
 
 namespace latinime {
@@ -38,17 +37,273 @@
     }
 }
 
+// This retrieves code points and the probability of the word by its terminal position.
+// Due to the fact that words are ordered in the dictionary in a strict breadth-first order,
+// it is possible to check for this with advantageous complexity. For each node, we search
+// for groups with children and compare the children position with the position we look for.
+// When we shoot the position we look for, it means the word we look for is in the children
+// of the previous group. The only tricky part is the fact that if we arrive at the end of a
+// node with the last group's children position still less than what we are searching for, we
+// must descend the last group's children (for example, if the word we are searching for starts
+// with a z, it's the last group of the root node, so all children addresses will be smaller
+// than the position we look for, and we have to descend the z node).
+/* Parameters :
+ * nodePos: the byte position of the terminal chargroup of the word we are searching for (this is
+ *   what is stored as the "bigram position" in each bigram)
+ * outCodePoints: an array to write the found word, with MAX_WORD_LENGTH size.
+ * outUnigramProbability: a pointer to an int to write the probability into.
+ * Return value : the code point count, of 0 if the word was not found.
+ */
+// TODO: Split this function to be more readable
 int PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
         const int nodePos, const int maxCodePointCount, int *const outCodePoints,
         int *const outUnigramProbability) const {
-    return BinaryFormat::getCodePointsAndProbabilityAndReturnCodePointCount(mDictRoot, nodePos,
-            maxCodePointCount, outCodePoints, outUnigramProbability);
+    int pos = getRootPosition();
+    int wordPos = 0;
+    // One iteration of the outer loop iterates through nodes. As stated above, we will only
+    // traverse nodes that are actually a part of the terminal we are searching, so each time
+    // we enter this loop we are one depth level further than last time.
+    // The only reason we count nodes is because we want to reduce the probability of infinite
+    // looping in case there is a bug. Since we know there is an upper bound to the depth we are
+    // supposed to traverse, it does not hurt to count iterations.
+    for (int loopCount = maxCodePointCount; loopCount > 0; --loopCount) {
+        int lastCandidateGroupPos = 0;
+        // Let's loop through char groups in this node searching for either the terminal
+        // or one of its ascendants.
+        for (int charGroupCount = PatriciaTrieReadingUtils::getGroupCountAndAdvancePosition(
+                mDictRoot, &pos); charGroupCount > 0; --charGroupCount) {
+            const int startPos = pos;
+            const PatriciaTrieReadingUtils::NodeFlags flags =
+                    PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(mDictRoot, &pos);
+            const int character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+                    mDictRoot, &pos);
+            if (nodePos == startPos) {
+                // We found the position. Copy the rest of the code points in the buffer and return
+                // the length.
+                outCodePoints[wordPos] = character;
+                if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) {
+                    int nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+                            mDictRoot, &pos);
+                    // We count code points in order to avoid infinite loops if the file is broken
+                    // or if there is some other bug
+                    int charCount = maxCodePointCount;
+                    while (NOT_A_CODE_POINT != nextChar && --charCount > 0) {
+                        outCodePoints[++wordPos] = nextChar;
+                        nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+                                mDictRoot, &pos);
+                    }
+                }
+                *outUnigramProbability =
+                        PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot,
+                                &pos);
+                return ++wordPos;
+            }
+            // We need to skip past this char group, so skip any remaining code points after the
+            // first and possibly the probability.
+            if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) {
+                PatriciaTrieReadingUtils::skipCharacters(mDictRoot, flags, MAX_WORD_LENGTH, &pos);
+            }
+            if (PatriciaTrieReadingUtils::isTerminal(flags)) {
+                PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos);
+            }
+            // The fact that this group has children is very important. Since we already know
+            // that this group does not match, if it has no children we know it is irrelevant
+            // to what we are searching for.
+            const bool hasChildren = PatriciaTrieReadingUtils::hasChildrenInFlags(flags);
+            // We will write in `found' whether we have passed the children position we are
+            // searching for. For example if we search for "beer", the children of b are less
+            // than the address we are searching for and the children of c are greater. When we
+            // come here for c, we realize this is too big, and that we should descend b.
+            bool found;
+            if (hasChildren) {
+                int currentPos = pos;
+                // Here comes the tricky part. First, read the children position.
+                const int childrenPos = PatriciaTrieReadingUtils
+                        ::readChildrenPositionAndAdvancePosition(mDictRoot, flags, &currentPos);
+                if (childrenPos > nodePos) {
+                    // If the children pos is greater than the position, it means the previous
+                    // chargroup, which position is stored in lastCandidateGroupPos, was the right
+                    // one.
+                    found = true;
+                } else if (1 >= charGroupCount) {
+                    // However if we are on the LAST group of this node, and we have NOT shot the
+                    // position we should descend THIS node. So we trick the lastCandidateGroupPos
+                    // so that we will descend this node, not the previous one.
+                    lastCandidateGroupPos = startPos;
+                    found = true;
+                } else {
+                    // Else, we should continue looking.
+                    found = false;
+                }
+            } else {
+                // Even if we don't have children here, we could still be on the last group of this
+                // node. If this is the case, we should descend the last group that had children,
+                // and their position is already in lastCandidateGroup.
+                found = (1 >= charGroupCount);
+            }
+
+            if (found) {
+                // Okay, we found the group we should descend. Its position is in
+                // the lastCandidateGroupPos variable, so we just re-read it.
+                if (0 != lastCandidateGroupPos) {
+                    const PatriciaTrieReadingUtils::NodeFlags lastFlags =
+                            PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(
+                                    mDictRoot, &lastCandidateGroupPos);
+                    const int lastChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+                            mDictRoot, &lastCandidateGroupPos);
+                    // We copy all the characters in this group to the buffer
+                    outCodePoints[wordPos] = lastChar;
+                    if (PatriciaTrieReadingUtils::hasMultipleChars(lastFlags)) {
+                        int nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+                                mDictRoot, &lastCandidateGroupPos);
+                        int charCount = maxCodePointCount;
+                        while (-1 != nextChar && --charCount > 0) {
+                            outCodePoints[++wordPos] = nextChar;
+                            nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+                                    mDictRoot, &lastCandidateGroupPos);
+                        }
+                    }
+                    ++wordPos;
+                    // Now we only need to branch to the children address. Skip the probability if
+                    // it's there, read pos, and break to resume the search at pos.
+                    if (PatriciaTrieReadingUtils::isTerminal(lastFlags)) {
+                        PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot,
+                                &lastCandidateGroupPos);
+                    }
+                    pos = PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
+                            mDictRoot, lastFlags, &lastCandidateGroupPos);
+                    break;
+                } else {
+                    // Here is a little tricky part: we come here if we found out that all children
+                    // addresses in this group are bigger than the address we are searching for.
+                    // Should we conclude the word is not in the dictionary? No! It could still be
+                    // one of the remaining chargroups in this node, so we have to keep looking in
+                    // this node until we find it (or we realize it's not there either, in which
+                    // case it's actually not in the dictionary). Pass the end of this group, ready
+                    // to start the next one.
+                    if (PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) {
+                        PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
+                                mDictRoot, flags, &pos);
+                    }
+                    if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) {
+                        mShortcutListPolicy.skipAllShortcuts(&pos);
+                    }
+                    if (PatriciaTrieReadingUtils::hasBigrams(flags)) {
+                        mBigramListPolicy.skipAllBigrams(&pos);
+                    }
+                }
+            } else {
+                // If we did not find it, we should record the last children address for the next
+                // iteration.
+                if (hasChildren) lastCandidateGroupPos = startPos;
+                // Now skip the end of this group (children pos and the attributes if any) so that
+                // our pos is after the end of this char group, at the start of the next one.
+                if (PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) {
+                    PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
+                            mDictRoot, flags, &pos);
+                }
+                if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) {
+                    mShortcutListPolicy.skipAllShortcuts(&pos);
+                }
+                if (PatriciaTrieReadingUtils::hasBigrams(flags)) {
+                    mBigramListPolicy.skipAllBigrams(&pos);
+                }
+            }
+
+        }
+    }
+    // If we have looked through all the chargroups and found no match, the nodePos is
+    // not the position of a terminal in this dictionary.
+    return 0;
 }
 
+// This function gets the position of the terminal node of the exact matching word in the
+// dictionary. If no match is found, it returns NOT_A_VALID_WORD_POS.
 int PatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const inWord,
         const int length, const bool forceLowerCaseSearch) const {
-    return BinaryFormat::getTerminalPosition(mDictRoot, inWord,
-            length, forceLowerCaseSearch);
+    int pos = getRootPosition();
+    int wordPos = 0;
+
+    while (true) {
+        // If we already traversed the tree further than the word is long, there means
+        // there was no match (or we would have found it).
+        if (wordPos >= length) return NOT_A_VALID_WORD_POS;
+        int charGroupCount = PatriciaTrieReadingUtils::getGroupCountAndAdvancePosition(mDictRoot,
+                &pos);
+        const int wChar = forceLowerCaseSearch
+                ? CharUtils::toLowerCase(inWord[wordPos]) : inWord[wordPos];
+        while (true) {
+            // If there are no more character groups in this node, it means we could not
+            // find a matching character for this depth, therefore there is no match.
+            if (0 >= charGroupCount) return NOT_A_VALID_WORD_POS;
+            const int charGroupPos = pos;
+            const PatriciaTrieReadingUtils::NodeFlags flags =
+                    PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(mDictRoot, &pos);
+            int character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(mDictRoot,
+                    &pos);
+            if (character == wChar) {
+                // This is the correct node. Only one character group may start with the same
+                // char within a node, so either we found our match in this node, or there is
+                // no match and we can return NOT_A_VALID_WORD_POS. So we will check all the
+                // characters in this character group indeed does match.
+                if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) {
+                    character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(mDictRoot,
+                            &pos);
+                    while (NOT_A_CODE_POINT != character) {
+                        ++wordPos;
+                        // If we shoot the length of the word we search for, or if we find a single
+                        // character that does not match, as explained above, it means the word is
+                        // not in the dictionary (by virtue of this chargroup being the only one to
+                        // match the word on the first character, but not matching the whole word).
+                        if (wordPos >= length) return NOT_A_VALID_WORD_POS;
+                        if (inWord[wordPos] != character) return NOT_A_VALID_WORD_POS;
+                        character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(
+                                mDictRoot, &pos);
+                    }
+                }
+                // If we come here we know that so far, we do match. Either we are on a terminal
+                // and we match the length, in which case we found it, or we traverse children.
+                // If we don't match the length AND don't have children, then a word in the
+                // dictionary fully matches a prefix of the searched word but not the full word.
+                ++wordPos;
+                if (PatriciaTrieReadingUtils::isTerminal(flags)) {
+                    if (wordPos == length) {
+                        return charGroupPos;
+                    }
+                    PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos);
+                }
+                if (!PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) {
+                    return NOT_A_VALID_WORD_POS;
+                }
+                // We have children and we are still shorter than the word we are searching for, so
+                // we need to traverse children. Put the pointer on the children position, and
+                // break
+                pos = PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(mDictRoot,
+                        flags, &pos);
+                break;
+            } else {
+                // This chargroup does not match, so skip the remaining part and go to the next.
+                if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) {
+                    PatriciaTrieReadingUtils::skipCharacters(mDictRoot, flags, MAX_WORD_LENGTH,
+                            &pos);
+                }
+                if (PatriciaTrieReadingUtils::isTerminal(flags)) {
+                    PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos);
+                }
+                if (PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) {
+                    PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(mDictRoot,
+                            flags, &pos);
+                }
+                if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) {
+                    mShortcutListPolicy.skipAllShortcuts(&pos);
+                }
+                if (PatriciaTrieReadingUtils::hasBigrams(flags)) {
+                    mBigramListPolicy.skipAllBigrams(&pos);
+                }
+            }
+            --charGroupCount;
+        }
+    }
 }
 
 int PatriciaTriePolicy::getUnigramProbability(const int nodePos) const {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
index 0ed7945..0d85050 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
@@ -24,6 +24,7 @@
 #include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
 #include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h"
+#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h"
 
 namespace latinime {
 
@@ -32,11 +33,14 @@
 
 class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
  public:
-    PatriciaTriePolicy(const uint8_t *const dictBuf)
-            : mHeaderPolicy(dictBuf), mDictRoot(dictBuf + mHeaderPolicy.getSize()),
+    PatriciaTriePolicy(const MmapedBuffer *const buffer)
+            : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()),
+              mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()),
               mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}
 
-    ~PatriciaTriePolicy() {}
+    ~PatriciaTriePolicy() {
+        delete mBuffer;
+    }
 
     AK_FORCE_INLINE int getRootPosition() const {
         return 0;
@@ -71,25 +75,29 @@
     }
 
     bool addUnigramWord(const int *const word, const int length, const int probability) {
-        // This dictionary format is not updatable.
+        // This method should not be called for non-updatable dictionary.
+        AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
         return false;
     }
 
     bool addBigramWords(const int *const word0, const int length0, const int *const word1,
             const int length1, const int probability) {
-        // This dictionary format is not updatable.
+        // This method should not be called for non-updatable dictionary.
+        AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
         return false;
     }
 
     bool removeBigramWords(const int *const word0, const int length0, const int *const word1,
             const int length1) {
-        // This dictionary format is not updatable.
+        // This method should not be called for non-updatable dictionary.
+        AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary.");
         return false;
     }
 
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy);
 
+    const MmapedBuffer *const mBuffer;
     const HeaderPolicy mHeaderPolicy;
     const uint8_t *const mDictRoot;
     const BigramListPolicy mBigramListPolicy;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h
new file mode 100644
index 0000000..08add03
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h
@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_MMAPED_BUFFER_H
+#define LATINIME_MMAPED_BUFFER_H
+
+#include <cerrno>
+#include <fcntl.h>
+#include <stdint.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "defines.h"
+
+namespace latinime {
+
+class MmapedBuffer {
+ public:
+    static MmapedBuffer* openBuffer(const char *const path, const int pathLength,
+            const int bufOffset, const int size, const bool isUpdatable) {
+        const int openMode = isUpdatable ? O_RDWR : O_RDONLY;
+        const int fd = open(path, openMode);
+        if (fd < 0) {
+            AKLOGE("DICT: Can't open the source. path=%s errno=%d", path, errno);
+            return 0;
+        }
+        const int pagesize = getpagesize();
+        const int offset = bufOffset % pagesize;
+        int adjOffset = bufOffset - offset;
+        int adjSize = size + offset;
+        const int protMode = isUpdatable ? PROT_READ | PROT_WRITE : PROT_READ;
+        void *const mmapedBuffer = mmap(0, adjSize, protMode, MAP_PRIVATE, fd, adjOffset);
+        if (mmapedBuffer == MAP_FAILED) {
+            AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno);
+            close(fd);
+            return 0;
+        }
+        uint8_t *const buffer = static_cast<uint8_t *>(mmapedBuffer) + offset;
+        if (!buffer) {
+            AKLOGE("DICT: buffer is null");
+            close(fd);
+            return 0;
+        }
+        return new MmapedBuffer(buffer, adjSize, fd, offset, isUpdatable);
+    }
+
+    ~MmapedBuffer() {
+        int ret = munmap(static_cast<void *>(mBuffer - mBufferOffset),
+                mBufferSize + mBufferOffset);
+        if (ret != 0) {
+            AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno);
+        }
+        ret = close(mMmapFd);
+        if (ret != 0) {
+            AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno);
+        }
+    }
+
+    AK_FORCE_INLINE uint8_t *getBuffer() const {
+        return mBuffer;
+    }
+
+    AK_FORCE_INLINE int getBufferSize() const {
+        return mBufferSize;
+    }
+
+    AK_FORCE_INLINE bool isUpdatable() const {
+        return mIsUpdatable;
+    }
+
+ private:
+    AK_FORCE_INLINE MmapedBuffer(uint8_t *const buffer, const int bufferSize, const int mmapFd,
+            const int bufferOffset, const bool isUpdatable)
+            : mBuffer(buffer), mBufferSize(bufferSize), mMmapFd(mmapFd),
+              mBufferOffset(bufferOffset), mIsUpdatable(isUpdatable) {}
+
+    DISALLOW_IMPLICIT_CONSTRUCTORS(MmapedBuffer);
+
+    uint8_t *const mBuffer;
+    const int mBufferSize;
+    const int mMmapFd;
+    const int mBufferOffset;
+    const bool mIsUpdatable;
+};
+}
+#endif /* LATINIME_MMAPED_BUFFER_H */
diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
index 175e511..c6fa943 100644
--- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
+++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
@@ -256,4 +256,16 @@
         // code for now True is acceptable.
         assertTrue(StringUtils.lastPartLooksLikeURL(".abc/def"));
     }
+
+    public void testHexStringUtils() {
+        final byte[] bytes = new byte[] { (byte)0x01, (byte)0x11, (byte)0x22, (byte)0x33,
+                (byte)0x55, (byte)0x88, (byte)0xEE };
+        final String bytesStr = StringUtils.byteArrayToHexString(bytes);
+        final byte[] bytes2 = StringUtils.hexStringToByteArray(bytesStr);
+        for (int i = 0; i < bytes.length; ++i) {
+            assertTrue(bytes[i] == bytes2[i]);
+        }
+        final String bytesStr2 = StringUtils.byteArrayToHexString(bytes2);
+        assertTrue(bytesStr.equals(bytesStr2));
+    }
 }