Detach bigram functionarities from unigram_dictionary
Change-Id: Ie35164a5f293e5370885a1ba13d6ed7caf6000ec
diff --git a/native/src/bigram_dictionary.cpp b/native/src/bigram_dictionary.cpp
index f0d5f8c..095b805 100644
--- a/native/src/bigram_dictionary.cpp
+++ b/native/src/bigram_dictionary.cpp
@@ -15,19 +15,240 @@
** limitations under the License.
*/
+#include <string.h>
+
#define LOG_TAG "LatinIME: bigram_dictionary.cpp"
#include "bigram_dictionary.h"
+#include "dictionary.h"
namespace latinime {
-BigramDictionary::BigramDictionary(void *dict, int typedLetterMultiplier, int fullWordMultiplier,
- int maxWordLength, int maxWords, int maxAlternatives, Dictionary *parentDictionary)
-{
+BigramDictionary::BigramDictionary(const unsigned char *dict, int maxWordLength,
+ int maxAlternatives, const bool isLatestDictVersion, const bool hasBigram,
+ Dictionary *parentDictionary)
+ : DICT(dict), MAX_WORD_LENGTH(maxWordLength),
+ MAX_ALTERNATIVES(maxAlternatives), IS_LATEST_DICT_VERSION(isLatestDictVersion),
+ HAS_BIGRAM(hasBigram), mParentDictionary(parentDictionary) {
+ LOGI("BigramDictionary - constructor");
+ LOGI("Has Bigram : %d \n", hasBigram);
}
-BigramDictionary::~BigramDictionary()
-{
+BigramDictionary::~BigramDictionary() {
}
+
+bool BigramDictionary::addWordBigram(unsigned short *word, int length, int frequency) {
+ word[length] = 0;
+ if (DEBUG_DICT) {
+ char s[length + 1];
+ for (int i = 0; i <= length; i++) s[i] = word[i];
+ LOGI("Bigram: Found word = %s, freq = %d : \n", s, frequency);
+ }
+
+ // Find the right insertion point
+ int insertAt = 0;
+ while (insertAt < mMaxBigrams) {
+ if (frequency > mBigramFreq[insertAt] || (mBigramFreq[insertAt] == frequency
+ && length < Dictionary::wideStrLen(mBigramChars + insertAt * MAX_WORD_LENGTH))) {
+ break;
+ }
+ insertAt++;
+ }
+ LOGI("Bigram: InsertAt -> %d maxBigrams: %d\n", insertAt, mMaxBigrams);
+ if (insertAt < mMaxBigrams) {
+ memmove((char*) mBigramFreq + (insertAt + 1) * sizeof(mBigramFreq[0]),
+ (char*) mBigramFreq + insertAt * sizeof(mBigramFreq[0]),
+ (mMaxBigrams - insertAt - 1) * sizeof(mBigramFreq[0]));
+ mBigramFreq[insertAt] = frequency;
+ memmove((char*) mBigramChars + (insertAt + 1) * MAX_WORD_LENGTH * sizeof(short),
+ (char*) mBigramChars + (insertAt ) * MAX_WORD_LENGTH * sizeof(short),
+ (mMaxBigrams - insertAt - 1) * sizeof(short) * MAX_WORD_LENGTH);
+ unsigned short *dest = mBigramChars + (insertAt ) * MAX_WORD_LENGTH;
+ while (length--) {
+ *dest++ = *word++;
+ }
+ *dest = 0; // NULL terminate
+ if (DEBUG_DICT) LOGI("Bigram: Added word at %d\n", insertAt);
+ return true;
+ }
+ return false;
+}
+
+int BigramDictionary::getBigramAddress(int *pos, bool advance) {
+ int address = 0;
+
+ address += (DICT[*pos] & 0x3F) << 16;
+ address += (DICT[*pos + 1] & 0xFF) << 8;
+ address += (DICT[*pos + 2] & 0xFF);
+
+ if (advance) {
+ *pos += 3;
+ }
+
+ return address;
+}
+
+int BigramDictionary::getBigramFreq(int *pos) {
+ int freq = DICT[(*pos)++] & FLAG_BIGRAM_FREQ;
+
+ return freq;
+}
+
+
+int BigramDictionary::getBigrams(unsigned short *prevWord, int prevWordLength, int *codes,
+ int codesSize, unsigned short *bigramChars, int *bigramFreq, int maxWordLength,
+ int maxBigrams, int maxAlternatives) {
+ mBigramFreq = bigramFreq;
+ mBigramChars = bigramChars;
+ mInputCodes = codes;
+ mInputLength = codesSize;
+ mMaxBigrams = maxBigrams;
+
+ if (HAS_BIGRAM && IS_LATEST_DICT_VERSION) {
+ int pos = mParentDictionary->isValidWordRec(
+ DICTIONARY_HEADER_SIZE, prevWord, 0, prevWordLength);
+ LOGI("Pos -> %d\n", pos);
+ if (pos < 0) {
+ return 0;
+ }
+
+ int bigramCount = 0;
+ int bigramExist = (DICT[pos] & FLAG_BIGRAM_READ);
+ if (bigramExist > 0) {
+ int nextBigramExist = 1;
+ while (nextBigramExist > 0 && bigramCount < maxBigrams) {
+ int bigramAddress = getBigramAddress(&pos, true);
+ int frequency = (FLAG_BIGRAM_FREQ & DICT[pos]);
+ // search for all bigrams and store them
+ searchForTerminalNode(bigramAddress, frequency);
+ nextBigramExist = (DICT[pos++] & FLAG_BIGRAM_CONTINUED);
+ bigramCount++;
+ }
+ }
+
+ return bigramCount;
+ }
+ return 0;
+}
+
+void BigramDictionary::searchForTerminalNode(int addressLookingFor, int frequency) {
+ // track word with such address and store it in an array
+ unsigned short word[MAX_WORD_LENGTH];
+
+ int pos;
+ int followDownBranchAddress = DICTIONARY_HEADER_SIZE;
+ bool found = false;
+ char followingChar = ' ';
+ int depth = -1;
+
+ while(!found) {
+ bool followDownAddressSearchStop = false;
+ bool firstAddress = true;
+ bool haveToSearchAll = true;
+
+ if (depth >= 0) {
+ word[depth] = (unsigned short) followingChar;
+ }
+ pos = followDownBranchAddress; // pos start at count
+ int count = DICT[pos] & 0xFF;
+ LOGI("count - %d\n",count);
+ pos++;
+ for (int i = 0; i < count; i++) {
+ // pos at data
+ pos++;
+ // pos now at flag
+ if (!getFirstBitOfByte(&pos)) { // non-terminal
+ if (!followDownAddressSearchStop) {
+ int addr = getBigramAddress(&pos, false);
+ if (addr > addressLookingFor) {
+ followDownAddressSearchStop = true;
+ if (firstAddress) {
+ firstAddress = false;
+ haveToSearchAll = true;
+ } else if (!haveToSearchAll) {
+ break;
+ }
+ } else {
+ followDownBranchAddress = addr;
+ followingChar = (char)(0xFF & DICT[pos-1]);
+ if (firstAddress) {
+ firstAddress = false;
+ haveToSearchAll = false;
+ }
+ }
+ }
+ pos += 3;
+ } else if (getFirstBitOfByte(&pos)) { // terminal
+ if (addressLookingFor == (pos-1)) { // found !!
+ depth++;
+ word[depth] = (0xFF & DICT[pos-1]);
+ found = true;
+ break;
+ }
+ if (getSecondBitOfByte(&pos)) { // address + freq (4 byte)
+ if (!followDownAddressSearchStop) {
+ int addr = getBigramAddress(&pos, false);
+ if (addr > addressLookingFor) {
+ followDownAddressSearchStop = true;
+ if (firstAddress) {
+ firstAddress = false;
+ haveToSearchAll = true;
+ } else if (!haveToSearchAll) {
+ break;
+ }
+ } else {
+ followDownBranchAddress = addr;
+ followingChar = (char)(0xFF & DICT[pos-1]);
+ if (firstAddress) {
+ firstAddress = false;
+ haveToSearchAll = true;
+ }
+ }
+ }
+ pos += 4;
+ } else { // freq only (2 byte)
+ pos += 2;
+ }
+
+ // skipping bigram
+ int bigramExist = (DICT[pos] & FLAG_BIGRAM_READ);
+ if (bigramExist > 0) {
+ int nextBigramExist = 1;
+ while (nextBigramExist > 0) {
+ pos += 3;
+ nextBigramExist = (DICT[pos++] & FLAG_BIGRAM_CONTINUED);
+ }
+ } else {
+ pos++;
+ }
+ }
+ }
+ depth++;
+ if (followDownBranchAddress == 0) {
+ LOGI("ERROR!!! Cannot find bigram!!");
+ break;
+ }
+ }
+ if (checkFirstCharacter(word)) {
+ addWordBigram(word, depth, frequency);
+ }
+}
+
+bool BigramDictionary::checkFirstCharacter(unsigned short *word) {
+ // Checks whether this word starts with same character or neighboring characters of
+ // what user typed.
+
+ int *inputCodes = mInputCodes;
+ int maxAlt = MAX_ALTERNATIVES;
+ while (maxAlt > 0) {
+ if ((unsigned int) *inputCodes == (unsigned int) *word) {
+ return true;
+ }
+ inputCodes++;
+ maxAlt--;
+ }
+ return false;
+}
+
// TODO: Move functions related to bigram to here
} // namespace latinime