blob: 157e47db2b52669974e8c26cab7ab178dc39670c [file] [log] [blame]
satok30088252010-12-01 21:22:15 +09001/*
2**
3** Copyright 2010, The Android Open Source Project
4**
5** Licensed under the Apache License, Version 2.0 (the "License");
6** you may not use this file except in compliance with the License.
7** You may obtain a copy of the License at
8**
9** http://www.apache.org/licenses/LICENSE-2.0
10**
11** Unless required by applicable law or agreed to in writing, software
12** distributed under the License is distributed on an "AS IS" BASIS,
13** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14** See the License for the specific language governing permissions and
15** limitations under the License.
16*/
17
satok48e432c2010-12-06 17:38:58 +090018#include <assert.h>
satok30088252010-12-01 21:22:15 +090019#include <string.h>
20
satoke808e432010-12-02 14:53:24 +090021#define LOG_TAG "LatinIME: unigram_dictionary.cpp"
satok30088252010-12-01 21:22:15 +090022
satok30088252010-12-01 21:22:15 +090023#include "char_utils.h"
Ken Wakasa3b088a22012-05-16 23:05:32 +090024#include "defines.h"
satoke808e432010-12-02 14:53:24 +090025#include "dictionary.h"
26#include "unigram_dictionary.h"
satok30088252010-12-01 21:22:15 +090027
Jean Chalard1059f272011-06-28 20:45:05 +090028#include "binary_format.h"
Jean Chalardcf9dbbd2011-12-26 15:16:59 +090029#include "terminal_attributes.h"
Jean Chalard1059f272011-06-28 20:45:05 +090030
satok30088252010-12-01 21:22:15 +090031namespace latinime {
32
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090033const UnigramDictionary::digraph_t UnigramDictionary::GERMAN_UMLAUT_DIGRAPHS[] =
Jean Chalardd3043382012-03-21 18:38:25 +090034 { { 'a', 'e', 0x00E4 }, // U+00E4 : LATIN SMALL LETTER A WITH DIAERESIS
35 { 'o', 'e', 0x00F6 }, // U+00F6 : LATIN SMALL LETTER O WITH DIAERESIS
36 { 'u', 'e', 0x00FC } }; // U+00FC : LATIN SMALL LETTER U WITH DIAERESIS
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090037
Jean Chalardcc78d032012-03-23 16:48:49 +090038const UnigramDictionary::digraph_t UnigramDictionary::FRENCH_LIGATURES_DIGRAPHS[] =
39 { { 'a', 'e', 0x00E6 }, // U+00E6 : LATIN SMALL LETTER AE
40 { 'o', 'e', 0x0153 } }; // U+0153 : LATIN SMALL LIGATURE OE
41
Jean Chalard293ece02011-06-16 20:55:16 +090042// TODO: check the header
43UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultiplier,
Jean Chalardcd274b12012-04-06 18:26:00 +090044 int fullWordMultiplier, int maxWordLength, int maxWords, const unsigned int flags)
Jean Chalard46a1eec2012-02-27 19:48:47 +090045 : DICT_ROOT(streamStart), MAX_WORD_LENGTH(maxWordLength), MAX_WORDS(maxWords),
satok662fe692010-12-08 17:05:39 +090046 TYPED_LETTER_MULTIPLIER(typedLetterMultiplier), FULL_WORD_MULTIPLIER(fullWordMultiplier),
Jean Chalard1059f272011-06-28 20:45:05 +090047 // TODO : remove this variable.
48 ROOT_POS(0),
satok9df4a452012-03-23 16:05:18 +090049 BYTES_IN_ONE_CHAR(sizeof(int)),
Jean Chalardcd274b12012-04-06 18:26:00 +090050 MAX_DIGRAPH_SEARCH_DEPTH(DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH), FLAGS(flags) {
Ken Wakasade3070a2011-03-19 09:16:42 +090051 if (DEBUG_DICT) {
satok9fb6f472012-01-13 18:01:22 +090052 AKLOGI("UnigramDictionary - constructor");
Ken Wakasade3070a2011-03-19 09:16:42 +090053 }
satok30088252010-12-01 21:22:15 +090054}
55
satok2df30602011-07-15 13:49:00 +090056UnigramDictionary::~UnigramDictionary() {
satok2df30602011-07-15 13:49:00 +090057}
satok30088252010-12-01 21:22:15 +090058
satok9df4a452012-03-23 16:05:18 +090059static inline unsigned int getCodesBufferSize(const int *codes, const int codesSize) {
60 return sizeof(*codes) * codesSize;
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090061}
62
Ken Wakasa951ab9d2012-03-09 19:18:59 +090063// TODO: This needs to take a const unsigned short* and not tinker with its contents
satok1147c7b2011-12-14 15:04:58 +090064static inline void addWord(
65 unsigned short *word, int length, int frequency, WordsPriorityQueue *queue) {
66 queue->push(frequency, word, length);
67}
68
Jean Chalardd3043382012-03-21 18:38:25 +090069// Return the replacement code point for a digraph, or 0 if none.
70int UnigramDictionary::getDigraphReplacement(const int *codes, const int i, const int codesSize,
Jean Chalard6c300612012-03-06 19:54:03 +090071 const digraph_t* const digraphs, const unsigned int digraphsSize) const {
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090072
73 // There can't be a digraph if we don't have at least 2 characters to examine
74 if (i + 2 > codesSize) return false;
75
76 // Search for the first char of some digraph
77 int lastDigraphIndex = -1;
satok9df4a452012-03-23 16:05:18 +090078 const int thisChar = codes[i];
Jean Chalard6c300612012-03-06 19:54:03 +090079 for (lastDigraphIndex = digraphsSize - 1; lastDigraphIndex >= 0; --lastDigraphIndex) {
80 if (thisChar == digraphs[lastDigraphIndex].first) break;
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090081 }
82 // No match: return early
Jean Chalardd3043382012-03-21 18:38:25 +090083 if (lastDigraphIndex < 0) return 0;
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090084
85 // It's an interesting digraph if the second char matches too.
satok9df4a452012-03-23 16:05:18 +090086 if (digraphs[lastDigraphIndex].second == codes[i + 1]) {
Jean Chalardd3043382012-03-21 18:38:25 +090087 return digraphs[lastDigraphIndex].replacement;
88 } else {
89 return 0;
90 }
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090091}
92
93// Mostly the same arguments as the non-recursive version, except:
94// codes is the original value. It points to the start of the work buffer, and gets passed as is.
95// codesSize is the size of the user input (thus, it is the size of codesSrc).
96// codesDest is the current point in the work buffer.
97// codesSrc is the current point in the user-input, original, content-unmodified buffer.
98// codesRemain is the remaining size in codesSrc.
satok1d7eaf82011-07-13 10:32:02 +090099void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
satok1147c7b2011-12-14 15:04:58 +0900100 const int *xcoordinates, const int *ycoordinates, const int *codesBuffer,
satok219a5142012-03-08 11:53:18 +0900101 int *xCoordinatesBuffer, int *yCoordinatesBuffer,
Jean Chalard8950ce62012-05-07 16:28:30 +0900102 const int codesBufferSize, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
Jean Chalard4d9b2022012-04-23 19:25:28 +0900103 const bool useFullEditDistance, const int *codesSrc,
satok1147c7b2011-12-14 15:04:58 +0900104 const int codesRemain, const int currentDepth, int *codesDest, Correction *correction,
Jean Chalard6c300612012-03-06 19:54:03 +0900105 WordsPriorityQueuePool *queuePool,
106 const digraph_t* const digraphs, const unsigned int digraphsSize) {
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900107
satok9df4a452012-03-23 16:05:18 +0900108 const int startIndex = codesDest - codesBuffer;
Jean Chalard6c300612012-03-06 19:54:03 +0900109 if (currentDepth < MAX_DIGRAPH_SEARCH_DEPTH) {
Jean Chalarda787dba2011-03-04 12:17:48 +0900110 for (int i = 0; i < codesRemain; ++i) {
satok219a5142012-03-08 11:53:18 +0900111 xCoordinatesBuffer[startIndex + i] = xcoordinates[codesBufferSize - codesRemain + i];
112 yCoordinatesBuffer[startIndex + i] = ycoordinates[codesBufferSize - codesRemain + i];
Jean Chalardd3043382012-03-21 18:38:25 +0900113 const int replacementCodePoint =
114 getDigraphReplacement(codesSrc, i, codesRemain, digraphs, digraphsSize);
115 if (0 != replacementCodePoint) {
Jean Chalarda787dba2011-03-04 12:17:48 +0900116 // Found a digraph. We will try both spellings. eg. the word is "pruefen"
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900117
Jean Chalardd3043382012-03-21 18:38:25 +0900118 // Copy the word up to the first char of the digraph, including proximity chars,
119 // and overwrite the primary code with the replacement code point. Then, continue
120 // processing on the remaining part of the word, skipping the second char of the
121 // digraph.
122 // In our example, copy "pru", replace "u" with the version with the diaeresis and
123 // continue running on "fen".
Jean Chalarda787dba2011-03-04 12:17:48 +0900124 // Make i the index of the second char of the digraph for simplicity. Forgetting
125 // to do that results in an infinite recursion so take care!
126 ++i;
127 memcpy(codesDest, codesSrc, i * BYTES_IN_ONE_CHAR);
Jean Chalardd3043382012-03-21 18:38:25 +0900128 codesDest[(i - 1) * (BYTES_IN_ONE_CHAR / sizeof(codesDest[0]))] =
129 replacementCodePoint;
Jean Chalarda787dba2011-03-04 12:17:48 +0900130 getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
Jean Chalard338d3ec2012-04-06 19:28:34 +0900131 codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesBufferSize,
Jean Chalard8950ce62012-05-07 16:28:30 +0900132 bigramMap, bigramFilter, useFullEditDistance, codesSrc + i + 1,
Jean Chalard4d9b2022012-04-23 19:25:28 +0900133 codesRemain - i - 1, currentDepth + 1, codesDest + i, correction,
Jean Chalard6c300612012-03-06 19:54:03 +0900134 queuePool, digraphs, digraphsSize);
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900135
Jean Chalarda787dba2011-03-04 12:17:48 +0900136 // Copy the second char of the digraph in place, then continue processing on
137 // the remaining part of the word.
138 // In our example, after "pru" in the buffer copy the "e", and continue on "fen"
satok9df4a452012-03-23 16:05:18 +0900139 memcpy(codesDest + i, codesSrc + i, BYTES_IN_ONE_CHAR);
Jean Chalarda787dba2011-03-04 12:17:48 +0900140 getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
Jean Chalard338d3ec2012-04-06 19:28:34 +0900141 codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesBufferSize,
Jean Chalard8950ce62012-05-07 16:28:30 +0900142 bigramMap, bigramFilter, useFullEditDistance, codesSrc + i, codesRemain - i,
Jean Chalard4d9b2022012-04-23 19:25:28 +0900143 currentDepth + 1, codesDest + i, correction, queuePool, digraphs,
144 digraphsSize);
Jean Chalarda787dba2011-03-04 12:17:48 +0900145 return;
146 }
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900147 }
148 }
149
150 // If we come here, we hit the end of the word: let's check it against the dictionary.
151 // In our example, we'll come here once for "prufen" and then once for "pruefen".
152 // If the word contains several digraphs, we'll come it for the product of them.
153 // eg. if the word is "ueberpruefen" we'll test, in order, against
154 // "uberprufen", "uberpruefen", "ueberprufen", "ueberpruefen".
155 const unsigned int remainingBytes = BYTES_IN_ONE_CHAR * codesRemain;
satok219a5142012-03-08 11:53:18 +0900156 if (0 != remainingBytes) {
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900157 memcpy(codesDest, codesSrc, remainingBytes);
satok219a5142012-03-08 11:53:18 +0900158 memcpy(&xCoordinatesBuffer[startIndex], &xcoordinates[codesBufferSize - codesRemain],
159 sizeof(int) * codesRemain);
160 memcpy(&yCoordinatesBuffer[startIndex], &ycoordinates[codesBufferSize - codesRemain],
161 sizeof(int) * codesRemain);
162 }
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900163
satok219a5142012-03-08 11:53:18 +0900164 getWordSuggestions(proximityInfo, xCoordinatesBuffer, yCoordinatesBuffer, codesBuffer,
Jean Chalard8950ce62012-05-07 16:28:30 +0900165 startIndex + codesRemain, bigramMap, bigramFilter, useFullEditDistance, correction,
satoka7e5a5a2011-12-15 16:49:12 +0900166 queuePool);
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900167}
168
Jean Chalard8950ce62012-05-07 16:28:30 +0900169// bigramMap contains the association <bigram address> -> <bigram frequency>
170// bigramFilter is a bloom filter for fast rejection: see functions setInFilter and isInFilter
171// in bigram_dictionary.cpp
satoka7e5a5a2011-12-15 16:49:12 +0900172int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo,
173 WordsPriorityQueuePool *queuePool, Correction *correction, const int *xcoordinates,
Jean Chalard338d3ec2012-04-06 19:28:34 +0900174 const int *ycoordinates, const int *codes, const int codesSize,
Jean Chalard8950ce62012-05-07 16:28:30 +0900175 const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
176 const bool useFullEditDistance, unsigned short *outWords, int *frequencies) {
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900177
satok219a5142012-03-08 11:53:18 +0900178 queuePool->clearAll();
satok1147c7b2011-12-14 15:04:58 +0900179 Correction* masterCorrection = correction;
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900180 correction->resetCorrection();
Jean Chalardaa8df592012-04-06 18:54:20 +0900181 if (BinaryFormat::REQUIRES_GERMAN_UMLAUT_PROCESSING & FLAGS)
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900182 { // Incrementally tune the word and try all possibilities
satok9df4a452012-03-23 16:05:18 +0900183 int codesBuffer[getCodesBufferSize(codes, codesSize)];
satok219a5142012-03-08 11:53:18 +0900184 int xCoordinatesBuffer[codesSize];
185 int yCoordinatesBuffer[codesSize];
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900186 getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
Jean Chalard8950ce62012-05-07 16:28:30 +0900187 xCoordinatesBuffer, yCoordinatesBuffer, codesSize, bigramMap, bigramFilter,
Jean Chalard4d9b2022012-04-23 19:25:28 +0900188 useFullEditDistance, codes, codesSize, 0, codesBuffer, masterCorrection,
Jean Chalard338d3ec2012-04-06 19:28:34 +0900189 queuePool, GERMAN_UMLAUT_DIGRAPHS,
Jean Chalard6c300612012-03-06 19:54:03 +0900190 sizeof(GERMAN_UMLAUT_DIGRAPHS) / sizeof(GERMAN_UMLAUT_DIGRAPHS[0]));
Jean Chalardaa8df592012-04-06 18:54:20 +0900191 } else if (BinaryFormat::REQUIRES_FRENCH_LIGATURES_PROCESSING & FLAGS) {
satokacb6c542012-03-23 20:58:18 +0900192 int codesBuffer[getCodesBufferSize(codes, codesSize)];
Jean Chalardcc78d032012-03-23 16:48:49 +0900193 int xCoordinatesBuffer[codesSize];
194 int yCoordinatesBuffer[codesSize];
195 getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
Jean Chalard8950ce62012-05-07 16:28:30 +0900196 xCoordinatesBuffer, yCoordinatesBuffer, codesSize, bigramMap, bigramFilter,
Jean Chalard4d9b2022012-04-23 19:25:28 +0900197 useFullEditDistance, codes, codesSize, 0, codesBuffer, masterCorrection,
Jean Chalard338d3ec2012-04-06 19:28:34 +0900198 queuePool, FRENCH_LIGATURES_DIGRAPHS,
Jean Chalardcc78d032012-03-23 16:48:49 +0900199 sizeof(FRENCH_LIGATURES_DIGRAPHS) / sizeof(FRENCH_LIGATURES_DIGRAPHS[0]));
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900200 } else { // Normal processing
Jean Chalard338d3ec2012-04-06 19:28:34 +0900201 getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize,
Jean Chalard8950ce62012-05-07 16:28:30 +0900202 bigramMap, bigramFilter, useFullEditDistance, masterCorrection, queuePool);
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900203 }
204
satok817e5172011-03-04 06:06:45 -0800205 PROF_START(20);
satok8330b482012-01-23 16:52:37 +0900206 if (DEBUG_DICT) {
satok0028ed32012-05-16 20:42:12 +0900207 float ns = queuePool->getMasterQueue()->getHighestNormalizedScore(
satok8330b482012-01-23 16:52:37 +0900208 proximityInfo->getPrimaryInputWord(), codesSize, 0, 0, 0);
209 ns += 0;
210 AKLOGI("Max normalized score = %f", ns);
211 }
satoka7e5a5a2011-12-15 16:49:12 +0900212 const int suggestedWordsCount =
satokdb1939d2012-05-15 17:43:31 +0900213 queuePool->getMasterQueue()->outputSuggestions(
214 proximityInfo->getPrimaryInputWord(), codesSize, frequencies, outWords);
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900215
216 if (DEBUG_DICT) {
satok0028ed32012-05-16 20:42:12 +0900217 float ns = queuePool->getMasterQueue()->getHighestNormalizedScore(
satok8330b482012-01-23 16:52:37 +0900218 proximityInfo->getPrimaryInputWord(), codesSize, 0, 0, 0);
219 ns += 0;
satok9fb6f472012-01-13 18:01:22 +0900220 AKLOGI("Returning %d words", suggestedWordsCount);
Jean Chalard980d6b62011-06-30 17:02:23 +0900221 /// Print the returned words
222 for (int j = 0; j < suggestedWordsCount; ++j) {
satok16379df2011-12-12 20:53:22 +0900223 short unsigned int* w = outWords + j * MAX_WORD_LENGTH;
Jean Chalard980d6b62011-06-30 17:02:23 +0900224 char s[MAX_WORD_LENGTH];
225 for (int i = 0; i <= MAX_WORD_LENGTH; i++) s[i] = w[i];
Jean-Baptiste Querucd7c4132012-05-16 16:56:11 -0700226 (void)s;
satok9fb6f472012-01-13 18:01:22 +0900227 AKLOGI("%s %i", s, frequencies[j]);
Jean Chalard980d6b62011-06-30 17:02:23 +0900228 }
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900229 }
satok817e5172011-03-04 06:06:45 -0800230 PROF_END(20);
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900231 PROF_CLOSE;
232 return suggestedWordsCount;
233}
234
satok1d7eaf82011-07-13 10:32:02 +0900235void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
satok1147c7b2011-12-14 15:04:58 +0900236 const int *xcoordinates, const int *ycoordinates, const int *codes,
Jean Chalard8950ce62012-05-07 16:28:30 +0900237 const int inputLength, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
238 const bool useFullEditDistance, Correction *correction, WordsPriorityQueuePool *queuePool) {
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900239
satok61e2f852011-01-05 14:13:07 +0900240 PROF_OPEN;
241 PROF_START(0);
satok61e2f852011-01-05 14:13:07 +0900242 PROF_END(0);
satok30088252010-12-01 21:22:15 +0900243
satok61e2f852011-01-05 14:13:07 +0900244 PROF_START(1);
Jean Chalard8950ce62012-05-07 16:28:30 +0900245 getOneWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, bigramMap, bigramFilter,
Jean Chalard4d9b2022012-04-23 19:25:28 +0900246 useFullEditDistance, inputLength, correction, queuePool);
satok61e2f852011-01-05 14:13:07 +0900247 PROF_END(1);
248
249 PROF_START(2);
satok10266c02011-08-19 22:05:59 +0900250 // Note: This line is intentionally left blank
satok61e2f852011-01-05 14:13:07 +0900251 PROF_END(2);
satokcdbbea72010-12-08 16:04:16 +0900252
satok61e2f852011-01-05 14:13:07 +0900253 PROF_START(3);
satok10266c02011-08-19 22:05:59 +0900254 // Note: This line is intentionally left blank
satok61e2f852011-01-05 14:13:07 +0900255 PROF_END(3);
satok30088252010-12-01 21:22:15 +0900256
satok61e2f852011-01-05 14:13:07 +0900257 PROF_START(4);
satok8330b482012-01-23 16:52:37 +0900258 bool hasAutoCorrectionCandidate = false;
259 WordsPriorityQueue* masterQueue = queuePool->getMasterQueue();
260 if (masterQueue->size() > 0) {
satok0028ed32012-05-16 20:42:12 +0900261 float nsForMaster = masterQueue->getHighestNormalizedScore(
satok8330b482012-01-23 16:52:37 +0900262 proximityInfo->getPrimaryInputWord(), inputLength, 0, 0, 0);
263 hasAutoCorrectionCandidate = (nsForMaster > START_TWO_WORDS_CORRECTION_THRESHOLD);
264 }
satok61e2f852011-01-05 14:13:07 +0900265 PROF_END(4);
satoka3d78f62010-12-09 22:08:33 +0900266
satok61e2f852011-01-05 14:13:07 +0900267 PROF_START(5);
satok99557162012-01-26 22:49:13 +0900268 // Multiple word suggestions
269 if (SUGGEST_MULTIPLE_WORDS
270 && inputLength >= MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION) {
satoka85f4922012-01-30 18:18:30 +0900271 getSplitMultipleWordsSuggestions(proximityInfo, xcoordinates, ycoordinates, codes,
satok1f6b52e2012-01-30 13:53:58 +0900272 useFullEditDistance, inputLength, correction, queuePool,
273 hasAutoCorrectionCandidate);
satok662fe692010-12-08 17:05:39 +0900274 }
satok61e2f852011-01-05 14:13:07 +0900275 PROF_END(5);
satok817e5172011-03-04 06:06:45 -0800276
277 PROF_START(6);
satok99557162012-01-26 22:49:13 +0900278 // Note: This line is intentionally left blank
satok817e5172011-03-04 06:06:45 -0800279 PROF_END(6);
satok99557162012-01-26 22:49:13 +0900280
satok29dc8062012-01-17 15:59:15 +0900281 if (DEBUG_DICT) {
satok6ad15fc2012-01-16 16:21:21 +0900282 queuePool->dumpSubQueue1TopSuggestions();
satok29dc8062012-01-17 15:59:15 +0900283 for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) {
satok7409d152012-01-26 16:13:25 +0900284 WordsPriorityQueue* queue = queuePool->getSubQueue(FIRST_WORD_INDEX, i);
satok29dc8062012-01-17 15:59:15 +0900285 if (queue->size() > 0) {
286 WordsPriorityQueue::SuggestedWord* sw = queue->top();
287 const int score = sw->mScore;
288 const unsigned short* word = sw->mWord;
289 const int wordLength = sw->mWordLength;
satok0028ed32012-05-16 20:42:12 +0900290 float ns = Correction::RankingAlgorithm::calcNormalizedScore(
satok29dc8062012-01-17 15:59:15 +0900291 proximityInfo->getPrimaryInputWord(), i, word, wordLength, score);
292 ns += 0;
293 AKLOGI("--- TOP SUB WORDS for %d --- %d %f [%d]", i, score, ns,
satok54af64a2012-01-17 15:58:23 +0900294 (ns > TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD));
satok29dc8062012-01-17 15:59:15 +0900295 DUMP_WORD(proximityInfo->getPrimaryInputWord(), i);
296 DUMP_WORD(word, wordLength);
297 }
298 }
satok6ad15fc2012-01-16 16:21:21 +0900299 }
satok30088252010-12-01 21:22:15 +0900300}
301
Yusuke Nojima258bfe62011-09-28 12:59:43 +0900302void UnigramDictionary::initSuggestions(ProximityInfo *proximityInfo, const int *xCoordinates,
satok6ad15fc2012-01-16 16:21:21 +0900303 const int *yCoordinates, const int *codes, const int inputLength, Correction *correction) {
Ken Wakasade3070a2011-03-19 09:16:42 +0900304 if (DEBUG_DICT) {
satok9fb6f472012-01-13 18:01:22 +0900305 AKLOGI("initSuggest");
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900306 DUMP_WORD_INT(codes, inputLength);
Ken Wakasade3070a2011-03-19 09:16:42 +0900307 }
Satoshi Kataoka3e8c58f2012-06-05 17:55:52 +0900308 proximityInfo->initInputParams(codes, inputLength, xCoordinates, yCoordinates);
satok1a6da632011-12-16 23:15:06 +0900309 const int maxDepth = min(inputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH);
310 correction->initCorrection(proximityInfo, inputLength, maxDepth);
satok30088252010-12-01 21:22:15 +0900311}
312
satok715514d2010-12-02 20:19:59 +0900313static const char QUOTE = '\'';
satok662fe692010-12-08 17:05:39 +0900314static const char SPACE = ' ';
satok30088252010-12-01 21:22:15 +0900315
satok744dab62011-12-15 22:29:05 +0900316void UnigramDictionary::getOneWordSuggestions(ProximityInfo *proximityInfo,
317 const int *xcoordinates, const int *ycoordinates, const int *codes,
Jean Chalard8950ce62012-05-07 16:28:30 +0900318 const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
319 const bool useFullEditDistance, const int inputLength,
Jean Chalard4d9b2022012-04-23 19:25:28 +0900320 Correction *correction, WordsPriorityQueuePool *queuePool) {
satok6ad15fc2012-01-16 16:21:21 +0900321 initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, inputLength, correction);
Jean Chalard8950ce62012-05-07 16:28:30 +0900322 getSuggestionCandidates(useFullEditDistance, inputLength, bigramMap, bigramFilter, correction,
Jean Chalard4d9b2022012-04-23 19:25:28 +0900323 queuePool, true /* doAutoCompletion */, DEFAULT_MAX_ERRORS, FIRST_WORD_INDEX);
satok744dab62011-12-15 22:29:05 +0900324}
325
satok1147c7b2011-12-14 15:04:58 +0900326void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance,
Jean Chalard8950ce62012-05-07 16:28:30 +0900327 const int inputLength, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
Jean Chalard4d9b2022012-04-23 19:25:28 +0900328 Correction *correction, WordsPriorityQueuePool *queuePool,
satok8330b482012-01-23 16:52:37 +0900329 const bool doAutoCompletion, const int maxErrors, const int currentWordIndex) {
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900330 uint8_t totalTraverseCount = correction->pushAndGetTotalTraverseCount();
331 if (DEBUG_DICT) {
332 AKLOGI("Traverse count %d", totalTraverseCount);
333 }
334 if (totalTraverseCount > MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT) {
335 if (DEBUG_DICT) {
336 AKLOGI("Abort traversing %d", totalTraverseCount);
337 }
338 return;
339 }
satok10266c02011-08-19 22:05:59 +0900340 // TODO: Remove setCorrectionParams
satok1147c7b2011-12-14 15:04:58 +0900341 correction->setCorrectionParams(0, 0, 0,
satokd03317c2011-12-14 21:38:11 +0900342 -1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance,
satok1a6da632011-12-16 23:15:06 +0900343 doAutoCompletion, maxErrors);
satok662fe692010-12-08 17:05:39 +0900344 int rootPosition = ROOT_POS;
Jean Chalard980d6b62011-06-30 17:02:23 +0900345 // Get the number of children of root, then increment the position
Jean Chalard6d419812012-01-16 15:19:47 +0900346 int childCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &rootPosition);
satok208268d2011-08-10 15:44:08 +0900347 int outputIndex = 0;
satokd2997922010-12-07 13:08:39 +0900348
satok1147c7b2011-12-14 15:04:58 +0900349 correction->initCorrectionState(rootPosition, childCount, (inputLength <= 0));
satokd2997922010-12-07 13:08:39 +0900350
satok662fe692010-12-08 17:05:39 +0900351 // Depth first search
satok208268d2011-08-10 15:44:08 +0900352 while (outputIndex >= 0) {
satok1147c7b2011-12-14 15:04:58 +0900353 if (correction->initProcessState(outputIndex)) {
354 int siblingPos = correction->getTreeSiblingPos(outputIndex);
satokd2997922010-12-07 13:08:39 +0900355 int firstChildPos;
satok0f6c8e82011-08-03 02:19:44 +0900356
satok4e4e74e2011-08-03 23:27:32 +0900357 const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos,
Jean Chalard8950ce62012-05-07 16:28:30 +0900358 bigramMap, bigramFilter, correction, &childCount, &firstChildPos, &siblingPos,
Jean Chalard4d9b2022012-04-23 19:25:28 +0900359 queuePool, currentWordIndex);
satok662fe692010-12-08 17:05:39 +0900360 // Update next sibling pos
satok1147c7b2011-12-14 15:04:58 +0900361 correction->setTreeSiblingPos(outputIndex, siblingPos);
satok208268d2011-08-10 15:44:08 +0900362
satokd2997922010-12-07 13:08:39 +0900363 if (needsToTraverseChildrenNodes) {
364 // Goes to child node
satok1147c7b2011-12-14 15:04:58 +0900365 outputIndex = correction->goDownTree(outputIndex, childCount, firstChildPos);
satokd2997922010-12-07 13:08:39 +0900366 }
367 } else {
satokcdbbea72010-12-08 16:04:16 +0900368 // Goes to parent sibling node
satok1147c7b2011-12-14 15:04:58 +0900369 outputIndex = correction->getTreeParentIndex(outputIndex);
satokd2997922010-12-07 13:08:39 +0900370 }
371 }
372}
373
Jean Chalard171d1802012-04-23 18:51:00 +0900374inline void UnigramDictionary::onTerminal(const int probability,
Jean Chalardcf9dbbd2011-12-26 15:16:59 +0900375 const TerminalAttributes& terminalAttributes, Correction *correction,
satok8330b482012-01-23 16:52:37 +0900376 WordsPriorityQueuePool *queuePool, const bool addToMasterQueue,
377 const int currentWordIndex) {
satok6ad15fc2012-01-16 16:21:21 +0900378 const int inputIndex = correction->getInputIndex();
379 const bool addToSubQueue = inputIndex < SUB_QUEUE_MAX_COUNT;
satok54af64a2012-01-17 15:58:23 +0900380
satok8876b752011-08-04 18:31:57 +0900381 int wordLength;
382 unsigned short* wordPointer;
satok54af64a2012-01-17 15:58:23 +0900383
satok1f6b52e2012-01-30 13:53:58 +0900384 if ((currentWordIndex == FIRST_WORD_INDEX) && addToMasterQueue) {
satok54af64a2012-01-17 15:58:23 +0900385 WordsPriorityQueue *masterQueue = queuePool->getMasterQueue();
Jean Chalard171d1802012-04-23 18:51:00 +0900386 const int finalProbability =
387 correction->getFinalProbability(probability, &wordPointer, &wordLength);
388 if (finalProbability != NOT_A_PROBABILITY) {
389 addWord(wordPointer, wordLength, finalProbability, masterQueue);
satok54af64a2012-01-17 15:58:23 +0900390
Jean Chalard171d1802012-04-23 18:51:00 +0900391 const int shortcutProbability = finalProbability > 0 ? finalProbability - 1 : 0;
satok54af64a2012-01-17 15:58:23 +0900392 // Please note that the shortcut candidates will be added to the master queue only.
393 TerminalAttributes::ShortcutIterator iterator =
394 terminalAttributes.getShortcutIterator();
395 while (iterator.hasNextShortcutTarget()) {
396 // TODO: addWord only supports weak ordering, meaning we have no means
397 // to control the order of the shortcuts relative to one another or to the word.
Jean Chalard171d1802012-04-23 18:51:00 +0900398 // We need to either modulate the probability of each shortcut according
399 // to its own shortcut probability or to make the queue
satok54af64a2012-01-17 15:58:23 +0900400 // so that the insert order is protected inside the queue for words
Jean Chalard9a933a72012-03-27 19:56:23 +0900401 // with the same score. For the moment we use -1 to make sure the shortcut will
402 // never be in front of the word.
satok54af64a2012-01-17 15:58:23 +0900403 uint16_t shortcutTarget[MAX_WORD_LENGTH_INTERNAL];
404 const int shortcutTargetStringLength = iterator.getNextShortcutTarget(
405 MAX_WORD_LENGTH_INTERNAL, shortcutTarget);
Jean Chalard171d1802012-04-23 18:51:00 +0900406 addWord(shortcutTarget, shortcutTargetStringLength, shortcutProbability,
407 masterQueue);
satok6ad15fc2012-01-16 16:21:21 +0900408 }
Jean Chalardcf9dbbd2011-12-26 15:16:59 +0900409 }
satok54af64a2012-01-17 15:58:23 +0900410 }
satok6ad15fc2012-01-16 16:21:21 +0900411
satok54af64a2012-01-17 15:58:23 +0900412 // We only allow two words + other error correction for words with SUB_QUEUE_MIN_WORD_LENGTH
413 // or more length.
414 if (inputIndex >= SUB_QUEUE_MIN_WORD_LENGTH && addToSubQueue) {
satok8330b482012-01-23 16:52:37 +0900415 WordsPriorityQueue *subQueue;
satok7409d152012-01-26 16:13:25 +0900416 subQueue = queuePool->getSubQueue(currentWordIndex, inputIndex);
417 if (!subQueue) {
satok8330b482012-01-23 16:52:37 +0900418 return;
419 }
Jean Chalard171d1802012-04-23 18:51:00 +0900420 const int finalProbability = correction->getFinalProbabilityForSubQueue(
421 probability, &wordPointer, &wordLength, inputIndex);
422 addWord(wordPointer, wordLength, finalProbability, subQueue);
Jean Chalardca5ef282011-06-17 15:36:26 +0900423 }
424}
425
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900426int UnigramDictionary::getSubStringSuggestion(
satok7409d152012-01-26 16:13:25 +0900427 ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates,
satok3c09bb12012-01-26 18:36:19 +0900428 const int *codes, const bool useFullEditDistance, Correction *correction,
429 WordsPriorityQueuePool* queuePool, const int inputLength,
430 const bool hasAutoCorrectionCandidate, const int currentWordIndex,
431 const int inputWordStartPos, const int inputWordLength,
satok99557162012-01-26 22:49:13 +0900432 const int outputWordStartPos, const bool isSpaceProximity, int *freqArray,
433 int*wordLengthArray, unsigned short* outputWord, int *outputWordLength) {
Satoshi Kataoka67e3cc82012-05-31 15:04:58 +0900434 if (inputWordLength > MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH) {
435 return FLAG_MULTIPLE_SUGGEST_ABORT;
436 }
437
438 /////////////////////////////////////////////
439 // safety net for multiple word suggestion //
440 // TODO: Remove this safety net //
441 /////////////////////////////////////////////
442 int smallWordCount = 0;
443 int singleLetterWordCount = 0;
444 if (inputWordLength == 1) {
445 ++singleLetterWordCount;
446 }
447 if (inputWordLength <= 2) {
448 // small word == single letter or 2-letter word
449 ++smallWordCount;
450 }
451 for (int i = 0; i < currentWordIndex; ++i) {
452 const int length = wordLengthArray[i];
453 if (length == 1) {
454 ++singleLetterWordCount;
455 // Safety net to avoid suggesting sequential single letter words
456 if (i < (currentWordIndex - 1)) {
457 if (wordLengthArray[i + 1] == 1) {
458 return FLAG_MULTIPLE_SUGGEST_ABORT;
459 }
460 } else if (inputWordLength == 1) {
461 return FLAG_MULTIPLE_SUGGEST_ABORT;
462 }
463 }
464 if (length <= 2) {
465 ++smallWordCount;
466 }
467 // Safety net to avoid suggesting multiple words with many (4 or more, for now) small words
468 if (singleLetterWordCount >= 3 || smallWordCount >= 4) {
469 return FLAG_MULTIPLE_SUGGEST_ABORT;
470 }
471 }
472 //////////////////////////////////////////////
473 // TODO: Remove the safety net above //
474 //////////////////////////////////////////////
475
satok3c09bb12012-01-26 18:36:19 +0900476 unsigned short* tempOutputWord = 0;
satok1f6b52e2012-01-30 13:53:58 +0900477 int nextWordLength = 0;
satok99557162012-01-26 22:49:13 +0900478 // TODO: Optimize init suggestion
479 initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes,
480 inputLength, correction);
481
satok3c09bb12012-01-26 18:36:19 +0900482 int freq = getMostFrequentWordLike(
483 inputWordStartPos, inputWordLength, proximityInfo, mWord);
484 if (freq > 0) {
satok1f6b52e2012-01-30 13:53:58 +0900485 nextWordLength = inputWordLength;
satok3c09bb12012-01-26 18:36:19 +0900486 tempOutputWord = mWord;
487 } else if (!hasAutoCorrectionCandidate) {
488 if (inputWordStartPos > 0) {
489 const int offset = inputWordStartPos;
490 initSuggestions(proximityInfo, &xcoordinates[offset], &ycoordinates[offset],
satok9df4a452012-03-23 16:05:18 +0900491 codes + offset, inputWordLength, correction);
satok3c09bb12012-01-26 18:36:19 +0900492 queuePool->clearSubQueue(currentWordIndex);
Jean Chalard4d9b2022012-04-23 19:25:28 +0900493 // TODO: pass the bigram list for substring suggestion
494 getSuggestionCandidates(useFullEditDistance, inputWordLength,
Jean Chalard8950ce62012-05-07 16:28:30 +0900495 0 /* bigramMap */, 0 /* bigramFilter */, correction, queuePool,
496 false /* doAutoCompletion */, MAX_ERRORS_FOR_TWO_WORDS, currentWordIndex);
satok3c09bb12012-01-26 18:36:19 +0900497 if (DEBUG_DICT) {
satok1f6b52e2012-01-30 13:53:58 +0900498 if (currentWordIndex < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS) {
satok3c09bb12012-01-26 18:36:19 +0900499 AKLOGI("Dump word candidates(%d) %d", currentWordIndex, inputWordLength);
500 for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) {
501 queuePool->getSubQueue(currentWordIndex, i)->dumpTopWord();
502 }
503 }
504 }
505 }
506 WordsPriorityQueue* queue = queuePool->getSubQueue(currentWordIndex, inputWordLength);
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900507 // TODO: Return the correct value depending on doAutoCompletion
508 if (!queue || queue->size() <= 0) {
509 return FLAG_MULTIPLE_SUGGEST_ABORT;
satok3c09bb12012-01-26 18:36:19 +0900510 }
511 int score = 0;
satok0028ed32012-05-16 20:42:12 +0900512 const float ns = queue->getHighestNormalizedScore(
satok3c09bb12012-01-26 18:36:19 +0900513 proximityInfo->getPrimaryInputWord(), inputWordLength,
satok1f6b52e2012-01-30 13:53:58 +0900514 &tempOutputWord, &score, &nextWordLength);
satok3c09bb12012-01-26 18:36:19 +0900515 if (DEBUG_DICT) {
516 AKLOGI("NS(%d) = %f, Score = %d", currentWordIndex, ns, score);
517 }
518 // Two words correction won't be done if the score of the first word doesn't exceed the
519 // threshold.
520 if (ns < TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD
satok1f6b52e2012-01-30 13:53:58 +0900521 || nextWordLength < SUB_QUEUE_MIN_WORD_LENGTH) {
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900522 return FLAG_MULTIPLE_SUGGEST_SKIP;
satok3c09bb12012-01-26 18:36:19 +0900523 }
satok1f6b52e2012-01-30 13:53:58 +0900524 freq = score >> (nextWordLength + TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER);
satok3c09bb12012-01-26 18:36:19 +0900525 }
526 if (DEBUG_DICT) {
satok1f6b52e2012-01-30 13:53:58 +0900527 AKLOGI("Freq(%d): %d, length: %d, input length: %d, input start: %d (%d)"
528 , currentWordIndex, freq, nextWordLength, inputWordLength, inputWordStartPos,
529 wordLengthArray[0]);
satok3c09bb12012-01-26 18:36:19 +0900530 }
satok1f6b52e2012-01-30 13:53:58 +0900531 if (freq <= 0 || nextWordLength <= 0
532 || MAX_WORD_LENGTH <= (outputWordStartPos + nextWordLength)) {
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900533 return FLAG_MULTIPLE_SUGGEST_SKIP;
satok3c09bb12012-01-26 18:36:19 +0900534 }
satok1f6b52e2012-01-30 13:53:58 +0900535 for (int i = 0; i < nextWordLength; ++i) {
satok3c09bb12012-01-26 18:36:19 +0900536 outputWord[outputWordStartPos + i] = tempOutputWord[i];
537 }
satok99557162012-01-26 22:49:13 +0900538
539 // Put output values
satok1f6b52e2012-01-30 13:53:58 +0900540 freqArray[currentWordIndex] = freq;
satok99557162012-01-26 22:49:13 +0900541 // TODO: put output length instead of input length
satok1f6b52e2012-01-30 13:53:58 +0900542 wordLengthArray[currentWordIndex] = inputWordLength;
543 const int tempOutputWordLength = outputWordStartPos + nextWordLength;
544 if (outputWordLength) {
545 *outputWordLength = tempOutputWordLength;
546 }
satok99557162012-01-26 22:49:13 +0900547
satok3c09bb12012-01-26 18:36:19 +0900548 if ((inputWordStartPos + inputWordLength) < inputLength) {
satok1f6b52e2012-01-30 13:53:58 +0900549 if (outputWordStartPos + nextWordLength >= MAX_WORD_LENGTH) {
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900550 return FLAG_MULTIPLE_SUGGEST_SKIP;
satok3c09bb12012-01-26 18:36:19 +0900551 }
satoka85f4922012-01-30 18:18:30 +0900552 outputWord[tempOutputWordLength] = SPACE;
satok1f6b52e2012-01-30 13:53:58 +0900553 if (outputWordLength) {
554 ++*outputWordLength;
555 }
556 } else if (currentWordIndex >= 1) {
satok99557162012-01-26 22:49:13 +0900557 // TODO: Handle 3 or more words
satoka85f4922012-01-30 18:18:30 +0900558 const int pairFreq = correction->getFreqForSplitMultipleWords(
559 freqArray, wordLengthArray, currentWordIndex + 1, isSpaceProximity, outputWord);
satok99557162012-01-26 22:49:13 +0900560 if (DEBUG_DICT) {
satoka85f4922012-01-30 18:18:30 +0900561 DUMP_WORD(outputWord, tempOutputWordLength);
satoka0ac31f2012-05-23 19:55:27 +0900562 for (int i = 0; i < currentWordIndex + 1; ++i) {
563 AKLOGI("Split %d,%d words: freq = %d, length = %d", i, currentWordIndex + 1,
564 freqArray[i], wordLengthArray[i]);
565 }
566 AKLOGI("Split two words: freq = %d, length = %d, %d, isSpace ? %d", pairFreq,
567 inputLength, tempOutputWordLength, isSpaceProximity);
satok99557162012-01-26 22:49:13 +0900568 }
satok1f6b52e2012-01-30 13:53:58 +0900569 addWord(outputWord, tempOutputWordLength, pairFreq, queuePool->getMasterQueue());
satok3c09bb12012-01-26 18:36:19 +0900570 }
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900571 return FLAG_MULTIPLE_SUGGEST_CONTINUE;
satok7409d152012-01-26 16:13:25 +0900572}
573
satok1f6b52e2012-01-30 13:53:58 +0900574void UnigramDictionary::getMultiWordsSuggestionRec(ProximityInfo *proximityInfo,
575 const int *xcoordinates, const int *ycoordinates, const int *codes,
576 const bool useFullEditDistance, const int inputLength,
577 Correction *correction, WordsPriorityQueuePool* queuePool,
578 const bool hasAutoCorrectionCandidate, const int startInputPos, const int startWordIndex,
579 const int outputWordLength, int *freqArray, int* wordLengthArray,
580 unsigned short* outputWord) {
581 if (startWordIndex >= (MULTIPLE_WORDS_SUGGESTION_MAX_WORDS - 1)) {
582 // Return if the last word index
583 return;
584 }
satoka85f4922012-01-30 18:18:30 +0900585 if (startWordIndex >= 1
586 && (hasAutoCorrectionCandidate
587 || inputLength < MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION)) {
588 // Do not suggest 3+ words if already has auto correction candidate
589 return;
590 }
591 for (int i = startInputPos + 1; i < inputLength; ++i) {
satok1f6b52e2012-01-30 13:53:58 +0900592 if (DEBUG_CORRECTION_FREQ) {
satoka85f4922012-01-30 18:18:30 +0900593 AKLOGI("Multi words(%d), start in %d sep %d start out %d",
594 startWordIndex, startInputPos, i, outputWordLength);
595 DUMP_WORD(outputWord, outputWordLength);
satok1f6b52e2012-01-30 13:53:58 +0900596 }
satoka85f4922012-01-30 18:18:30 +0900597 int tempOutputWordLength = 0;
598 // Current word
599 int inputWordStartPos = startInputPos;
600 int inputWordLength = i - startInputPos;
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900601 const int suggestionFlag = getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates,
602 codes, useFullEditDistance, correction, queuePool, inputLength,
603 hasAutoCorrectionCandidate, startWordIndex, inputWordStartPos, inputWordLength,
604 outputWordLength, true /* not used */, freqArray, wordLengthArray, outputWord,
605 &tempOutputWordLength);
606 if (suggestionFlag == FLAG_MULTIPLE_SUGGEST_ABORT) {
607 // TODO: break here
608 continue;
609 } else if (suggestionFlag == FLAG_MULTIPLE_SUGGEST_SKIP) {
satok1f6b52e2012-01-30 13:53:58 +0900610 continue;
611 }
612
satoka85f4922012-01-30 18:18:30 +0900613 if (DEBUG_CORRECTION_FREQ) {
614 AKLOGI("Do missing space correction");
615 }
616 // Next word
satok1f6b52e2012-01-30 13:53:58 +0900617 // Missing space
618 inputWordStartPos = i;
619 inputWordLength = inputLength - i;
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900620 if(getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, codes,
satok1f6b52e2012-01-30 13:53:58 +0900621 useFullEditDistance, correction, queuePool, inputLength, hasAutoCorrectionCandidate,
satoka85f4922012-01-30 18:18:30 +0900622 startWordIndex + 1, inputWordStartPos, inputWordLength, tempOutputWordLength,
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900623 false /* missing space */, freqArray, wordLengthArray, outputWord, 0)
624 != FLAG_MULTIPLE_SUGGEST_CONTINUE) {
satoka85f4922012-01-30 18:18:30 +0900625 getMultiWordsSuggestionRec(proximityInfo, xcoordinates, ycoordinates, codes,
626 useFullEditDistance, inputLength, correction, queuePool,
627 hasAutoCorrectionCandidate, inputWordStartPos, startWordIndex + 1,
628 tempOutputWordLength, freqArray, wordLengthArray, outputWord);
629 }
satok1f6b52e2012-01-30 13:53:58 +0900630
631 // Mistyped space
632 ++inputWordStartPos;
633 --inputWordLength;
634
635 if (inputWordLength <= 0) {
636 continue;
637 }
638
639 const int x = xcoordinates[inputWordStartPos - 1];
640 const int y = ycoordinates[inputWordStartPos - 1];
641 if (!proximityInfo->hasSpaceProximity(x, y)) {
642 continue;
643 }
644
satoka85f4922012-01-30 18:18:30 +0900645 if (DEBUG_CORRECTION_FREQ) {
646 AKLOGI("Do mistyped space correction");
647 }
satok1f6b52e2012-01-30 13:53:58 +0900648 getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, codes,
649 useFullEditDistance, correction, queuePool, inputLength, hasAutoCorrectionCandidate,
satoka85f4922012-01-30 18:18:30 +0900650 startWordIndex + 1, inputWordStartPos, inputWordLength, tempOutputWordLength,
651 true /* mistyped space */, freqArray, wordLengthArray, outputWord, 0);
satok1f6b52e2012-01-30 13:53:58 +0900652 }
653}
654
satoka85f4922012-01-30 18:18:30 +0900655void UnigramDictionary::getSplitMultipleWordsSuggestions(ProximityInfo *proximityInfo,
satok744dab62011-12-15 22:29:05 +0900656 const int *xcoordinates, const int *ycoordinates, const int *codes,
satok1f6b52e2012-01-30 13:53:58 +0900657 const bool useFullEditDistance, const int inputLength,
satok99557162012-01-26 22:49:13 +0900658 Correction *correction, WordsPriorityQueuePool* queuePool,
satok8330b482012-01-23 16:52:37 +0900659 const bool hasAutoCorrectionCandidate) {
satokbd6ccdd2012-01-23 12:30:20 +0900660 if (inputLength >= MAX_WORD_LENGTH) return;
satok612c6e42011-08-01 19:35:27 +0900661 if (DEBUG_DICT) {
satok1f6b52e2012-01-30 13:53:58 +0900662 AKLOGI("--- Suggest multiple words");
663 }
satok54af64a2012-01-17 15:58:23 +0900664
satokbd6ccdd2012-01-23 12:30:20 +0900665 // Allocating fixed length array on stack
666 unsigned short outputWord[MAX_WORD_LENGTH];
satok1f6b52e2012-01-30 13:53:58 +0900667 int freqArray[MULTIPLE_WORDS_SUGGESTION_MAX_WORDS];
668 int wordLengthArray[MULTIPLE_WORDS_SUGGESTION_MAX_WORDS];
669 const int outputWordLength = 0;
670 const int startInputPos = 0;
671 const int startWordIndex = 0;
672 getMultiWordsSuggestionRec(proximityInfo, xcoordinates, ycoordinates, codes,
673 useFullEditDistance, inputLength, correction, queuePool, hasAutoCorrectionCandidate,
674 startInputPos, startWordIndex, outputWordLength, freqArray, wordLengthArray,
675 outputWord);
Jean Chalarde6715e32011-06-30 19:47:25 +0900676}
677
Jean Chalard1059f272011-06-28 20:45:05 +0900678// Wrapper for getMostFrequentWordLikeInner, which matches it to the previous
679// interface.
680inline int UnigramDictionary::getMostFrequentWordLike(const int startInputIndex,
satok1147c7b2011-12-14 15:04:58 +0900681 const int inputLength, ProximityInfo *proximityInfo, unsigned short *word) {
Jean Chalard1059f272011-06-28 20:45:05 +0900682 uint16_t inWord[inputLength];
683
684 for (int i = 0; i < inputLength; ++i) {
satok1147c7b2011-12-14 15:04:58 +0900685 inWord[i] = (uint16_t)proximityInfo->getPrimaryCharAt(startInputIndex + i);
Jean Chalard1059f272011-06-28 20:45:05 +0900686 }
687 return getMostFrequentWordLikeInner(inWord, inputLength, word);
688}
689
690// This function will take the position of a character array within a CharGroup,
691// and check it actually like-matches the word in inWord starting at startInputIndex,
692// that is, it matches it with case and accents squashed.
693// The function returns true if there was a full match, false otherwise.
694// The function will copy on-the-fly the characters in the CharGroup to outNewWord.
695// It will also place the end position of the array in outPos; in outInputIndex,
696// it will place the index of the first char AFTER the match if there was a match,
697// and the initial position if there was not. It makes sense because if there was
698// a match we want to continue searching, but if there was not, we want to go to
699// the next CharGroup.
700// In and out parameters may point to the same location. This function takes care
701// not to use any input parameters after it wrote into its outputs.
702static inline bool testCharGroupForContinuedLikeness(const uint8_t flags,
703 const uint8_t* const root, const int startPos,
704 const uint16_t* const inWord, const int startInputIndex,
705 int32_t* outNewWord, int* outInputIndex, int* outPos) {
706 const bool hasMultipleChars = (0 != (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & flags));
707 int pos = startPos;
708 int32_t character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos);
Tadashi G. Takaoka6e3cb272011-11-11 14:26:13 +0900709 int32_t baseChar = toBaseLowerCase(character);
710 const uint16_t wChar = toBaseLowerCase(inWord[startInputIndex]);
Jean Chalard1059f272011-06-28 20:45:05 +0900711
712 if (baseChar != wChar) {
713 *outPos = hasMultipleChars ? BinaryFormat::skipOtherCharacters(root, pos) : pos;
714 *outInputIndex = startInputIndex;
715 return false;
716 }
717 int inputIndex = startInputIndex;
718 outNewWord[inputIndex] = character;
719 if (hasMultipleChars) {
720 character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos);
721 while (NOT_A_CHARACTER != character) {
Tadashi G. Takaoka6e3cb272011-11-11 14:26:13 +0900722 baseChar = toBaseLowerCase(character);
723 if (toBaseLowerCase(inWord[++inputIndex]) != baseChar) {
Jean Chalard1059f272011-06-28 20:45:05 +0900724 *outPos = BinaryFormat::skipOtherCharacters(root, pos);
725 *outInputIndex = startInputIndex;
726 return false;
727 }
728 outNewWord[inputIndex] = character;
729 character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos);
730 }
731 }
732 *outInputIndex = inputIndex + 1;
733 *outPos = pos;
734 return true;
735}
736
737// This function is invoked when a word like the word searched for is found.
738// It will compare the frequency to the max frequency, and if greater, will
739// copy the word into the output buffer. In output value maxFreq, it will
740// write the new maximum frequency if it changed.
741static inline void onTerminalWordLike(const int freq, int32_t* newWord, const int length,
742 short unsigned int* outWord, int* maxFreq) {
743 if (freq > *maxFreq) {
744 for (int q = 0; q < length; ++q)
745 outWord[q] = newWord[q];
746 outWord[length] = 0;
747 *maxFreq = freq;
748 }
749}
750
751// Will find the highest frequency of the words like the one passed as an argument,
752// that is, everything that only differs by case/accents.
753int UnigramDictionary::getMostFrequentWordLikeInner(const uint16_t * const inWord,
754 const int length, short unsigned int* outWord) {
755 int32_t newWord[MAX_WORD_LENGTH_INTERNAL];
756 int depth = 0;
757 int maxFreq = -1;
758 const uint8_t* const root = DICT_ROOT;
759
Jean Chalard4c0eca62012-01-16 15:15:53 +0900760 int startPos = 0;
761 mStackChildCount[0] = BinaryFormat::getGroupCountAndForwardPointer(root, &startPos);
Jean Chalard1059f272011-06-28 20:45:05 +0900762 mStackInputIndex[0] = 0;
Jean Chalard4c0eca62012-01-16 15:15:53 +0900763 mStackSiblingPos[0] = startPos;
Jean Chalard1059f272011-06-28 20:45:05 +0900764 while (depth >= 0) {
765 const int charGroupCount = mStackChildCount[depth];
766 int pos = mStackSiblingPos[depth];
767 for (int charGroupIndex = charGroupCount - 1; charGroupIndex >= 0; --charGroupIndex) {
768 int inputIndex = mStackInputIndex[depth];
769 const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
770 // Test whether all chars in this group match with the word we are searching for. If so,
771 // we want to traverse its children (or if the length match, evaluate its frequency).
772 // Note that this function will output the position regardless, but will only write
773 // into inputIndex if there is a match.
774 const bool isAlike = testCharGroupForContinuedLikeness(flags, root, pos, inWord,
775 inputIndex, newWord, &inputIndex, &pos);
776 if (isAlike && (FLAG_IS_TERMINAL & flags) && (inputIndex == length)) {
777 const int frequency = BinaryFormat::readFrequencyWithoutMovingPointer(root, pos);
778 onTerminalWordLike(frequency, newWord, inputIndex, outWord, &maxFreq);
779 }
780 pos = BinaryFormat::skipFrequency(flags, pos);
781 const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(root, flags, pos);
782 const int childrenNodePos = BinaryFormat::readChildrenPosition(root, flags, pos);
783 // If we had a match and the word has children, we want to traverse them. We don't have
784 // to traverse words longer than the one we are searching for, since they will not match
785 // anyway, so don't traverse unless inputIndex < length.
786 if (isAlike && (-1 != childrenNodePos) && (inputIndex < length)) {
787 // Save position for this depth, to get back to this once children are done
788 mStackChildCount[depth] = charGroupIndex;
789 mStackSiblingPos[depth] = siblingPos;
790 // Prepare stack values for next depth
791 ++depth;
792 int childrenPos = childrenNodePos;
793 mStackChildCount[depth] =
794 BinaryFormat::getGroupCountAndForwardPointer(root, &childrenPos);
795 mStackSiblingPos[depth] = childrenPos;
796 mStackInputIndex[depth] = inputIndex;
797 pos = childrenPos;
798 // Go to the next depth level.
799 ++depth;
800 break;
801 } else {
802 // No match, or no children, or word too long to ever match: go the next sibling.
803 pos = siblingPos;
804 }
805 }
806 --depth;
807 }
808 return maxFreq;
809}
810
Satoshi Kataoka2f854e12012-05-29 15:58:13 +0900811int UnigramDictionary::getFrequency(const int32_t* const inWord, const int length) const {
812 const uint8_t* const root = DICT_ROOT;
813 int pos = BinaryFormat::getTerminalPosition(root, inWord, length);
814 if (NOT_VALID_WORD == pos) {
815 return NOT_A_PROBABILITY;
816 }
817 const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
818 const bool hasMultipleChars = (0 != (FLAG_HAS_MULTIPLE_CHARS & flags));
819 if (hasMultipleChars) {
820 pos = BinaryFormat::skipOtherCharacters(root, pos);
821 } else {
822 BinaryFormat::getCharCodeAndForwardPointer(DICT_ROOT, &pos);
823 }
824 const int unigramFreq = BinaryFormat::readFrequencyWithoutMovingPointer(root, pos);
825 return unigramFreq;
Jean Chalard1059f272011-06-28 20:45:05 +0900826}
827
828// TODO: remove this function.
829int UnigramDictionary::getBigramPosition(int pos, unsigned short *word, int offset,
830 int length) const {
831 return -1;
832}
833
834// ProcessCurrentNode returns a boolean telling whether to traverse children nodes or not.
835// If the return value is false, then the caller should read in the output "nextSiblingPosition"
836// to find out the address of the next sibling node and pass it to a new call of processCurrentNode.
837// It is worthy to note that when false is returned, the output values other than
838// nextSiblingPosition are undefined.
839// If the return value is true, then the caller must proceed to traverse the children of this
840// node. processCurrentNode will output the information about the children: their count in
841// newCount, their position in newChildrenPosition, the traverseAllNodes flag in
842// newTraverseAllNodes, the match weight into newMatchRate, the input index into newInputIndex, the
843// diffs into newDiffs, the sibling position in nextSiblingPosition, and the output index into
844// newOutputIndex. Please also note the following caveat: processCurrentNode does not know when
845// there aren't any more nodes at this level, it merely returns the address of the first byte after
846// the current node in nextSiblingPosition. Thus, the caller must keep count of the nodes at any
847// given level, as output into newCount when traversing this level's parent.
satok8876b752011-08-04 18:31:57 +0900848inline bool UnigramDictionary::processCurrentNode(const int initialPos,
Jean Chalard8950ce62012-05-07 16:28:30 +0900849 const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, Correction *correction,
850 int *newCount, int *newChildrenPosition, int *nextSiblingPosition,
851 WordsPriorityQueuePool *queuePool, const int currentWordIndex) {
Jean Chalard85a1d1e2011-06-21 22:23:21 +0900852 if (DEBUG_DICT) {
satokcfca3c62011-08-10 14:30:10 +0900853 correction->checkState();
Jean Chalard85a1d1e2011-06-21 22:23:21 +0900854 }
Jean Chalard0584f022011-06-30 19:23:16 +0900855 int pos = initialPos;
Jean Chalard0584f022011-06-30 19:23:16 +0900856
Jean Chalard1059f272011-06-28 20:45:05 +0900857 // Flags contain the following information:
858 // - Address type (MASK_GROUP_ADDRESS_TYPE) on two bits:
859 // - FLAG_GROUP_ADDRESS_TYPE_{ONE,TWO,THREE}_BYTES means there are children and their address
860 // is on the specified number of bytes.
861 // - FLAG_GROUP_ADDRESS_TYPE_NOADDRESS means there are no children, and therefore no address.
862 // - FLAG_HAS_MULTIPLE_CHARS: whether this node has multiple char or not.
863 // - FLAG_IS_TERMINAL: whether this node is a terminal or not (it may still have children)
864 // - FLAG_HAS_BIGRAMS: whether this node has bigrams or not
865 const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(DICT_ROOT, &pos);
866 const bool hasMultipleChars = (0 != (FLAG_HAS_MULTIPLE_CHARS & flags));
satok8876b752011-08-04 18:31:57 +0900867 const bool isTerminalNode = (0 != (FLAG_IS_TERMINAL & flags));
868
869 bool needsToInvokeOnTerminal = false;
Jean Chalard85a1d1e2011-06-21 22:23:21 +0900870
Jean Chalard1059f272011-06-28 20:45:05 +0900871 // This gets only ONE character from the stream. Next there will be:
872 // if FLAG_HAS_MULTIPLE CHARS: the other characters of the same node
873 // else if FLAG_IS_TERMINAL: the frequency
874 // else if MASK_GROUP_ADDRESS_TYPE is not NONE: the children address
875 // Note that you can't have a node that both is not a terminal and has no children.
876 int32_t c = BinaryFormat::getCharCodeAndForwardPointer(DICT_ROOT, &pos);
877 assert(NOT_A_CHARACTER != c);
Jean Chalard85a1d1e2011-06-21 22:23:21 +0900878
Jean Chalard1059f272011-06-28 20:45:05 +0900879 // We are going to loop through each character and make it look like it's a different
880 // node each time. To do that, we will process characters in this node in order until
881 // we find the character terminator. This is signalled by getCharCode* returning
882 // NOT_A_CHARACTER.
883 // As a special case, if there is only one character in this node, we must not read the
884 // next bytes so we will simulate the NOT_A_CHARACTER return by testing the flags.
885 // This way, each loop run will look like a "virtual node".
886 do {
887 // We prefetch the next char. If 'c' is the last char of this node, we will have
888 // NOT_A_CHARACTER in the next char. From this we can decide whether this virtual node
889 // should behave as a terminal or not and whether we have children.
890 const int32_t nextc = hasMultipleChars
891 ? BinaryFormat::getCharCodeAndForwardPointer(DICT_ROOT, &pos) : NOT_A_CHARACTER;
892 const bool isLastChar = (NOT_A_CHARACTER == nextc);
893 // If there are more chars in this nodes, then this virtual node is not a terminal.
894 // If we are on the last char, this virtual node is a terminal if this node is.
satok8876b752011-08-04 18:31:57 +0900895 const bool isTerminal = isLastChar && isTerminalNode;
Jean Chalard85a1d1e2011-06-21 22:23:21 +0900896
satokcfca3c62011-08-10 14:30:10 +0900897 Correction::CorrectionType stateType = correction->processCharAndCalcState(
satok8876b752011-08-04 18:31:57 +0900898 c, isTerminal);
satokcfca3c62011-08-10 14:30:10 +0900899 if (stateType == Correction::TRAVERSE_ALL_ON_TERMINAL
900 || stateType == Correction::ON_TERMINAL) {
satok8876b752011-08-04 18:31:57 +0900901 needsToInvokeOnTerminal = true;
satokd03317c2011-12-14 21:38:11 +0900902 } else if (stateType == Correction::UNRELATED || correction->needsToPrune()) {
satok8876b752011-08-04 18:31:57 +0900903 // We found that this is an unrelated character, so we should give up traversing
904 // this node and its children entirely.
905 // However we may not be on the last virtual node yet so we skip the remaining
906 // characters in this node, the frequency if it's there, read the next sibling
907 // position to output it, then return false.
908 // We don't have to output other values because we return false, as in
909 // "don't traverse children".
Jean Chalard1059f272011-06-28 20:45:05 +0900910 if (!isLastChar) {
911 pos = BinaryFormat::skipOtherCharacters(DICT_ROOT, pos);
912 }
913 pos = BinaryFormat::skipFrequency(flags, pos);
914 *nextSiblingPosition =
915 BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
916 return false;
Jean Chalard85a1d1e2011-06-21 22:23:21 +0900917 }
918
Jean Chalard1059f272011-06-28 20:45:05 +0900919 // Prepare for the next character. Promote the prefetched char to current char - the loop
920 // will take care of prefetching the next. If we finally found our last char, nextc will
921 // contain NOT_A_CHARACTER.
922 c = nextc;
Jean Chalard1059f272011-06-28 20:45:05 +0900923 } while (NOT_A_CHARACTER != c);
Jean Chalard85a1d1e2011-06-21 22:23:21 +0900924
satok8876b752011-08-04 18:31:57 +0900925 if (isTerminalNode) {
satok6ad15fc2012-01-16 16:21:21 +0900926 // The frequency should be here, because we come here only if this is actually
927 // a terminal node, and we are on its last char.
Jean Chalard171d1802012-04-23 18:51:00 +0900928 const int unigramFreq = BinaryFormat::readFrequencyWithoutMovingPointer(DICT_ROOT, pos);
satok6ad15fc2012-01-16 16:21:21 +0900929 const int childrenAddressPos = BinaryFormat::skipFrequency(flags, pos);
930 const int attributesPos = BinaryFormat::skipChildrenPosition(flags, childrenAddressPos);
931 TerminalAttributes terminalAttributes(DICT_ROOT, flags, attributesPos);
Jean Chalard8950ce62012-05-07 16:28:30 +0900932 // bigramMap contains the bigram frequencies indexed by addresses for fast lookup.
933 // bigramFilter is a bloom filter of said frequencies for even faster rejection.
Jean Chalard49ba1352012-05-07 20:14:00 +0900934 const int probability = BinaryFormat::getProbability(initialPos, bigramMap, bigramFilter,
935 unigramFreq);
Jean Chalard171d1802012-04-23 18:51:00 +0900936 onTerminal(probability, terminalAttributes, correction, queuePool, needsToInvokeOnTerminal,
satok8330b482012-01-23 16:52:37 +0900937 currentWordIndex);
Jean Chalard1059f272011-06-28 20:45:05 +0900938
satok8876b752011-08-04 18:31:57 +0900939 // If there are more chars in this node, then this virtual node has children.
940 // If we are on the last char, this virtual node has children if this node has.
941 const bool hasChildren = BinaryFormat::hasChildrenInFlags(flags);
942
943 // This character matched the typed character (enough to traverse the node at least)
944 // so we just evaluated it. Now we should evaluate this virtual node's children - that
945 // is, if it has any. If it has no children, we're done here - so we skip the end of
946 // the node, output the siblings position, and return false "don't traverse children".
947 // Note that !hasChildren implies isLastChar, so we know we don't have to skip any
948 // remaining char in this group for there can't be any.
949 if (!hasChildren) {
950 pos = BinaryFormat::skipFrequency(flags, pos);
951 *nextSiblingPosition =
952 BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
953 return false;
954 }
955
956 // Optimization: Prune out words that are too long compared to how much was typed.
satokcfca3c62011-08-10 14:30:10 +0900957 if (correction->needsToPrune()) {
satok8876b752011-08-04 18:31:57 +0900958 pos = BinaryFormat::skipFrequency(flags, pos);
959 *nextSiblingPosition =
960 BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
satok10266c02011-08-19 22:05:59 +0900961 if (DEBUG_DICT_FULL) {
satok9fb6f472012-01-13 18:01:22 +0900962 AKLOGI("Traversing was pruned.");
satok10266c02011-08-19 22:05:59 +0900963 }
satok8876b752011-08-04 18:31:57 +0900964 return false;
965 }
966 }
Jean Chalard1059f272011-06-28 20:45:05 +0900967
968 // Now we finished processing this node, and we want to traverse children. If there are no
969 // children, we can't come here.
970 assert(BinaryFormat::hasChildrenInFlags(flags));
971
972 // If this node was a terminal it still has the frequency under the pointer (it may have been
973 // read, but not skipped - see readFrequencyWithoutMovingPointer).
974 // Next come the children position, then possibly attributes (attributes are bigrams only for
975 // now, maybe something related to shortcuts in the future).
976 // Once this is read, we still need to output the number of nodes in the immediate children of
977 // this node, so we read and output it before returning true, as in "please traverse children".
978 pos = BinaryFormat::skipFrequency(flags, pos);
979 int childrenPos = BinaryFormat::readChildrenPosition(DICT_ROOT, flags, pos);
980 *nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
981 *newCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &childrenPos);
982 *newChildrenPosition = childrenPos;
983 return true;
Jean Chalard85a1d1e2011-06-21 22:23:21 +0900984}
985
satok30088252010-12-01 21:22:15 +0900986} // namespace latinime