blob: b85913ffb8c607c4005f469c833b23db7e2b070c [file] [log] [blame]
satok30088252010-12-01 21:22:15 +09001/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef LATINIME_UNIGRAM_DICTIONARY_H
18#define LATINIME_UNIGRAM_DICTIONARY_H
19
Jean Chalard293ece02011-06-16 20:55:16 +090020#include <stdint.h>
satokcfca3c62011-08-10 14:30:10 +090021#include "correction.h"
satok208268d2011-08-10 15:44:08 +090022#include "correction_state.h"
satoke808e432010-12-02 14:53:24 +090023#include "defines.h"
satok8fbd5522011-02-22 17:28:55 +090024#include "proximity_info.h"
satok16379df2011-12-12 20:53:22 +090025#include "words_priority_queue.h"
satoka7e5a5a2011-12-15 16:49:12 +090026#include "words_priority_queue_pool.h"
satoke808e432010-12-02 14:53:24 +090027
satok30088252010-12-01 21:22:15 +090028namespace latinime {
29
Jean Chalardcf9dbbd2011-12-26 15:16:59 +090030class TerminalAttributes;
satok30088252010-12-01 21:22:15 +090031class UnigramDictionary {
Jean Chalardd3043382012-03-21 18:38:25 +090032 typedef struct { int first; int second; int replacement; } digraph_t;
Jean Chalard6c300612012-03-06 19:54:03 +090033
Ken Wakasae12e9b52012-01-06 12:24:38 +090034 public:
Jean Chalard1059f272011-06-28 20:45:05 +090035 // Mask and flags for children address type selection.
36 static const int MASK_GROUP_ADDRESS_TYPE = 0xC0;
37 static const int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00;
38 static const int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40;
39 static const int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80;
40 static const int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0;
41
42 // Flag for single/multiple char group
43 static const int FLAG_HAS_MULTIPLE_CHARS = 0x20;
44
45 // Flag for terminal groups
46 static const int FLAG_IS_TERMINAL = 0x10;
47
Jean Chalardc8c65852011-12-27 12:58:23 +090048 // Flag for shortcut targets presence
49 static const int FLAG_HAS_SHORTCUT_TARGETS = 0x08;
Jean Chalard1059f272011-06-28 20:45:05 +090050 // Flag for bigram presence
51 static const int FLAG_HAS_BIGRAMS = 0x04;
Jean Chalardc8c65852011-12-27 12:58:23 +090052 // Flag for shortcut-only words. Some words are shortcut-only, which means they match when
53 // the user types them but they don't pop in the suggestion strip, only the words they are
54 // shortcuts for do.
55 static const int FLAG_IS_SHORTCUT_ONLY = 0x02;
Jean Chalard1059f272011-06-28 20:45:05 +090056
57 // Attribute (bigram/shortcut) related flags:
58 // Flag for presence of more attributes
59 static const int FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
60 // Flag for sign of offset. If this flag is set, the offset value must be negated.
61 static const int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
62
63 // Mask for attribute frequency, stored on 4 bits inside the flags byte.
64 static const int MASK_ATTRIBUTE_FREQUENCY = 0x0F;
65
66 // Mask and flags for attribute address type selection.
67 static const int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30;
68 static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
69 static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
70 static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
Jean Chalard1059f272011-06-28 20:45:05 +090071
satok4d355982011-12-15 14:53:19 +090072 // Error tolerances
73 static const int DEFAULT_MAX_ERRORS = 2;
74 static const int MAX_ERRORS_FOR_TWO_WORDS = 1;
75
Jean Chalard293ece02011-06-16 20:55:16 +090076 UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultipler,
77 int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars,
78 const bool isLatestDictVersion);
Jean Chalard1059f272011-06-28 20:45:05 +090079 bool isValidWord(const uint16_t* const inWord, const int length) const;
Jean Chalard581335c2011-06-17 12:45:17 +090080 int getBigramPosition(int pos, unsigned short *word, int offset, int length) const;
satoka7e5a5a2011-12-15 16:49:12 +090081 int getSuggestions(ProximityInfo *proximityInfo, WordsPriorityQueuePool *queuePool,
satok1147c7b2011-12-14 15:04:58 +090082 Correction *correction, const int *xcoordinates,
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090083 const int *ycoordinates, const int *codes, const int codesSize, const int flags,
84 unsigned short *outWords, int *frequencies);
satok2df30602011-07-15 13:49:00 +090085 virtual ~UnigramDictionary();
satok30088252010-12-01 21:22:15 +090086
Ken Wakasae12e9b52012-01-06 12:24:38 +090087 private:
satok1d7eaf82011-07-13 10:32:02 +090088 void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
satok1147c7b2011-12-14 15:04:58 +090089 const int *ycoordinates, const int *codes, const int inputLength,
satoka7e5a5a2011-12-15 16:49:12 +090090 const int flags, Correction *correction, WordsPriorityQueuePool *queuePool);
Jean Chalardd3043382012-03-21 18:38:25 +090091 int getDigraphReplacement(const int *codes, const int i, const int codesSize,
Jean Chalard6c300612012-03-06 19:54:03 +090092 const digraph_t* const digraphs, const unsigned int digraphsSize) const;
satok1d7eaf82011-07-13 10:32:02 +090093 void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090094 const int *xcoordinates, const int* ycoordinates, const int *codesBuffer,
satok219a5142012-03-08 11:53:18 +090095 int *xCoordinatesBuffer, int *yCoordinatesBuffer,
satok1147c7b2011-12-14 15:04:58 +090096 const int codesBufferSize, const int flags, const int* codesSrc,
97 const int codesRemain, const int currentDepth, int* codesDest, Correction *correction,
Jean Chalard6c300612012-03-06 19:54:03 +090098 WordsPriorityQueuePool* queuePool, const digraph_t* const digraphs,
99 const unsigned int digraphsSize);
satok1d7eaf82011-07-13 10:32:02 +0900100 void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
satok6ad15fc2012-01-16 16:21:21 +0900101 const int *ycoordinates, const int *codes, const int codesSize, Correction *correction);
satok744dab62011-12-15 22:29:05 +0900102 void getOneWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
103 const int *ycoordinates, const int *codes, const bool useFullEditDistance,
104 const int inputLength, Correction *correction, WordsPriorityQueuePool* queuePool);
satok1147c7b2011-12-14 15:04:58 +0900105 void getSuggestionCandidates(
106 const bool useFullEditDistance, const int inputLength, Correction *correction,
satok8330b482012-01-23 16:52:37 +0900107 WordsPriorityQueuePool* queuePool, const bool doAutoCompletion, const int maxErrors,
108 const int currentWordIndex);
satoka85f4922012-01-30 18:18:30 +0900109 void getSplitMultipleWordsSuggestions(ProximityInfo *proximityInfo,
satok744dab62011-12-15 22:29:05 +0900110 const int *xcoordinates, const int *ycoordinates, const int *codes,
satok1f6b52e2012-01-30 13:53:58 +0900111 const bool useFullEditDistance, const int inputLength,
satok99557162012-01-26 22:49:13 +0900112 Correction *correction, WordsPriorityQueuePool* queuePool,
satok8330b482012-01-23 16:52:37 +0900113 const bool hasAutoCorrectionCandidate);
Jean Chalardcf9dbbd2011-12-26 15:16:59 +0900114 void onTerminal(const int freq, const TerminalAttributes& terminalAttributes,
satok8330b482012-01-23 16:52:37 +0900115 Correction *correction, WordsPriorityQueuePool *queuePool, const bool addToMasterQueue,
116 const int currentWordIndex);
satok28bd03b2010-12-03 16:39:16 +0900117 bool needsToSkipCurrentNode(const unsigned short c,
satok68319262010-12-03 19:38:08 +0900118 const int inputIndex, const int skipPos, const int depth);
satok662fe692010-12-08 17:05:39 +0900119 // Process a node by considering proximity, missing and excessive character
satok1147c7b2011-12-14 15:04:58 +0900120 bool processCurrentNode(const int initialPos, Correction *correction, int *newCount,
satok8330b482012-01-23 16:52:37 +0900121 int *newChildPosition, int *nextSiblingPosition, WordsPriorityQueuePool *queuePool,
122 const int currentWordIndex);
Jean Chalardbb15e772011-06-30 20:14:38 +0900123 int getMostFrequentWordLike(const int startInputIndex, const int inputLength,
satok1147c7b2011-12-14 15:04:58 +0900124 ProximityInfo *proximityInfo, unsigned short *word);
Jean Chalard1059f272011-06-28 20:45:05 +0900125 int getMostFrequentWordLikeInner(const uint16_t* const inWord, const int length,
satok1147c7b2011-12-14 15:04:58 +0900126 short unsigned int *outWord);
satok99557162012-01-26 22:49:13 +0900127 bool getSubStringSuggestion(
satok7409d152012-01-26 16:13:25 +0900128 ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates,
satok3c09bb12012-01-26 18:36:19 +0900129 const int *codes, const bool useFullEditDistance, Correction *correction,
130 WordsPriorityQueuePool* queuePool, const int inputLength,
131 const bool hasAutoCorrectionCandidate, const int currentWordIndex,
132 const int inputWordStartPos, const int inputWordLength,
satok99557162012-01-26 22:49:13 +0900133 const int outputWordStartPos, const bool isSpaceProximity, int *freqArray,
134 int *wordLengthArray, unsigned short* outputWord, int *outputWordLength);
satok1f6b52e2012-01-30 13:53:58 +0900135 void getMultiWordsSuggestionRec(ProximityInfo *proximityInfo,
136 const int *xcoordinates, const int *ycoordinates, const int *codes,
137 const bool useFullEditDistance, const int inputLength,
138 Correction *correction, WordsPriorityQueuePool* queuePool,
139 const bool hasAutoCorrectionCandidate, const int startPos, const int startWordIndex,
140 const int outputWordLength, int *freqArray, int* wordLengthArray,
141 unsigned short* outputWord);
Jean Chalard293ece02011-06-16 20:55:16 +0900142
143 const uint8_t* const DICT_ROOT;
satok30088252010-12-01 21:22:15 +0900144 const int MAX_WORD_LENGTH;
Ken Wakasae90b3332011-01-07 15:01:51 +0900145 const int MAX_WORDS;
satok662fe692010-12-08 17:05:39 +0900146 const int MAX_PROXIMITY_CHARS;
satoke808e432010-12-02 14:53:24 +0900147 const bool IS_LATEST_DICT_VERSION;
satok18c28f42010-12-02 18:11:54 +0900148 const int TYPED_LETTER_MULTIPLIER;
149 const int FULL_WORD_MULTIPLIER;
Ken Wakasae90b3332011-01-07 15:01:51 +0900150 const int ROOT_POS;
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900151 const unsigned int BYTES_IN_ONE_CHAR;
Jean Chalard6c300612012-03-06 19:54:03 +0900152 const int MAX_DIGRAPH_SEARCH_DEPTH;
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900153
154 // Flags for special processing
155 // Those *must* match the flags in BinaryDictionary.Flags.ALL_FLAGS in BinaryDictionary.java
156 // or something very bad (like, the apocalypse) will happen.
157 // Please update both at the same time.
158 enum {
satok40a5f6f2011-09-29 18:36:56 +0900159 REQUIRES_GERMAN_UMLAUT_PROCESSING = 0x1,
160 USE_FULL_EDIT_DISTANCE = 0x2
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900161 };
Jean Chalard6c300612012-03-06 19:54:03 +0900162 static const digraph_t GERMAN_UMLAUT_DIGRAPHS[];
satok30088252010-12-01 21:22:15 +0900163
satok1147c7b2011-12-14 15:04:58 +0900164 // Still bundled members
165 unsigned short mWord[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
satok208268d2011-08-10 15:44:08 +0900166 int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
167 int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
168 int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
satok30088252010-12-01 21:22:15 +0900169};
Ken Wakasace9e52a2011-06-18 13:09:55 +0900170} // namespace latinime
satok30088252010-12-01 21:22:15 +0900171
172#endif // LATINIME_UNIGRAM_DICTIONARY_H