satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2010 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #ifndef LATINIME_UNIGRAM_DICTIONARY_H |
| 18 | #define LATINIME_UNIGRAM_DICTIONARY_H |
| 19 | |
Jean Chalard | 293ece0 | 2011-06-16 20:55:16 +0900 | [diff] [blame] | 20 | #include <stdint.h> |
satok | cfca3c6 | 2011-08-10 14:30:10 +0900 | [diff] [blame] | 21 | #include "correction.h" |
satok | 208268d | 2011-08-10 15:44:08 +0900 | [diff] [blame] | 22 | #include "correction_state.h" |
satok | e808e43 | 2010-12-02 14:53:24 +0900 | [diff] [blame] | 23 | #include "defines.h" |
satok | 8fbd552 | 2011-02-22 17:28:55 +0900 | [diff] [blame] | 24 | #include "proximity_info.h" |
satok | 16379df | 2011-12-12 20:53:22 +0900 | [diff] [blame] | 25 | #include "words_priority_queue.h" |
satok | a7e5a5a | 2011-12-15 16:49:12 +0900 | [diff] [blame] | 26 | #include "words_priority_queue_pool.h" |
satok | e808e43 | 2010-12-02 14:53:24 +0900 | [diff] [blame] | 27 | |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 28 | namespace latinime { |
| 29 | |
Jean Chalard | cf9dbbd | 2011-12-26 15:16:59 +0900 | [diff] [blame] | 30 | class TerminalAttributes; |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 31 | class UnigramDictionary { |
Jean Chalard | d304338 | 2012-03-21 18:38:25 +0900 | [diff] [blame^] | 32 | typedef struct { int first; int second; int replacement; } digraph_t; |
Jean Chalard | 6c30061 | 2012-03-06 19:54:03 +0900 | [diff] [blame] | 33 | |
Ken Wakasa | e12e9b5 | 2012-01-06 12:24:38 +0900 | [diff] [blame] | 34 | public: |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 35 | // Mask and flags for children address type selection. |
| 36 | static const int MASK_GROUP_ADDRESS_TYPE = 0xC0; |
| 37 | static const int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00; |
| 38 | static const int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40; |
| 39 | static const int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80; |
| 40 | static const int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0; |
| 41 | |
| 42 | // Flag for single/multiple char group |
| 43 | static const int FLAG_HAS_MULTIPLE_CHARS = 0x20; |
| 44 | |
| 45 | // Flag for terminal groups |
| 46 | static const int FLAG_IS_TERMINAL = 0x10; |
| 47 | |
Jean Chalard | c8c6585 | 2011-12-27 12:58:23 +0900 | [diff] [blame] | 48 | // Flag for shortcut targets presence |
| 49 | static const int FLAG_HAS_SHORTCUT_TARGETS = 0x08; |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 50 | // Flag for bigram presence |
| 51 | static const int FLAG_HAS_BIGRAMS = 0x04; |
Jean Chalard | c8c6585 | 2011-12-27 12:58:23 +0900 | [diff] [blame] | 52 | // Flag for shortcut-only words. Some words are shortcut-only, which means they match when |
| 53 | // the user types them but they don't pop in the suggestion strip, only the words they are |
| 54 | // shortcuts for do. |
| 55 | static const int FLAG_IS_SHORTCUT_ONLY = 0x02; |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 56 | |
| 57 | // Attribute (bigram/shortcut) related flags: |
| 58 | // Flag for presence of more attributes |
| 59 | static const int FLAG_ATTRIBUTE_HAS_NEXT = 0x80; |
| 60 | // Flag for sign of offset. If this flag is set, the offset value must be negated. |
| 61 | static const int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; |
| 62 | |
| 63 | // Mask for attribute frequency, stored on 4 bits inside the flags byte. |
| 64 | static const int MASK_ATTRIBUTE_FREQUENCY = 0x0F; |
| 65 | |
| 66 | // Mask and flags for attribute address type selection. |
| 67 | static const int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30; |
| 68 | static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10; |
| 69 | static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20; |
| 70 | static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30; |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 71 | |
satok | 4d35598 | 2011-12-15 14:53:19 +0900 | [diff] [blame] | 72 | // Error tolerances |
| 73 | static const int DEFAULT_MAX_ERRORS = 2; |
| 74 | static const int MAX_ERRORS_FOR_TWO_WORDS = 1; |
| 75 | |
Jean Chalard | 293ece0 | 2011-06-16 20:55:16 +0900 | [diff] [blame] | 76 | UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultipler, |
| 77 | int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars, |
| 78 | const bool isLatestDictVersion); |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 79 | bool isValidWord(const uint16_t* const inWord, const int length) const; |
Jean Chalard | 581335c | 2011-06-17 12:45:17 +0900 | [diff] [blame] | 80 | int getBigramPosition(int pos, unsigned short *word, int offset, int length) const; |
satok | a7e5a5a | 2011-12-15 16:49:12 +0900 | [diff] [blame] | 81 | int getSuggestions(ProximityInfo *proximityInfo, WordsPriorityQueuePool *queuePool, |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 82 | Correction *correction, const int *xcoordinates, |
Jean Chalard | c2bbc6a | 2011-02-25 17:56:53 +0900 | [diff] [blame] | 83 | const int *ycoordinates, const int *codes, const int codesSize, const int flags, |
| 84 | unsigned short *outWords, int *frequencies); |
satok | 2df3060 | 2011-07-15 13:49:00 +0900 | [diff] [blame] | 85 | virtual ~UnigramDictionary(); |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 86 | |
Ken Wakasa | e12e9b5 | 2012-01-06 12:24:38 +0900 | [diff] [blame] | 87 | private: |
satok | 1d7eaf8 | 2011-07-13 10:32:02 +0900 | [diff] [blame] | 88 | void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 89 | const int *ycoordinates, const int *codes, const int inputLength, |
satok | a7e5a5a | 2011-12-15 16:49:12 +0900 | [diff] [blame] | 90 | const int flags, Correction *correction, WordsPriorityQueuePool *queuePool); |
Jean Chalard | d304338 | 2012-03-21 18:38:25 +0900 | [diff] [blame^] | 91 | int getDigraphReplacement(const int *codes, const int i, const int codesSize, |
Jean Chalard | 6c30061 | 2012-03-06 19:54:03 +0900 | [diff] [blame] | 92 | const digraph_t* const digraphs, const unsigned int digraphsSize) const; |
satok | 1d7eaf8 | 2011-07-13 10:32:02 +0900 | [diff] [blame] | 93 | void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo, |
Jean Chalard | c2bbc6a | 2011-02-25 17:56:53 +0900 | [diff] [blame] | 94 | const int *xcoordinates, const int* ycoordinates, const int *codesBuffer, |
satok | 219a514 | 2012-03-08 11:53:18 +0900 | [diff] [blame] | 95 | int *xCoordinatesBuffer, int *yCoordinatesBuffer, |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 96 | const int codesBufferSize, const int flags, const int* codesSrc, |
| 97 | const int codesRemain, const int currentDepth, int* codesDest, Correction *correction, |
Jean Chalard | 6c30061 | 2012-03-06 19:54:03 +0900 | [diff] [blame] | 98 | WordsPriorityQueuePool* queuePool, const digraph_t* const digraphs, |
| 99 | const unsigned int digraphsSize); |
satok | 1d7eaf8 | 2011-07-13 10:32:02 +0900 | [diff] [blame] | 100 | void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, |
satok | 6ad15fc | 2012-01-16 16:21:21 +0900 | [diff] [blame] | 101 | const int *ycoordinates, const int *codes, const int codesSize, Correction *correction); |
satok | 744dab6 | 2011-12-15 22:29:05 +0900 | [diff] [blame] | 102 | void getOneWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, |
| 103 | const int *ycoordinates, const int *codes, const bool useFullEditDistance, |
| 104 | const int inputLength, Correction *correction, WordsPriorityQueuePool* queuePool); |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 105 | void getSuggestionCandidates( |
| 106 | const bool useFullEditDistance, const int inputLength, Correction *correction, |
satok | 8330b48 | 2012-01-23 16:52:37 +0900 | [diff] [blame] | 107 | WordsPriorityQueuePool* queuePool, const bool doAutoCompletion, const int maxErrors, |
| 108 | const int currentWordIndex); |
satok | a85f492 | 2012-01-30 18:18:30 +0900 | [diff] [blame] | 109 | void getSplitMultipleWordsSuggestions(ProximityInfo *proximityInfo, |
satok | 744dab6 | 2011-12-15 22:29:05 +0900 | [diff] [blame] | 110 | const int *xcoordinates, const int *ycoordinates, const int *codes, |
satok | 1f6b52e | 2012-01-30 13:53:58 +0900 | [diff] [blame] | 111 | const bool useFullEditDistance, const int inputLength, |
satok | 9955716 | 2012-01-26 22:49:13 +0900 | [diff] [blame] | 112 | Correction *correction, WordsPriorityQueuePool* queuePool, |
satok | 8330b48 | 2012-01-23 16:52:37 +0900 | [diff] [blame] | 113 | const bool hasAutoCorrectionCandidate); |
Jean Chalard | cf9dbbd | 2011-12-26 15:16:59 +0900 | [diff] [blame] | 114 | void onTerminal(const int freq, const TerminalAttributes& terminalAttributes, |
satok | 8330b48 | 2012-01-23 16:52:37 +0900 | [diff] [blame] | 115 | Correction *correction, WordsPriorityQueuePool *queuePool, const bool addToMasterQueue, |
| 116 | const int currentWordIndex); |
satok | 28bd03b | 2010-12-03 16:39:16 +0900 | [diff] [blame] | 117 | bool needsToSkipCurrentNode(const unsigned short c, |
satok | 6831926 | 2010-12-03 19:38:08 +0900 | [diff] [blame] | 118 | const int inputIndex, const int skipPos, const int depth); |
satok | 662fe69 | 2010-12-08 17:05:39 +0900 | [diff] [blame] | 119 | // Process a node by considering proximity, missing and excessive character |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 120 | bool processCurrentNode(const int initialPos, Correction *correction, int *newCount, |
satok | 8330b48 | 2012-01-23 16:52:37 +0900 | [diff] [blame] | 121 | int *newChildPosition, int *nextSiblingPosition, WordsPriorityQueuePool *queuePool, |
| 122 | const int currentWordIndex); |
Jean Chalard | bb15e77 | 2011-06-30 20:14:38 +0900 | [diff] [blame] | 123 | int getMostFrequentWordLike(const int startInputIndex, const int inputLength, |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 124 | ProximityInfo *proximityInfo, unsigned short *word); |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 125 | int getMostFrequentWordLikeInner(const uint16_t* const inWord, const int length, |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 126 | short unsigned int *outWord); |
satok | 9955716 | 2012-01-26 22:49:13 +0900 | [diff] [blame] | 127 | bool getSubStringSuggestion( |
satok | 7409d15 | 2012-01-26 16:13:25 +0900 | [diff] [blame] | 128 | ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, |
satok | 3c09bb1 | 2012-01-26 18:36:19 +0900 | [diff] [blame] | 129 | const int *codes, const bool useFullEditDistance, Correction *correction, |
| 130 | WordsPriorityQueuePool* queuePool, const int inputLength, |
| 131 | const bool hasAutoCorrectionCandidate, const int currentWordIndex, |
| 132 | const int inputWordStartPos, const int inputWordLength, |
satok | 9955716 | 2012-01-26 22:49:13 +0900 | [diff] [blame] | 133 | const int outputWordStartPos, const bool isSpaceProximity, int *freqArray, |
| 134 | int *wordLengthArray, unsigned short* outputWord, int *outputWordLength); |
satok | 1f6b52e | 2012-01-30 13:53:58 +0900 | [diff] [blame] | 135 | void getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, |
| 136 | const int *xcoordinates, const int *ycoordinates, const int *codes, |
| 137 | const bool useFullEditDistance, const int inputLength, |
| 138 | Correction *correction, WordsPriorityQueuePool* queuePool, |
| 139 | const bool hasAutoCorrectionCandidate, const int startPos, const int startWordIndex, |
| 140 | const int outputWordLength, int *freqArray, int* wordLengthArray, |
| 141 | unsigned short* outputWord); |
Jean Chalard | 293ece0 | 2011-06-16 20:55:16 +0900 | [diff] [blame] | 142 | |
| 143 | const uint8_t* const DICT_ROOT; |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 144 | const int MAX_WORD_LENGTH; |
Ken Wakasa | e90b333 | 2011-01-07 15:01:51 +0900 | [diff] [blame] | 145 | const int MAX_WORDS; |
satok | 662fe69 | 2010-12-08 17:05:39 +0900 | [diff] [blame] | 146 | const int MAX_PROXIMITY_CHARS; |
satok | e808e43 | 2010-12-02 14:53:24 +0900 | [diff] [blame] | 147 | const bool IS_LATEST_DICT_VERSION; |
satok | 18c28f4 | 2010-12-02 18:11:54 +0900 | [diff] [blame] | 148 | const int TYPED_LETTER_MULTIPLIER; |
| 149 | const int FULL_WORD_MULTIPLIER; |
Ken Wakasa | e90b333 | 2011-01-07 15:01:51 +0900 | [diff] [blame] | 150 | const int ROOT_POS; |
Jean Chalard | c2bbc6a | 2011-02-25 17:56:53 +0900 | [diff] [blame] | 151 | const unsigned int BYTES_IN_ONE_CHAR; |
Jean Chalard | 6c30061 | 2012-03-06 19:54:03 +0900 | [diff] [blame] | 152 | const int MAX_DIGRAPH_SEARCH_DEPTH; |
Jean Chalard | c2bbc6a | 2011-02-25 17:56:53 +0900 | [diff] [blame] | 153 | |
| 154 | // Flags for special processing |
| 155 | // Those *must* match the flags in BinaryDictionary.Flags.ALL_FLAGS in BinaryDictionary.java |
| 156 | // or something very bad (like, the apocalypse) will happen. |
| 157 | // Please update both at the same time. |
| 158 | enum { |
satok | 40a5f6f | 2011-09-29 18:36:56 +0900 | [diff] [blame] | 159 | REQUIRES_GERMAN_UMLAUT_PROCESSING = 0x1, |
| 160 | USE_FULL_EDIT_DISTANCE = 0x2 |
Jean Chalard | c2bbc6a | 2011-02-25 17:56:53 +0900 | [diff] [blame] | 161 | }; |
Jean Chalard | 6c30061 | 2012-03-06 19:54:03 +0900 | [diff] [blame] | 162 | static const digraph_t GERMAN_UMLAUT_DIGRAPHS[]; |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 163 | |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 164 | // Still bundled members |
| 165 | unsigned short mWord[MAX_WORD_LENGTH_INTERNAL];// TODO: remove |
satok | 208268d | 2011-08-10 15:44:08 +0900 | [diff] [blame] | 166 | int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];// TODO: remove |
| 167 | int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];// TODO: remove |
| 168 | int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];// TODO: remove |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 169 | }; |
Ken Wakasa | ce9e52a | 2011-06-18 13:09:55 +0900 | [diff] [blame] | 170 | } // namespace latinime |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 171 | |
| 172 | #endif // LATINIME_UNIGRAM_DICTIONARY_H |