satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2010 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #ifndef LATINIME_UNIGRAM_DICTIONARY_H |
| 18 | #define LATINIME_UNIGRAM_DICTIONARY_H |
| 19 | |
Jean Chalard | 293ece0 | 2011-06-16 20:55:16 +0900 | [diff] [blame] | 20 | #include <stdint.h> |
satok | cfca3c6 | 2011-08-10 14:30:10 +0900 | [diff] [blame] | 21 | #include "correction.h" |
satok | 208268d | 2011-08-10 15:44:08 +0900 | [diff] [blame] | 22 | #include "correction_state.h" |
satok | e808e43 | 2010-12-02 14:53:24 +0900 | [diff] [blame] | 23 | #include "defines.h" |
satok | 8fbd552 | 2011-02-22 17:28:55 +0900 | [diff] [blame] | 24 | #include "proximity_info.h" |
satok | 16379df | 2011-12-12 20:53:22 +0900 | [diff] [blame] | 25 | #include "words_priority_queue.h" |
satok | a7e5a5a | 2011-12-15 16:49:12 +0900 | [diff] [blame] | 26 | #include "words_priority_queue_pool.h" |
satok | e808e43 | 2010-12-02 14:53:24 +0900 | [diff] [blame] | 27 | |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 28 | namespace latinime { |
| 29 | |
Jean Chalard | cf9dbbd | 2011-12-26 15:16:59 +0900 | [diff] [blame] | 30 | class TerminalAttributes; |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 31 | class UnigramDictionary { |
Ken Wakasa | e12e9b5 | 2012-01-06 12:24:38 +0900 | [diff] [blame] | 32 | public: |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 33 | // Mask and flags for children address type selection. |
| 34 | static const int MASK_GROUP_ADDRESS_TYPE = 0xC0; |
| 35 | static const int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00; |
| 36 | static const int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40; |
| 37 | static const int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80; |
| 38 | static const int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0; |
| 39 | |
| 40 | // Flag for single/multiple char group |
| 41 | static const int FLAG_HAS_MULTIPLE_CHARS = 0x20; |
| 42 | |
| 43 | // Flag for terminal groups |
| 44 | static const int FLAG_IS_TERMINAL = 0x10; |
| 45 | |
Jean Chalard | c8c6585 | 2011-12-27 12:58:23 +0900 | [diff] [blame] | 46 | // Flag for shortcut targets presence |
| 47 | static const int FLAG_HAS_SHORTCUT_TARGETS = 0x08; |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 48 | // Flag for bigram presence |
| 49 | static const int FLAG_HAS_BIGRAMS = 0x04; |
Jean Chalard | c8c6585 | 2011-12-27 12:58:23 +0900 | [diff] [blame] | 50 | // Flag for shortcut-only words. Some words are shortcut-only, which means they match when |
| 51 | // the user types them but they don't pop in the suggestion strip, only the words they are |
| 52 | // shortcuts for do. |
| 53 | static const int FLAG_IS_SHORTCUT_ONLY = 0x02; |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 54 | |
| 55 | // Attribute (bigram/shortcut) related flags: |
| 56 | // Flag for presence of more attributes |
| 57 | static const int FLAG_ATTRIBUTE_HAS_NEXT = 0x80; |
| 58 | // Flag for sign of offset. If this flag is set, the offset value must be negated. |
| 59 | static const int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; |
| 60 | |
| 61 | // Mask for attribute frequency, stored on 4 bits inside the flags byte. |
| 62 | static const int MASK_ATTRIBUTE_FREQUENCY = 0x0F; |
| 63 | |
| 64 | // Mask and flags for attribute address type selection. |
| 65 | static const int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30; |
| 66 | static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10; |
| 67 | static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20; |
| 68 | static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30; |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 69 | |
satok | 4d35598 | 2011-12-15 14:53:19 +0900 | [diff] [blame] | 70 | // Error tolerances |
| 71 | static const int DEFAULT_MAX_ERRORS = 2; |
| 72 | static const int MAX_ERRORS_FOR_TWO_WORDS = 1; |
| 73 | |
Jean Chalard | 293ece0 | 2011-06-16 20:55:16 +0900 | [diff] [blame] | 74 | UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultipler, |
| 75 | int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars, |
| 76 | const bool isLatestDictVersion); |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 77 | bool isValidWord(const uint16_t* const inWord, const int length) const; |
Jean Chalard | 581335c | 2011-06-17 12:45:17 +0900 | [diff] [blame] | 78 | int getBigramPosition(int pos, unsigned short *word, int offset, int length) const; |
satok | a7e5a5a | 2011-12-15 16:49:12 +0900 | [diff] [blame] | 79 | int getSuggestions(ProximityInfo *proximityInfo, WordsPriorityQueuePool *queuePool, |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 80 | Correction *correction, const int *xcoordinates, |
Jean Chalard | c2bbc6a | 2011-02-25 17:56:53 +0900 | [diff] [blame] | 81 | const int *ycoordinates, const int *codes, const int codesSize, const int flags, |
| 82 | unsigned short *outWords, int *frequencies); |
satok | 2df3060 | 2011-07-15 13:49:00 +0900 | [diff] [blame] | 83 | virtual ~UnigramDictionary(); |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 84 | |
Ken Wakasa | e12e9b5 | 2012-01-06 12:24:38 +0900 | [diff] [blame] | 85 | private: |
satok | 1d7eaf8 | 2011-07-13 10:32:02 +0900 | [diff] [blame] | 86 | void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 87 | const int *ycoordinates, const int *codes, const int inputLength, |
satok | a7e5a5a | 2011-12-15 16:49:12 +0900 | [diff] [blame] | 88 | const int flags, Correction *correction, WordsPriorityQueuePool *queuePool); |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 89 | bool isDigraph(const int *codes, const int i, const int codesSize) const; |
satok | 1d7eaf8 | 2011-07-13 10:32:02 +0900 | [diff] [blame] | 90 | void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo, |
Jean Chalard | c2bbc6a | 2011-02-25 17:56:53 +0900 | [diff] [blame] | 91 | const int *xcoordinates, const int* ycoordinates, const int *codesBuffer, |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 92 | const int codesBufferSize, const int flags, const int* codesSrc, |
| 93 | const int codesRemain, const int currentDepth, int* codesDest, Correction *correction, |
satok | a7e5a5a | 2011-12-15 16:49:12 +0900 | [diff] [blame] | 94 | WordsPriorityQueuePool* queuePool); |
satok | 1d7eaf8 | 2011-07-13 10:32:02 +0900 | [diff] [blame] | 95 | void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, |
satok | 6ad15fc | 2012-01-16 16:21:21 +0900 | [diff] [blame] | 96 | const int *ycoordinates, const int *codes, const int codesSize, Correction *correction); |
satok | 744dab6 | 2011-12-15 22:29:05 +0900 | [diff] [blame] | 97 | void getOneWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, |
| 98 | const int *ycoordinates, const int *codes, const bool useFullEditDistance, |
| 99 | const int inputLength, Correction *correction, WordsPriorityQueuePool* queuePool); |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 100 | void getSuggestionCandidates( |
| 101 | const bool useFullEditDistance, const int inputLength, Correction *correction, |
satok | 8330b48 | 2012-01-23 16:52:37 +0900 | [diff] [blame] | 102 | WordsPriorityQueuePool* queuePool, const bool doAutoCompletion, const int maxErrors, |
| 103 | const int currentWordIndex); |
satok | 744dab6 | 2011-12-15 22:29:05 +0900 | [diff] [blame] | 104 | void getSplitTwoWordsSuggestions(ProximityInfo *proximityInfo, |
| 105 | const int *xcoordinates, const int *ycoordinates, const int *codes, |
satok | 1f6b52e | 2012-01-30 13:53:58 +0900 | [diff] [blame^] | 106 | const bool useFullEditDistance, const int inputLength, |
satok | 9955716 | 2012-01-26 22:49:13 +0900 | [diff] [blame] | 107 | Correction *correction, WordsPriorityQueuePool* queuePool, |
satok | 8330b48 | 2012-01-23 16:52:37 +0900 | [diff] [blame] | 108 | const bool hasAutoCorrectionCandidate); |
Jean Chalard | cf9dbbd | 2011-12-26 15:16:59 +0900 | [diff] [blame] | 109 | void onTerminal(const int freq, const TerminalAttributes& terminalAttributes, |
satok | 8330b48 | 2012-01-23 16:52:37 +0900 | [diff] [blame] | 110 | Correction *correction, WordsPriorityQueuePool *queuePool, const bool addToMasterQueue, |
| 111 | const int currentWordIndex); |
satok | 28bd03b | 2010-12-03 16:39:16 +0900 | [diff] [blame] | 112 | bool needsToSkipCurrentNode(const unsigned short c, |
satok | 6831926 | 2010-12-03 19:38:08 +0900 | [diff] [blame] | 113 | const int inputIndex, const int skipPos, const int depth); |
satok | 662fe69 | 2010-12-08 17:05:39 +0900 | [diff] [blame] | 114 | // Process a node by considering proximity, missing and excessive character |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 115 | bool processCurrentNode(const int initialPos, Correction *correction, int *newCount, |
satok | 8330b48 | 2012-01-23 16:52:37 +0900 | [diff] [blame] | 116 | int *newChildPosition, int *nextSiblingPosition, WordsPriorityQueuePool *queuePool, |
| 117 | const int currentWordIndex); |
Jean Chalard | bb15e77 | 2011-06-30 20:14:38 +0900 | [diff] [blame] | 118 | int getMostFrequentWordLike(const int startInputIndex, const int inputLength, |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 119 | ProximityInfo *proximityInfo, unsigned short *word); |
Jean Chalard | 1059f27 | 2011-06-28 20:45:05 +0900 | [diff] [blame] | 120 | int getMostFrequentWordLikeInner(const uint16_t* const inWord, const int length, |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 121 | short unsigned int *outWord); |
satok | 9955716 | 2012-01-26 22:49:13 +0900 | [diff] [blame] | 122 | bool getSubStringSuggestion( |
satok | 7409d15 | 2012-01-26 16:13:25 +0900 | [diff] [blame] | 123 | ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, |
satok | 3c09bb1 | 2012-01-26 18:36:19 +0900 | [diff] [blame] | 124 | const int *codes, const bool useFullEditDistance, Correction *correction, |
| 125 | WordsPriorityQueuePool* queuePool, const int inputLength, |
| 126 | const bool hasAutoCorrectionCandidate, const int currentWordIndex, |
| 127 | const int inputWordStartPos, const int inputWordLength, |
satok | 9955716 | 2012-01-26 22:49:13 +0900 | [diff] [blame] | 128 | const int outputWordStartPos, const bool isSpaceProximity, int *freqArray, |
| 129 | int *wordLengthArray, unsigned short* outputWord, int *outputWordLength); |
satok | 1f6b52e | 2012-01-30 13:53:58 +0900 | [diff] [blame^] | 130 | void getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, |
| 131 | const int *xcoordinates, const int *ycoordinates, const int *codes, |
| 132 | const bool useFullEditDistance, const int inputLength, |
| 133 | Correction *correction, WordsPriorityQueuePool* queuePool, |
| 134 | const bool hasAutoCorrectionCandidate, const int startPos, const int startWordIndex, |
| 135 | const int outputWordLength, int *freqArray, int* wordLengthArray, |
| 136 | unsigned short* outputWord); |
Jean Chalard | 293ece0 | 2011-06-16 20:55:16 +0900 | [diff] [blame] | 137 | |
| 138 | const uint8_t* const DICT_ROOT; |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 139 | const int MAX_WORD_LENGTH; |
Ken Wakasa | e90b333 | 2011-01-07 15:01:51 +0900 | [diff] [blame] | 140 | const int MAX_WORDS; |
satok | 662fe69 | 2010-12-08 17:05:39 +0900 | [diff] [blame] | 141 | const int MAX_PROXIMITY_CHARS; |
satok | e808e43 | 2010-12-02 14:53:24 +0900 | [diff] [blame] | 142 | const bool IS_LATEST_DICT_VERSION; |
satok | 18c28f4 | 2010-12-02 18:11:54 +0900 | [diff] [blame] | 143 | const int TYPED_LETTER_MULTIPLIER; |
| 144 | const int FULL_WORD_MULTIPLIER; |
Ken Wakasa | e90b333 | 2011-01-07 15:01:51 +0900 | [diff] [blame] | 145 | const int ROOT_POS; |
Jean Chalard | c2bbc6a | 2011-02-25 17:56:53 +0900 | [diff] [blame] | 146 | const unsigned int BYTES_IN_ONE_CHAR; |
satok | 3c4bb77 | 2011-03-04 22:50:19 -0800 | [diff] [blame] | 147 | const int MAX_UMLAUT_SEARCH_DEPTH; |
Jean Chalard | c2bbc6a | 2011-02-25 17:56:53 +0900 | [diff] [blame] | 148 | |
| 149 | // Flags for special processing |
| 150 | // Those *must* match the flags in BinaryDictionary.Flags.ALL_FLAGS in BinaryDictionary.java |
| 151 | // or something very bad (like, the apocalypse) will happen. |
| 152 | // Please update both at the same time. |
| 153 | enum { |
satok | 40a5f6f | 2011-09-29 18:36:56 +0900 | [diff] [blame] | 154 | REQUIRES_GERMAN_UMLAUT_PROCESSING = 0x1, |
| 155 | USE_FULL_EDIT_DISTANCE = 0x2 |
Jean Chalard | c2bbc6a | 2011-02-25 17:56:53 +0900 | [diff] [blame] | 156 | }; |
| 157 | static const struct digraph_t { int first; int second; } GERMAN_UMLAUT_DIGRAPHS[]; |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 158 | |
satok | 1147c7b | 2011-12-14 15:04:58 +0900 | [diff] [blame] | 159 | // Still bundled members |
| 160 | unsigned short mWord[MAX_WORD_LENGTH_INTERNAL];// TODO: remove |
satok | 208268d | 2011-08-10 15:44:08 +0900 | [diff] [blame] | 161 | int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];// TODO: remove |
| 162 | int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];// TODO: remove |
| 163 | int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];// TODO: remove |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 164 | }; |
Ken Wakasa | ce9e52a | 2011-06-18 13:09:55 +0900 | [diff] [blame] | 165 | } // namespace latinime |
satok | 3008825 | 2010-12-01 21:22:15 +0900 | [diff] [blame] | 166 | |
| 167 | #endif // LATINIME_UNIGRAM_DICTIONARY_H |