blob: 2d5d076b1aab83489a175c6a4f9677b57c8a7426 [file] [log] [blame]
satok30088252010-12-01 21:22:15 +09001/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef LATINIME_UNIGRAM_DICTIONARY_H
18#define LATINIME_UNIGRAM_DICTIONARY_H
19
Jean Chalard293ece02011-06-16 20:55:16 +090020#include <stdint.h>
satokcfca3c62011-08-10 14:30:10 +090021#include "correction.h"
satok208268d2011-08-10 15:44:08 +090022#include "correction_state.h"
satoke808e432010-12-02 14:53:24 +090023#include "defines.h"
satok8fbd5522011-02-22 17:28:55 +090024#include "proximity_info.h"
satok16379df2011-12-12 20:53:22 +090025#include "words_priority_queue.h"
satoka7e5a5a2011-12-15 16:49:12 +090026#include "words_priority_queue_pool.h"
satoke808e432010-12-02 14:53:24 +090027
satok30088252010-12-01 21:22:15 +090028namespace latinime {
29
Jean Chalardcf9dbbd2011-12-26 15:16:59 +090030class TerminalAttributes;
satok30088252010-12-01 21:22:15 +090031class UnigramDictionary {
Ken Wakasae12e9b52012-01-06 12:24:38 +090032 public:
Jean Chalard1059f272011-06-28 20:45:05 +090033 // Mask and flags for children address type selection.
34 static const int MASK_GROUP_ADDRESS_TYPE = 0xC0;
35 static const int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00;
36 static const int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40;
37 static const int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80;
38 static const int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0;
39
40 // Flag for single/multiple char group
41 static const int FLAG_HAS_MULTIPLE_CHARS = 0x20;
42
43 // Flag for terminal groups
44 static const int FLAG_IS_TERMINAL = 0x10;
45
Jean Chalardc8c65852011-12-27 12:58:23 +090046 // Flag for shortcut targets presence
47 static const int FLAG_HAS_SHORTCUT_TARGETS = 0x08;
Jean Chalard1059f272011-06-28 20:45:05 +090048 // Flag for bigram presence
49 static const int FLAG_HAS_BIGRAMS = 0x04;
Jean Chalardc8c65852011-12-27 12:58:23 +090050 // Flag for shortcut-only words. Some words are shortcut-only, which means they match when
51 // the user types them but they don't pop in the suggestion strip, only the words they are
52 // shortcuts for do.
53 static const int FLAG_IS_SHORTCUT_ONLY = 0x02;
Jean Chalard1059f272011-06-28 20:45:05 +090054
55 // Attribute (bigram/shortcut) related flags:
56 // Flag for presence of more attributes
57 static const int FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
58 // Flag for sign of offset. If this flag is set, the offset value must be negated.
59 static const int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
60
61 // Mask for attribute frequency, stored on 4 bits inside the flags byte.
62 static const int MASK_ATTRIBUTE_FREQUENCY = 0x0F;
63
64 // Mask and flags for attribute address type selection.
65 static const int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30;
66 static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
67 static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
68 static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
Jean Chalard1059f272011-06-28 20:45:05 +090069
satok4d355982011-12-15 14:53:19 +090070 // Error tolerances
71 static const int DEFAULT_MAX_ERRORS = 2;
72 static const int MAX_ERRORS_FOR_TWO_WORDS = 1;
73
Jean Chalard293ece02011-06-16 20:55:16 +090074 UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultipler,
75 int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars,
76 const bool isLatestDictVersion);
Jean Chalard1059f272011-06-28 20:45:05 +090077 bool isValidWord(const uint16_t* const inWord, const int length) const;
Jean Chalard581335c2011-06-17 12:45:17 +090078 int getBigramPosition(int pos, unsigned short *word, int offset, int length) const;
satoka7e5a5a2011-12-15 16:49:12 +090079 int getSuggestions(ProximityInfo *proximityInfo, WordsPriorityQueuePool *queuePool,
satok1147c7b2011-12-14 15:04:58 +090080 Correction *correction, const int *xcoordinates,
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090081 const int *ycoordinates, const int *codes, const int codesSize, const int flags,
82 unsigned short *outWords, int *frequencies);
satok2df30602011-07-15 13:49:00 +090083 virtual ~UnigramDictionary();
satok30088252010-12-01 21:22:15 +090084
Ken Wakasae12e9b52012-01-06 12:24:38 +090085 private:
satok1d7eaf82011-07-13 10:32:02 +090086 void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
satok1147c7b2011-12-14 15:04:58 +090087 const int *ycoordinates, const int *codes, const int inputLength,
satoka7e5a5a2011-12-15 16:49:12 +090088 const int flags, Correction *correction, WordsPriorityQueuePool *queuePool);
satok1147c7b2011-12-14 15:04:58 +090089 bool isDigraph(const int *codes, const int i, const int codesSize) const;
satok1d7eaf82011-07-13 10:32:02 +090090 void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090091 const int *xcoordinates, const int* ycoordinates, const int *codesBuffer,
satok1147c7b2011-12-14 15:04:58 +090092 const int codesBufferSize, const int flags, const int* codesSrc,
93 const int codesRemain, const int currentDepth, int* codesDest, Correction *correction,
satoka7e5a5a2011-12-15 16:49:12 +090094 WordsPriorityQueuePool* queuePool);
satok1d7eaf82011-07-13 10:32:02 +090095 void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
satok6ad15fc2012-01-16 16:21:21 +090096 const int *ycoordinates, const int *codes, const int codesSize, Correction *correction);
satok744dab62011-12-15 22:29:05 +090097 void getOneWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
98 const int *ycoordinates, const int *codes, const bool useFullEditDistance,
99 const int inputLength, Correction *correction, WordsPriorityQueuePool* queuePool);
satok1147c7b2011-12-14 15:04:58 +0900100 void getSuggestionCandidates(
101 const bool useFullEditDistance, const int inputLength, Correction *correction,
satok8330b482012-01-23 16:52:37 +0900102 WordsPriorityQueuePool* queuePool, const bool doAutoCompletion, const int maxErrors,
103 const int currentWordIndex);
satok744dab62011-12-15 22:29:05 +0900104 void getSplitTwoWordsSuggestions(ProximityInfo *proximityInfo,
105 const int *xcoordinates, const int *ycoordinates, const int *codes,
satok1f6b52e2012-01-30 13:53:58 +0900106 const bool useFullEditDistance, const int inputLength,
satok99557162012-01-26 22:49:13 +0900107 Correction *correction, WordsPriorityQueuePool* queuePool,
satok8330b482012-01-23 16:52:37 +0900108 const bool hasAutoCorrectionCandidate);
Jean Chalardcf9dbbd2011-12-26 15:16:59 +0900109 void onTerminal(const int freq, const TerminalAttributes& terminalAttributes,
satok8330b482012-01-23 16:52:37 +0900110 Correction *correction, WordsPriorityQueuePool *queuePool, const bool addToMasterQueue,
111 const int currentWordIndex);
satok28bd03b2010-12-03 16:39:16 +0900112 bool needsToSkipCurrentNode(const unsigned short c,
satok68319262010-12-03 19:38:08 +0900113 const int inputIndex, const int skipPos, const int depth);
satok662fe692010-12-08 17:05:39 +0900114 // Process a node by considering proximity, missing and excessive character
satok1147c7b2011-12-14 15:04:58 +0900115 bool processCurrentNode(const int initialPos, Correction *correction, int *newCount,
satok8330b482012-01-23 16:52:37 +0900116 int *newChildPosition, int *nextSiblingPosition, WordsPriorityQueuePool *queuePool,
117 const int currentWordIndex);
Jean Chalardbb15e772011-06-30 20:14:38 +0900118 int getMostFrequentWordLike(const int startInputIndex, const int inputLength,
satok1147c7b2011-12-14 15:04:58 +0900119 ProximityInfo *proximityInfo, unsigned short *word);
Jean Chalard1059f272011-06-28 20:45:05 +0900120 int getMostFrequentWordLikeInner(const uint16_t* const inWord, const int length,
satok1147c7b2011-12-14 15:04:58 +0900121 short unsigned int *outWord);
satok99557162012-01-26 22:49:13 +0900122 bool getSubStringSuggestion(
satok7409d152012-01-26 16:13:25 +0900123 ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates,
satok3c09bb12012-01-26 18:36:19 +0900124 const int *codes, const bool useFullEditDistance, Correction *correction,
125 WordsPriorityQueuePool* queuePool, const int inputLength,
126 const bool hasAutoCorrectionCandidate, const int currentWordIndex,
127 const int inputWordStartPos, const int inputWordLength,
satok99557162012-01-26 22:49:13 +0900128 const int outputWordStartPos, const bool isSpaceProximity, int *freqArray,
129 int *wordLengthArray, unsigned short* outputWord, int *outputWordLength);
satok1f6b52e2012-01-30 13:53:58 +0900130 void getMultiWordsSuggestionRec(ProximityInfo *proximityInfo,
131 const int *xcoordinates, const int *ycoordinates, const int *codes,
132 const bool useFullEditDistance, const int inputLength,
133 Correction *correction, WordsPriorityQueuePool* queuePool,
134 const bool hasAutoCorrectionCandidate, const int startPos, const int startWordIndex,
135 const int outputWordLength, int *freqArray, int* wordLengthArray,
136 unsigned short* outputWord);
Jean Chalard293ece02011-06-16 20:55:16 +0900137
138 const uint8_t* const DICT_ROOT;
satok30088252010-12-01 21:22:15 +0900139 const int MAX_WORD_LENGTH;
Ken Wakasae90b3332011-01-07 15:01:51 +0900140 const int MAX_WORDS;
satok662fe692010-12-08 17:05:39 +0900141 const int MAX_PROXIMITY_CHARS;
satoke808e432010-12-02 14:53:24 +0900142 const bool IS_LATEST_DICT_VERSION;
satok18c28f42010-12-02 18:11:54 +0900143 const int TYPED_LETTER_MULTIPLIER;
144 const int FULL_WORD_MULTIPLIER;
Ken Wakasae90b3332011-01-07 15:01:51 +0900145 const int ROOT_POS;
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900146 const unsigned int BYTES_IN_ONE_CHAR;
satok3c4bb772011-03-04 22:50:19 -0800147 const int MAX_UMLAUT_SEARCH_DEPTH;
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900148
149 // Flags for special processing
150 // Those *must* match the flags in BinaryDictionary.Flags.ALL_FLAGS in BinaryDictionary.java
151 // or something very bad (like, the apocalypse) will happen.
152 // Please update both at the same time.
153 enum {
satok40a5f6f2011-09-29 18:36:56 +0900154 REQUIRES_GERMAN_UMLAUT_PROCESSING = 0x1,
155 USE_FULL_EDIT_DISTANCE = 0x2
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900156 };
157 static const struct digraph_t { int first; int second; } GERMAN_UMLAUT_DIGRAPHS[];
satok30088252010-12-01 21:22:15 +0900158
satok1147c7b2011-12-14 15:04:58 +0900159 // Still bundled members
160 unsigned short mWord[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
satok208268d2011-08-10 15:44:08 +0900161 int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
162 int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
163 int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
satok30088252010-12-01 21:22:15 +0900164};
Ken Wakasace9e52a2011-06-18 13:09:55 +0900165} // namespace latinime
satok30088252010-12-01 21:22:15 +0900166
167#endif // LATINIME_UNIGRAM_DICTIONARY_H