blob: 4f4fef267f341b16baa9b7a36b161676ead85688 [file] [log] [blame]
satok30088252010-12-01 21:22:15 +09001/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef LATINIME_UNIGRAM_DICTIONARY_H
18#define LATINIME_UNIGRAM_DICTIONARY_H
19
Jean Chalard293ece02011-06-16 20:55:16 +090020#include <stdint.h>
satokcfca3c62011-08-10 14:30:10 +090021#include "correction.h"
satok208268d2011-08-10 15:44:08 +090022#include "correction_state.h"
satoke808e432010-12-02 14:53:24 +090023#include "defines.h"
satok8fbd5522011-02-22 17:28:55 +090024#include "proximity_info.h"
satoke808e432010-12-02 14:53:24 +090025
satok30088252010-12-01 21:22:15 +090026namespace latinime {
27
satok30088252010-12-01 21:22:15 +090028class UnigramDictionary {
Jean Chalard8dc754a2011-01-27 14:20:22 +090029
satok30088252010-12-01 21:22:15 +090030public:
Jean Chalard1059f272011-06-28 20:45:05 +090031
32 // Mask and flags for children address type selection.
33 static const int MASK_GROUP_ADDRESS_TYPE = 0xC0;
34 static const int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00;
35 static const int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40;
36 static const int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80;
37 static const int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0;
38
39 // Flag for single/multiple char group
40 static const int FLAG_HAS_MULTIPLE_CHARS = 0x20;
41
42 // Flag for terminal groups
43 static const int FLAG_IS_TERMINAL = 0x10;
44
45 // Flag for bigram presence
46 static const int FLAG_HAS_BIGRAMS = 0x04;
47
48 // Attribute (bigram/shortcut) related flags:
49 // Flag for presence of more attributes
50 static const int FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
51 // Flag for sign of offset. If this flag is set, the offset value must be negated.
52 static const int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
53
54 // Mask for attribute frequency, stored on 4 bits inside the flags byte.
55 static const int MASK_ATTRIBUTE_FREQUENCY = 0x0F;
56
57 // Mask and flags for attribute address type selection.
58 static const int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30;
59 static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
60 static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
61 static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
Jean Chalard1059f272011-06-28 20:45:05 +090062
Jean Chalard293ece02011-06-16 20:55:16 +090063 UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultipler,
64 int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars,
65 const bool isLatestDictVersion);
Jean Chalard1059f272011-06-28 20:45:05 +090066 bool isValidWord(const uint16_t* const inWord, const int length) const;
Jean Chalard581335c2011-06-17 12:45:17 +090067 int getBigramPosition(int pos, unsigned short *word, int offset, int length) const;
satok1d7eaf82011-07-13 10:32:02 +090068 int getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090069 const int *ycoordinates, const int *codes, const int codesSize, const int flags,
70 unsigned short *outWords, int *frequencies);
satok2df30602011-07-15 13:49:00 +090071 virtual ~UnigramDictionary();
satok30088252010-12-01 21:22:15 +090072
73private:
satok612c6e42011-08-01 19:35:27 +090074
satok1d7eaf82011-07-13 10:32:02 +090075 void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090076 const int *ycoordinates, const int *codes, const int codesSize,
satok40a5f6f2011-09-29 18:36:56 +090077 unsigned short *outWords, int *frequencies, const int flags);
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090078 bool isDigraph(const int* codes, const int i, const int codesSize) const;
satok1d7eaf82011-07-13 10:32:02 +090079 void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
Jean Chalardc2bbc6a2011-02-25 17:56:53 +090080 const int *xcoordinates, const int* ycoordinates, const int *codesBuffer,
81 const int codesBufferSize, const int flags, const int* codesSrc, const int codesRemain,
satok3c4bb772011-03-04 22:50:19 -080082 const int currentDepth, int* codesDest, unsigned short* outWords, int* frequencies);
satok1d7eaf82011-07-13 10:32:02 +090083 void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
84 const int *ycoordinates, const int *codes, const int codesSize,
85 unsigned short *outWords, int *frequencies);
satok40a5f6f2011-09-29 18:36:56 +090086 void getSuggestionCandidates(const bool useFullEditDistance);
satok30088252010-12-01 21:22:15 +090087 bool addWord(unsigned short *word, int length, int frequency);
satokcfca3c62011-08-10 14:30:10 +090088 void getSplitTwoWordsSuggestion(const int inputLength, Correction *correction);
satok40a5f6f2011-09-29 18:36:56 +090089 void getMissingSpaceWords(const int inputLength, const int missingSpacePos,
90 Correction *correction, const bool useFullEditDistance);
91 void getMistypedSpaceWords(const int inputLength, const int spaceProximityPos,
92 Correction *correction, const bool useFullEditDistance);
satokcfca3c62011-08-10 14:30:10 +090093 void onTerminal(const int freq, Correction *correction);
satok28bd03b2010-12-03 16:39:16 +090094 bool needsToSkipCurrentNode(const unsigned short c,
satok68319262010-12-03 19:38:08 +090095 const int inputIndex, const int skipPos, const int depth);
satok662fe692010-12-08 17:05:39 +090096 // Process a node by considering proximity, missing and excessive character
satok8876b752011-08-04 18:31:57 +090097 bool processCurrentNode(const int initialPos,
satokcfca3c62011-08-10 14:30:10 +090098 Correction *correction, int *newCount,
satok8876b752011-08-04 18:31:57 +090099 int *newChildPosition, int *nextSiblingPosition);
Jean Chalardbb15e772011-06-30 20:14:38 +0900100 int getMostFrequentWordLike(const int startInputIndex, const int inputLength,
101 unsigned short *word);
Jean Chalard1059f272011-06-28 20:45:05 +0900102 int getMostFrequentWordLikeInner(const uint16_t* const inWord, const int length,
103 short unsigned int* outWord);
Jean Chalard293ece02011-06-16 20:55:16 +0900104
105 const uint8_t* const DICT_ROOT;
satok30088252010-12-01 21:22:15 +0900106 const int MAX_WORD_LENGTH;
Ken Wakasae90b3332011-01-07 15:01:51 +0900107 const int MAX_WORDS;
satok662fe692010-12-08 17:05:39 +0900108 const int MAX_PROXIMITY_CHARS;
satoke808e432010-12-02 14:53:24 +0900109 const bool IS_LATEST_DICT_VERSION;
satok18c28f42010-12-02 18:11:54 +0900110 const int TYPED_LETTER_MULTIPLIER;
111 const int FULL_WORD_MULTIPLIER;
Ken Wakasae90b3332011-01-07 15:01:51 +0900112 const int ROOT_POS;
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900113 const unsigned int BYTES_IN_ONE_CHAR;
satok3c4bb772011-03-04 22:50:19 -0800114 const int MAX_UMLAUT_SEARCH_DEPTH;
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900115
116 // Flags for special processing
117 // Those *must* match the flags in BinaryDictionary.Flags.ALL_FLAGS in BinaryDictionary.java
118 // or something very bad (like, the apocalypse) will happen.
119 // Please update both at the same time.
120 enum {
satok40a5f6f2011-09-29 18:36:56 +0900121 REQUIRES_GERMAN_UMLAUT_PROCESSING = 0x1,
122 USE_FULL_EDIT_DISTANCE = 0x2
Jean Chalardc2bbc6a2011-02-25 17:56:53 +0900123 };
124 static const struct digraph_t { int first; int second; } GERMAN_UMLAUT_DIGRAPHS[];
satok30088252010-12-01 21:22:15 +0900125
satok30088252010-12-01 21:22:15 +0900126 int *mFrequencies;
satok30088252010-12-01 21:22:15 +0900127 unsigned short *mOutputChars;
satok2df30602011-07-15 13:49:00 +0900128 ProximityInfo *mProximityInfo;
satokcfca3c62011-08-10 14:30:10 +0900129 Correction *mCorrection;
satok30088252010-12-01 21:22:15 +0900130 int mInputLength;
satok715514d2010-12-02 20:19:59 +0900131 // MAX_WORD_LENGTH_INTERNAL must be bigger than MAX_WORD_LENGTH
132 unsigned short mWord[MAX_WORD_LENGTH_INTERNAL];
satokd2997922010-12-07 13:08:39 +0900133
satok208268d2011-08-10 15:44:08 +0900134 int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
135 int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
136 int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
satok30088252010-12-01 21:22:15 +0900137};
Ken Wakasace9e52a2011-06-18 13:09:55 +0900138} // namespace latinime
satok30088252010-12-01 21:22:15 +0900139
140#endif // LATINIME_UNIGRAM_DICTIONARY_H