blob: b61ebd23bc53ee7682277127916bd343a2c5b1f0 [file] [log] [blame]
satoke808e432010-12-02 14:53:24 +09001/*
2**
3** Copyright 2010, The Android Open Source Project
4**
5** Licensed under the Apache License, Version 2.0 (the "License");
6** you may not use this file except in compliance with the License.
7** You may obtain a copy of the License at
8**
9** http://www.apache.org/licenses/LICENSE-2.0
10**
11** Unless required by applicable law or agreed to in writing, software
12** distributed under the License is distributed on an "AS IS" BASIS,
13** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14** See the License for the specific language governing permissions and
15** limitations under the License.
16*/
17
18#ifndef LATINIME_DEFINES_H
19#define LATINIME_DEFINES_H
20
satok827ced82011-07-14 09:01:09 +090021#if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG)
22#include <cutils/log.h>
satok9fb6f472012-01-13 18:01:22 +090023#define AKLOGE ALOGE
24#define AKLOGI ALOGI
satok6ad15fc2012-01-16 16:21:21 +090025
26#define DUMP_WORD(word, length) do { dumpWord(word, length); } while(0)
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +090027#define DUMP_WORD_INT(word, length) do { dumpWordInt(word, length); } while(0)
satok6ad15fc2012-01-16 16:21:21 +090028
Tadashi G. Takaokad1dbdb62012-03-06 15:35:46 +090029static inline void dumpWord(const unsigned short* word, const int length) {
30 static char charBuf[50];
satok6ad15fc2012-01-16 16:21:21 +090031
satok6ad15fc2012-01-16 16:21:21 +090032 for (int i = 0; i < length; ++i) {
33 charBuf[i] = word[i];
34 }
35 charBuf[length] = 0;
36 AKLOGI("[ %s ]", charBuf);
37}
38
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +090039static inline void dumpWordInt(const int* word, const int length) {
40 static char charBuf[50];
41
42 for (int i = 0; i < length; ++i) {
43 charBuf[i] = word[i];
44 }
45 charBuf[length] = 0;
46 AKLOGI("i[ %s ]", charBuf);
47}
48
satok827ced82011-07-14 09:01:09 +090049#else
satok9fb6f472012-01-13 18:01:22 +090050#define AKLOGE(fmt, ...)
51#define AKLOGI(fmt, ...)
satok6ad15fc2012-01-16 16:21:21 +090052#define DUMP_WORD(word, length)
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +090053#define DUMP_WORD_INT(word, length)
satok827ced82011-07-14 09:01:09 +090054#endif
55
satok20d9fda2011-07-13 14:40:30 +090056#ifdef FLAG_DO_PROFILE
satok61e2f852011-01-05 14:13:07 +090057// Profiler
58#include <time.h>
satok9fb6f472012-01-13 18:01:22 +090059
satok61e2f852011-01-05 14:13:07 +090060#define PROF_BUF_SIZE 100
satok0028ed32012-05-16 20:42:12 +090061static float profile_buf[PROF_BUF_SIZE];
62static float profile_old[PROF_BUF_SIZE];
satok61e2f852011-01-05 14:13:07 +090063static unsigned int profile_counter[PROF_BUF_SIZE];
64
Ken Wakasae90b3332011-01-07 15:01:51 +090065#define PROF_RESET prof_reset()
66#define PROF_COUNT(prof_buf_id) ++profile_counter[prof_buf_id]
67#define PROF_OPEN do { PROF_RESET; PROF_START(PROF_BUF_SIZE - 1); } while(0)
68#define PROF_START(prof_buf_id) do { \
69 PROF_COUNT(prof_buf_id); profile_old[prof_buf_id] = (clock()); } while(0)
70#define PROF_CLOSE do { PROF_END(PROF_BUF_SIZE - 1); PROF_OUTALL; } while(0)
71#define PROF_END(prof_buf_id) profile_buf[prof_buf_id] += ((clock()) - profile_old[prof_buf_id])
72#define PROF_CLOCKOUT(prof_buf_id) \
satok9fb6f472012-01-13 18:01:22 +090073 AKLOGI("%s : clock is %f", __FUNCTION__, (clock() - profile_old[prof_buf_id]))
74#define PROF_OUTALL do { AKLOGI("--- %s ---", __FUNCTION__); prof_out(); } while(0)
satok61e2f852011-01-05 14:13:07 +090075
Tadashi G. Takaokad1dbdb62012-03-06 15:35:46 +090076static inline void prof_reset(void) {
Ken Wakasae90b3332011-01-07 15:01:51 +090077 for (int i = 0; i < PROF_BUF_SIZE; ++i) {
satok61e2f852011-01-05 14:13:07 +090078 profile_buf[i] = 0;
79 profile_old[i] = 0;
80 profile_counter[i] = 0;
81 }
82}
83
Tadashi G. Takaokad1dbdb62012-03-06 15:35:46 +090084static inline void prof_out(void) {
satok61e2f852011-01-05 14:13:07 +090085 if (profile_counter[PROF_BUF_SIZE - 1] != 1) {
satok9fb6f472012-01-13 18:01:22 +090086 AKLOGI("Error: You must call PROF_OPEN before PROF_CLOSE.");
satok61e2f852011-01-05 14:13:07 +090087 }
satok9fb6f472012-01-13 18:01:22 +090088 AKLOGI("Total time is %6.3f ms.",
satok0028ed32012-05-16 20:42:12 +090089 profile_buf[PROF_BUF_SIZE - 1] * 1000 / (float)CLOCKS_PER_SEC);
90 float all = 0;
Ken Wakasae90b3332011-01-07 15:01:51 +090091 for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) {
satok61e2f852011-01-05 14:13:07 +090092 all += profile_buf[i];
93 }
Ken Wakasae90b3332011-01-07 15:01:51 +090094 if (all == 0) all = 1;
95 for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) {
96 if (profile_buf[i] != 0) {
satok9fb6f472012-01-13 18:01:22 +090097 AKLOGI("(%d): Used %4.2f%%, %8.4f ms. Called %d times.",
Ken Wakasae90b3332011-01-07 15:01:51 +090098 i, (profile_buf[i] * 100 / all),
satok0028ed32012-05-16 20:42:12 +090099 profile_buf[i] * 1000 / (float)CLOCKS_PER_SEC, profile_counter[i]);
Ken Wakasae90b3332011-01-07 15:01:51 +0900100 }
satok61e2f852011-01-05 14:13:07 +0900101 }
102}
103
satok20d9fda2011-07-13 14:40:30 +0900104#else // FLAG_DO_PROFILE
satok61e2f852011-01-05 14:13:07 +0900105#define PROF_BUF_SIZE 0
106#define PROF_RESET
107#define PROF_COUNT(prof_buf_id)
108#define PROF_OPEN
109#define PROF_START(prof_buf_id)
110#define PROF_CLOSE
111#define PROF_END(prof_buf_id)
112#define PROF_CLOCK_OUT(prof_buf_id)
113#define PROF_CLOCKOUT(prof_buf_id)
114#define PROF_OUTALL
115
satok20d9fda2011-07-13 14:40:30 +0900116#endif // FLAG_DO_PROFILE
117
118#ifdef FLAG_DBG
119#include <cutils/log.h>
120#ifndef LOG_TAG
121#define LOG_TAG "LatinIME: "
122#endif
123#define DEBUG_DICT true
124#define DEBUG_DICT_FULL false
satok0cedd2b2011-08-12 01:05:27 +0900125#define DEBUG_EDIT_DISTANCE false
satok10266c02011-08-19 22:05:59 +0900126#define DEBUG_SHOW_FOUND_WORD false
satok20d9fda2011-07-13 14:40:30 +0900127#define DEBUG_NODE DEBUG_DICT_FULL
128#define DEBUG_TRACE DEBUG_DICT_FULL
satok1a6da632011-12-16 23:15:06 +0900129#define DEBUG_PROXIMITY_INFO false
satok0cb20972012-03-13 22:07:56 +0900130#define DEBUG_PROXIMITY_CHARS false
satok10266c02011-08-19 22:05:59 +0900131#define DEBUG_CORRECTION false
satok29dc8062012-01-17 15:59:15 +0900132#define DEBUG_CORRECTION_FREQ false
133#define DEBUG_WORDS_PRIORITY_QUEUE false
satok20d9fda2011-07-13 14:40:30 +0900134
135#else // FLAG_DBG
satok827ced82011-07-14 09:01:09 +0900136
satok20d9fda2011-07-13 14:40:30 +0900137#define DEBUG_DICT false
138#define DEBUG_DICT_FULL false
satok0cedd2b2011-08-12 01:05:27 +0900139#define DEBUG_EDIT_DISTANCE false
satok20d9fda2011-07-13 14:40:30 +0900140#define DEBUG_SHOW_FOUND_WORD false
141#define DEBUG_NODE false
142#define DEBUG_TRACE false
143#define DEBUG_PROXIMITY_INFO false
satok0cb20972012-03-13 22:07:56 +0900144#define DEBUG_PROXIMITY_CHARS false
satok10266c02011-08-19 22:05:59 +0900145#define DEBUG_CORRECTION false
146#define DEBUG_CORRECTION_FREQ false
satok16379df2011-12-12 20:53:22 +0900147#define DEBUG_WORDS_PRIORITY_QUEUE false
satok20d9fda2011-07-13 14:40:30 +0900148
satok0cedd2b2011-08-12 01:05:27 +0900149
satoke808e432010-12-02 14:53:24 +0900150#endif // FLAG_DBG
151
satok662fe692010-12-08 17:05:39 +0900152#ifndef U_SHORT_MAX
Yusuke Nojima63095932011-09-20 14:03:42 +0900153#define U_SHORT_MAX 65535 // ((1 << 16) - 1)
satok662fe692010-12-08 17:05:39 +0900154#endif
Jean Chalarda5d58492011-02-18 17:50:58 +0900155#ifndef S_INT_MAX
satok3c4bb772011-03-04 22:50:19 -0800156#define S_INT_MAX 2147483647 // ((1 << 31) - 1)
Jean Chalarda5d58492011-02-18 17:50:58 +0900157#endif
satok662fe692010-12-08 17:05:39 +0900158
Ken Wakasae90b3332011-01-07 15:01:51 +0900159// Define this to use mmap() for dictionary loading. Undefine to use malloc() instead of mmap().
160// We measured and compared performance of both, and found mmap() is fairly good in terms of
161// loading time, and acceptable even for several initial lookups which involve page faults.
162#define USE_MMAP_FOR_DICTIONARY
163
satoke808e432010-12-02 14:53:24 +0900164// 22-bit address = ~4MB dictionary size limit, which on average would be about 200k-300k words
165#define ADDRESS_MASK 0x3FFFFF
166
167// The bit that decides if an address follows in the next 22 bits
168#define FLAG_ADDRESS_MASK 0x40
169// The bit that decides if this is a terminal node for a word. The node could still have children,
170// if the word has other endings.
171#define FLAG_TERMINAL_MASK 0x80
172
173#define FLAG_BIGRAM_READ 0x80
174#define FLAG_BIGRAM_CHILDEXIST 0x40
175#define FLAG_BIGRAM_CONTINUED 0x80
176#define FLAG_BIGRAM_FREQ 0x7F
177
178#define DICTIONARY_VERSION_MIN 200
satoke808e432010-12-02 14:53:24 +0900179#define NOT_VALID_WORD -99
Jean Chalard1059f272011-06-28 20:45:05 +0900180#define NOT_A_CHARACTER -1
Yusuke Nojimae4ba8222011-10-05 14:55:07 +0900181#define NOT_A_DISTANCE -1
Jean Chalard0bfe3592012-01-25 18:11:26 +0900182#define NOT_A_COORDINATE -1
Yusuke Nojimaa4c1f1c2011-10-06 19:12:20 +0900183#define EQUIVALENT_CHAR_WITHOUT_DISTANCE_INFO -2
184#define PROXIMITY_CHAR_WITHOUT_DISTANCE_INFO -3
satoke05b3f42012-01-31 17:15:43 +0900185#define ADDITIONAL_PROXIMITY_CHAR_DISTANCE_INFO -4
Jean Chalardbbc25602012-03-23 17:05:03 +0900186#define NOT_AN_INDEX -1
Jean Chalard171d1802012-04-23 18:51:00 +0900187#define NOT_A_PROBABILITY -1
satoke808e432010-12-02 14:53:24 +0900188
satok817e5172011-03-04 06:06:45 -0800189#define KEYCODE_SPACE ' '
190
Yusuke Nojimaafb90762011-10-05 18:11:42 +0900191#define CALIBRATE_SCORE_BY_TOUCH_COORDINATES true
Yusuke Nojima258bfe62011-09-28 12:59:43 +0900192
satok662fe692010-12-08 17:05:39 +0900193#define SUGGEST_WORDS_WITH_MISSING_CHARACTER true
satok662fe692010-12-08 17:05:39 +0900194#define SUGGEST_WORDS_WITH_EXCESSIVE_CHARACTER true
satoka3d78f62010-12-09 22:08:33 +0900195#define SUGGEST_WORDS_WITH_TRANSPOSED_CHARACTERS true
satok99557162012-01-26 22:49:13 +0900196#define SUGGEST_MULTIPLE_WORDS true
satoka3d78f62010-12-09 22:08:33 +0900197
Jean Chalard8dc754a2011-01-27 14:20:22 +0900198// The following "rate"s are used as a multiplier before dividing by 100, so they are in percent.
satok72bc17e2011-04-13 17:23:27 +0900199#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE 80
satokdc5301e2011-04-11 16:14:45 +0900200#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X 12
satok54af64a2012-01-17 15:58:23 +0900201#define WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE 58
satok8330b482012-01-23 16:52:37 +0900202#define WORDS_WITH_MISTYPED_SPACE_DEMOTION_RATE 50
satoka3d78f62010-12-09 22:08:33 +0900203#define WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE 75
satok54fe9e02010-12-13 14:42:35 +0900204#define WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE 75
satoka161a4a2012-01-16 18:38:32 +0900205#define WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE 70
satok58c49b92011-01-27 03:23:39 +0900206#define FULL_MATCHED_WORDS_PROMOTION_RATE 120
satok9d2a3022011-04-14 19:13:34 +0900207#define WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE 90
satok1b9fa942012-02-02 18:49:22 +0900208#define WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE 70
satok635f68e2011-08-10 22:19:33 +0900209#define WORDS_WITH_MATCH_SKIP_PROMOTION_RATE 105
satok1b9fa942012-02-02 18:49:22 +0900210#define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_RATE 148
211#define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_MULTIPLIER 3
satok10266c02011-08-19 22:05:59 +0900212#define CORRECTION_COUNT_RATE_DEMOTION_RATE_BASE 45
213#define INPUT_EXCEEDS_OUTPUT_DEMOTION_RATE 70
214#define FIRST_CHAR_DIFFERENT_DEMOTION_RATE 96
satokeb050fc2011-10-03 19:21:13 +0900215#define TWO_WORDS_CAPITALIZED_DEMOTION_RATE 50
satok54af64a2012-01-17 15:58:23 +0900216#define TWO_WORDS_CORRECTION_DEMOTION_BASE 80
217#define TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER 1
Yusuke Nojimaa4c1f1c2011-10-06 19:12:20 +0900218#define ZERO_DISTANCE_PROMOTION_RATE 110
219#define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f
220#define HALF_SCORE_SQUARED_RADIUS 32.0f
satoka85f4922012-01-30 18:18:30 +0900221#define MAX_FREQ 255
Jean Chalard9416c812012-05-15 19:24:47 +0900222#define MAX_BIGRAM_FREQ 15
satoke808e432010-12-02 14:53:24 +0900223
satok1147c7b2011-12-14 15:04:58 +0900224// This must be greater than or equal to MAX_WORD_LENGTH defined in BinaryDictionary.java
satokf5cded12010-12-06 21:28:24 +0900225// This is only used for the size of array. Not to be used in c functions.
226#define MAX_WORD_LENGTH_INTERNAL 48
satok715514d2010-12-02 20:19:59 +0900227
satoke05b3f42012-01-31 17:15:43 +0900228// This must be equal to ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE in KeyDetector.java
229#define ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE 2
230
satoka7e5a5a2011-12-15 16:49:12 +0900231// Word limit for sub queues used in WordsPriorityQueuePool. Sub queues are temporary queues used
232// for better performance.
satok6ad15fc2012-01-16 16:21:21 +0900233// Holds up to 1 candidate for each word
234#define SUB_QUEUE_MAX_WORDS 1
satokb9604772012-01-13 15:41:17 +0900235#define SUB_QUEUE_MAX_COUNT 10
satok54af64a2012-01-17 15:58:23 +0900236#define SUB_QUEUE_MIN_WORD_LENGTH 4
satoka85f4922012-01-30 18:18:30 +0900237#define MULTIPLE_WORDS_SUGGESTION_MAX_WORDS 10
Satoshi Kataoka6cbe2042012-05-30 17:28:34 +0900238// TODO: Remove this limitation
239#define MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH 12
240// TODO: Remove this limitation
241#define MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT 110
satoka85f4922012-01-30 18:18:30 +0900242#define MULTIPLE_WORDS_DEMOTION_RATE 80
243#define MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION 6
satoka7e5a5a2011-12-15 16:49:12 +0900244
Jean Chalard9416c812012-05-15 19:24:47 +0900245#define TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD 0.35
246#define START_TWO_WORDS_CORRECTION_THRESHOLD 0.185
satoka0ac31f2012-05-23 19:55:27 +0900247/* heuristic... This should be changed if we change the unit of the frequency. */
248#define SUPPRESS_SHORT_MULTIPLE_WORDS_THRESHOLD_FREQ (MAX_FREQ * 58 / 100)
satok29dc8062012-01-17 15:59:15 +0900249
satok68319262010-12-03 19:38:08 +0900250#define MAX_DEPTH_MULTIPLIER 3
251
satok1f6b52e2012-01-30 13:53:58 +0900252#define FIRST_WORD_INDEX 0
satok8330b482012-01-23 16:52:37 +0900253
Jean Chalard6c300612012-03-06 19:54:03 +0900254// TODO: Reduce this constant if possible; check the maximum number of digraphs in the same
255// word in the dictionary for languages with digraphs, like German and French
256#define DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH 5
Jean Chalarda787dba2011-03-04 12:17:48 +0900257
satok54fe9e02010-12-13 14:42:35 +0900258// Minimum suggest depth for one word for all cases except for missing space suggestions.
259#define MIN_SUGGEST_DEPTH 1
satok99557162012-01-26 22:49:13 +0900260#define MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION 3
satok54fe9e02010-12-13 14:42:35 +0900261#define MIN_USER_TYPED_LENGTH_FOR_EXCESSIVE_CHARACTER_SUGGESTION 3
satok662fe692010-12-08 17:05:39 +0900262
Jean Chalardf1634c82012-05-02 19:05:27 +0900263// Size, in bytes, of the bloom filter index for bigrams
264// 128 gives us 1024 buckets. The probability of false positive is (1 - e ** (-kn/m))**k,
265// where k is the number of hash functions, n the number of bigrams, and m the number of
266// bits we can test.
267// At the moment 100 is the maximum number of bigrams for a word with the current
268// dictionaries, so n = 100. 1024 buckets give us m = 1024.
269// With 1 hash function, our false positive rate is about 9.3%, which should be enough for
270// our uses since we are only using this to increase average performance. For the record,
271// k = 2 gives 3.1% and k = 3 gives 1.6%. With k = 1, making m = 2048 gives 4.8%,
272// and m = 4096 gives 2.4%.
273#define BIGRAM_FILTER_BYTE_SIZE 128
274// Must be smaller than BIGRAM_FILTER_BYTE_SIZE * 8, and preferably prime. 1021 is the largest
275// prime under 128 * 8.
276#define BIGRAM_FILTER_MODULO 1021
277#if BIGRAM_FILTER_BYTE_SIZE * 8 < BIGRAM_FILTER_MODULO
278#error "BIGRAM_FILTER_MODULO is larger than BIGRAM_FILTER_BYTE_SIZE"
279#endif
280
Tadashi G. Takaoka09baa362012-02-03 21:24:53 +0900281template<typename T> inline T min(T a, T b) { return a < b ? a : b; }
282template<typename T> inline T max(T a, T b) { return a > b ? a : b; }
satokf5cded12010-12-06 21:28:24 +0900283
Yusuke Nojima258bfe62011-09-28 12:59:43 +0900284// The ratio of neutral area radius to sweet spot radius.
285#define NEUTRAL_AREA_RADIUS_RATIO 1.3f
286
satoke05b3f42012-01-31 17:15:43 +0900287// DEBUG
satoka85f4922012-01-30 18:18:30 +0900288#define INPUTLENGTH_FOR_DEBUG -1
satoke05b3f42012-01-31 17:15:43 +0900289#define MIN_OUTPUT_INDEX_FOR_DEBUG -1
290
satoke808e432010-12-02 14:53:24 +0900291#endif // LATINIME_DEFINES_H