keep history after reset to jb-ub-latinimegoogle-bayo
diff --git a/dictionaries/cs_wordlist.combined.gz b/dictionaries/cs_wordlist.combined.gz
index d69ef64..7829d65 100644
--- a/dictionaries/cs_wordlist.combined.gz
+++ b/dictionaries/cs_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/da_wordlist.combined.gz b/dictionaries/da_wordlist.combined.gz
index 919d28e..e714019 100644
--- a/dictionaries/da_wordlist.combined.gz
+++ b/dictionaries/da_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/de_wordlist.combined.gz b/dictionaries/de_wordlist.combined.gz
index f5cce9d..6a4bd44 100644
--- a/dictionaries/de_wordlist.combined.gz
+++ b/dictionaries/de_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/en_GB_wordlist.combined.gz b/dictionaries/en_GB_wordlist.combined.gz
index d28ef48..839f3ef 100644
--- a/dictionaries/en_GB_wordlist.combined.gz
+++ b/dictionaries/en_GB_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/en_US_wordlist.combined.gz b/dictionaries/en_US_wordlist.combined.gz
index b656f88..5595c75 100644
--- a/dictionaries/en_US_wordlist.combined.gz
+++ b/dictionaries/en_US_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/en_wordlist.combined.gz b/dictionaries/en_wordlist.combined.gz
index 8aa40e9..69c39d5 100644
--- a/dictionaries/en_wordlist.combined.gz
+++ b/dictionaries/en_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/es_wordlist.combined.gz b/dictionaries/es_wordlist.combined.gz
index 53b8607..0a48b6d 100644
--- a/dictionaries/es_wordlist.combined.gz
+++ b/dictionaries/es_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/fi_wordlist.combined.gz b/dictionaries/fi_wordlist.combined.gz
index 2720116..eefbfe5 100644
--- a/dictionaries/fi_wordlist.combined.gz
+++ b/dictionaries/fi_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/fr_wordlist.combined.gz b/dictionaries/fr_wordlist.combined.gz
index 1d988d6..1a18320 100644
--- a/dictionaries/fr_wordlist.combined.gz
+++ b/dictionaries/fr_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/hr_wordlist.combined.gz b/dictionaries/hr_wordlist.combined.gz
index 7694a2a..864f676 100644
--- a/dictionaries/hr_wordlist.combined.gz
+++ b/dictionaries/hr_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/it_wordlist.combined.gz b/dictionaries/it_wordlist.combined.gz
index 3b84cd7..dfb1752 100644
--- a/dictionaries/it_wordlist.combined.gz
+++ b/dictionaries/it_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/lt_wordlist.combined.gz b/dictionaries/lt_wordlist.combined.gz
index 316a5af..029722d 100644
--- a/dictionaries/lt_wordlist.combined.gz
+++ b/dictionaries/lt_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/lv_wordlist.combined.gz b/dictionaries/lv_wordlist.combined.gz
index b036ac2..41e1c28 100644
--- a/dictionaries/lv_wordlist.combined.gz
+++ b/dictionaries/lv_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/nb_wordlist.combined.gz b/dictionaries/nb_wordlist.combined.gz
index b6e0d42..b699912 100644
--- a/dictionaries/nb_wordlist.combined.gz
+++ b/dictionaries/nb_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/nl_wordlist.combined.gz b/dictionaries/nl_wordlist.combined.gz
index 48ab0f4..89c2388 100644
--- a/dictionaries/nl_wordlist.combined.gz
+++ b/dictionaries/nl_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/pl_wordlist.combined.gz b/dictionaries/pl_wordlist.combined.gz
index bf02298..2b53f69 100644
--- a/dictionaries/pl_wordlist.combined.gz
+++ b/dictionaries/pl_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/pt_BR_wordlist.combined.gz b/dictionaries/pt_BR_wordlist.combined.gz
index 221ea75..2d22447 100644
--- a/dictionaries/pt_BR_wordlist.combined.gz
+++ b/dictionaries/pt_BR_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/pt_PT_wordlist.combined.gz b/dictionaries/pt_PT_wordlist.combined.gz
index 6a041d9..1504165 100644
--- a/dictionaries/pt_PT_wordlist.combined.gz
+++ b/dictionaries/pt_PT_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/sl_wordlist.combined.gz b/dictionaries/sl_wordlist.combined.gz
index 41a576b..55e1bb1 100644
--- a/dictionaries/sl_wordlist.combined.gz
+++ b/dictionaries/sl_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/sr_wordlist.combined.gz b/dictionaries/sr_wordlist.combined.gz
index dec6ae8..8488a08 100644
--- a/dictionaries/sr_wordlist.combined.gz
+++ b/dictionaries/sr_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/sv_wordlist.combined.gz b/dictionaries/sv_wordlist.combined.gz
index 0471772..6342520 100644
--- a/dictionaries/sv_wordlist.combined.gz
+++ b/dictionaries/sv_wordlist.combined.gz
Binary files differ
diff --git a/dictionaries/tr_wordlist.combined.gz b/dictionaries/tr_wordlist.combined.gz
index fae79ca..0251778 100644
--- a/dictionaries/tr_wordlist.combined.gz
+++ b/dictionaries/tr_wordlist.combined.gz
Binary files differ
diff --git a/java/res/drawable-hdpi/btn_keyboard_key_dark_active_holo.9.png b/java/res/drawable-hdpi/btn_keyboard_key_dark_active_holo.9.png
index 87211a5..fa2cb85 100644
--- a/java/res/drawable-hdpi/btn_keyboard_key_dark_active_holo.9.png
+++ b/java/res/drawable-hdpi/btn_keyboard_key_dark_active_holo.9.png
Binary files differ
diff --git a/java/res/drawable-hdpi/sym_keyboard_voice_holo_dark.png b/java/res/drawable-hdpi/sym_keyboard_voice_holo_dark.png
index c1e16a6..8a6336a 100644
--- a/java/res/drawable-hdpi/sym_keyboard_voice_holo_dark.png
+++ b/java/res/drawable-hdpi/sym_keyboard_voice_holo_dark.png
Binary files differ
diff --git a/java/res/drawable-hdpi/sym_keyboard_voice_off_holo_dark.png b/java/res/drawable-hdpi/sym_keyboard_voice_off_holo_dark.png
index 26d0684..edf1379 100644
--- a/java/res/drawable-hdpi/sym_keyboard_voice_off_holo_dark.png
+++ b/java/res/drawable-hdpi/sym_keyboard_voice_off_holo_dark.png
Binary files differ
diff --git a/java/res/drawable-mdpi/btn_keyboard_key_dark_active_holo.9.png b/java/res/drawable-mdpi/btn_keyboard_key_dark_active_holo.9.png
index f98653e..8e9a349 100644
--- a/java/res/drawable-mdpi/btn_keyboard_key_dark_active_holo.9.png
+++ b/java/res/drawable-mdpi/btn_keyboard_key_dark_active_holo.9.png
Binary files differ
diff --git a/java/res/drawable-mdpi/sym_keyboard_voice_holo_dark.png b/java/res/drawable-mdpi/sym_keyboard_voice_holo_dark.png
index 16be37d..0795fcc 100644
--- a/java/res/drawable-mdpi/sym_keyboard_voice_holo_dark.png
+++ b/java/res/drawable-mdpi/sym_keyboard_voice_holo_dark.png
Binary files differ
diff --git a/java/res/drawable-mdpi/sym_keyboard_voice_off_holo_dark.png b/java/res/drawable-mdpi/sym_keyboard_voice_off_holo_dark.png
index 95d718a..f76da57 100644
--- a/java/res/drawable-mdpi/sym_keyboard_voice_off_holo_dark.png
+++ b/java/res/drawable-mdpi/sym_keyboard_voice_off_holo_dark.png
Binary files differ
diff --git a/java/res/drawable-xhdpi/btn_keyboard_key_dark_active_holo.9.png b/java/res/drawable-xhdpi/btn_keyboard_key_dark_active_holo.9.png
index 738316d..a2f6ac0 100644
--- a/java/res/drawable-xhdpi/btn_keyboard_key_dark_active_holo.9.png
+++ b/java/res/drawable-xhdpi/btn_keyboard_key_dark_active_holo.9.png
Binary files differ
diff --git a/java/res/drawable-xhdpi/sym_keyboard_voice_holo_dark.png b/java/res/drawable-xhdpi/sym_keyboard_voice_holo_dark.png
index 944a852..b2bb9b8 100644
--- a/java/res/drawable-xhdpi/sym_keyboard_voice_holo_dark.png
+++ b/java/res/drawable-xhdpi/sym_keyboard_voice_holo_dark.png
Binary files differ
diff --git a/java/res/drawable-xhdpi/sym_keyboard_voice_off_holo_dark.png b/java/res/drawable-xhdpi/sym_keyboard_voice_off_holo_dark.png
index 2016caf..23e75bf 100644
--- a/java/res/drawable-xhdpi/sym_keyboard_voice_off_holo_dark.png
+++ b/java/res/drawable-xhdpi/sym_keyboard_voice_off_holo_dark.png
Binary files differ
diff --git a/java/res/drawable-xxhdpi/btn_keyboard_key_dark_active_holo.9.png b/java/res/drawable-xxhdpi/btn_keyboard_key_dark_active_holo.9.png
index b35c29f..17f0a7a 100644
--- a/java/res/drawable-xxhdpi/btn_keyboard_key_dark_active_holo.9.png
+++ b/java/res/drawable-xxhdpi/btn_keyboard_key_dark_active_holo.9.png
Binary files differ
diff --git a/java/res/drawable-xxhdpi/sym_keyboard_voice_holo_dark.png b/java/res/drawable-xxhdpi/sym_keyboard_voice_holo_dark.png
index 6809f07..f04cadf 100644
--- a/java/res/drawable-xxhdpi/sym_keyboard_voice_holo_dark.png
+++ b/java/res/drawable-xxhdpi/sym_keyboard_voice_holo_dark.png
Binary files differ
diff --git a/java/res/drawable-xxhdpi/sym_keyboard_voice_off_holo_dark.png b/java/res/drawable-xxhdpi/sym_keyboard_voice_off_holo_dark.png
index 6bd506a..e74d523 100644
--- a/java/res/drawable-xxhdpi/sym_keyboard_voice_off_holo_dark.png
+++ b/java/res/drawable-xxhdpi/sym_keyboard_voice_off_holo_dark.png
Binary files differ
diff --git a/java/res/raw/main_de.dict b/java/res/raw/main_de.dict
index 5d35e64..69796bb 100644
--- a/java/res/raw/main_de.dict
+++ b/java/res/raw/main_de.dict
Binary files differ
diff --git a/java/res/raw/main_en.dict b/java/res/raw/main_en.dict
index 6564d47..bef6b10 100644
--- a/java/res/raw/main_en.dict
+++ b/java/res/raw/main_en.dict
Binary files differ
diff --git a/java/res/raw/main_es.dict b/java/res/raw/main_es.dict
index f5906c2..261ab8c 100644
--- a/java/res/raw/main_es.dict
+++ b/java/res/raw/main_es.dict
Binary files differ
diff --git a/java/res/raw/main_fr.dict b/java/res/raw/main_fr.dict
index 10adad0..18f5298 100644
--- a/java/res/raw/main_fr.dict
+++ b/java/res/raw/main_fr.dict
Binary files differ
diff --git a/java/res/raw/main_it.dict b/java/res/raw/main_it.dict
index 523f645..e161c24 100644
--- a/java/res/raw/main_it.dict
+++ b/java/res/raw/main_it.dict
Binary files differ
diff --git a/java/res/raw/main_pt_br.dict b/java/res/raw/main_pt_br.dict
index f9ae9b5..21bbe7c 100644
--- a/java/res/raw/main_pt_br.dict
+++ b/java/res/raw/main_pt_br.dict
Binary files differ
diff --git a/java/res/values-af/strings.xml b/java/res/values-af/strings.xml
index 01a57e7..a3c95b2 100644
--- a/java/res/values-af/strings.xml
+++ b/java/res/values-af/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Koppel \'n kopstuk om te hoor hoe wagwoordsleutels hardop gesê word."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Huidige teks is %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Geen teks ingevoer nie"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Sleutelkode %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift aan (tik om te deaktiveer)"</string>
diff --git a/java/res/values-am/strings.xml b/java/res/values-am/strings.xml
index 774f2c5..89e37b2 100644
--- a/java/res/values-am/strings.xml
+++ b/java/res/values-am/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"የይለፍቃል ቁልፎች ጮክ በለው ሲነገሩ ለመስማት የጆሮ ማዳመጫ ሰካ::"</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"የአሁኑ ፅሁፍ %s ነው"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"ምንም ፅሁፍ አልገባም"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"የቁልፍ ኮድ%d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"ቀይር"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"ቅያር በርቷል (ለማሰናክል ንካ)"</string>
@@ -211,7 +215,7 @@
<string name="message_updating" msgid="4457761393932375219">"ዝማኔዎችን በመፈለግ ላይ"</string>
<string name="message_loading" msgid="8689096636874758814">"በመጫን ላይ..."</string>
<string name="main_dict_description" msgid="3072821352793492143">"ዋና መዝገበ-ቃላት"</string>
- <string name="cancel" msgid="6830980399865683324">"ይቅር"</string>
+ <string name="cancel" msgid="6830980399865683324">"ሰርዝ"</string>
<string name="install_dict" msgid="180852772562189365">"ጫን"</string>
<string name="cancel_download_dict" msgid="7843340278507019303">"ሰርዝ"</string>
<string name="delete_dict" msgid="756853268088330054">"ሰርዝ"</string>
diff --git a/java/res/values-ar/strings.xml b/java/res/values-ar/strings.xml
index 7e39b59..9c3ff51 100644
--- a/java/res/values-ar/strings.xml
+++ b/java/res/values-ar/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"يمكنك توصيل سماعة رأس لسماع مفاتيح كلمة المرور منطوقة بصوت عالٍ."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"النص الحالي هو %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"لم يتم إدخال نص"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"رمز المفتاح %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"العالي"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift يعمل (انقر للتعطيل)"</string>
diff --git a/java/res/values-be/strings.xml b/java/res/values-be/strings.xml
index db40c8a..9e591de 100644
--- a/java/res/values-be/strings.xml
+++ b/java/res/values-be/strings.xml
@@ -87,6 +87,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Каб праслухаць паролi, падключыце гарнiтуру."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Бягучы тэкст %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Тэкст не ўведзены"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Клавішны код %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Зрух"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift уключаны (націснiце, каб адключыць)"</string>
diff --git a/java/res/values-bg/strings.xml b/java/res/values-bg/strings.xml
index d3879cc..c21b534 100644
--- a/java/res/values-bg/strings.xml
+++ b/java/res/values-bg/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Включете слушалки, за да чуете клавишите за паролата на висок глас."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Текущият текст е %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Няма въведен текст"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Код на клавишa %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"„Shift“ е включен (докоснете за деактивиране)"</string>
diff --git a/java/res/values-ca/strings.xml b/java/res/values-ca/strings.xml
index 03e2f3f..d791452 100644
--- a/java/res/values-ca/strings.xml
+++ b/java/res/values-ca/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Connecta un auricular per escoltar les claus de la contrasenya en veu alta."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"El text actual és %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"No s\'ha introduït cap text"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Clau de codi %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Maj"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Maj activat (pica per desactivar)"</string>
diff --git a/java/res/values-cs/strings.xml b/java/res/values-cs/strings.xml
index a707f26..2cd159d 100644
--- a/java/res/values-cs/strings.xml
+++ b/java/res/values-cs/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Chcete-li slyšet, které klávesy jste při zadávání hesla stiskli, připojte sluchátka."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Aktuální text je %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Není zadán žádný text"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Kód klávesy %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Klávesa Shift je zapnutá (vypnete ji klepnutím)."</string>
diff --git a/java/res/values-da/strings.xml b/java/res/values-da/strings.xml
index 1876895..6c9d9d7 100644
--- a/java/res/values-da/strings.xml
+++ b/java/res/values-da/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Tilslut et headset for at høre indtastningen blive læst højt ved angivelse af adgangskode."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Nuværende tekst er %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Der er ingen indtastet tekst"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Tastekode %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift-tast"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Skift er slået til (tryk for at deaktivere)"</string>
diff --git a/java/res/values-de/strings.xml b/java/res/values-de/strings.xml
index 5f3e619..1a4e385 100644
--- a/java/res/values-de/strings.xml
+++ b/java/res/values-de/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Schließen Sie ein Headset an, um das Passwort gesprochen zu hören."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Aktueller Text lautet %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Kein Text eingegeben"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Tastencode %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Umschalttaste"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Umschalttaste aktiviert (zum Deaktivieren berühren)"</string>
diff --git a/java/res/values-el/strings.xml b/java/res/values-el/strings.xml
index 27eb865..c2ee2d3 100644
--- a/java/res/values-el/strings.xml
+++ b/java/res/values-el/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Συνδέστε ένα σετ ακουστικών για να ακούσετε τα πλήκτρα του κωδικού πρόσβασης να εκφωνούνται δυνατά."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Το τρέχον κείμενο είναι %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Δεν υπάρχει κείμενο"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Κωδικός πλήκτρου %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Το Shift είναι ενεργοποιημένο (πατήστε για απενεργοποίηση)"</string>
diff --git a/java/res/values-en-rGB/strings.xml b/java/res/values-en-rGB/strings.xml
index 6337aa0..0586bcc 100644
--- a/java/res/values-en-rGB/strings.xml
+++ b/java/res/values-en-rGB/strings.xml
@@ -77,13 +77,17 @@
<string name="label_go_key" msgid="1635148082137219148">"Go"</string>
<string name="label_next_key" msgid="362972844525672568">"Next"</string>
<string name="label_previous_key" msgid="1211868118071386787">"Prev"</string>
- <string name="label_done_key" msgid="2441578748772529288">"Finished"</string>
+ <string name="label_done_key" msgid="2441578748772529288">"Done"</string>
<string name="label_send_key" msgid="2815056534433717444">"Send"</string>
<string name="label_pause_key" msgid="181098308428035340">"Pause"</string>
<string name="label_wait_key" msgid="6402152600878093134">"Wait"</string>
<string name="spoken_use_headphones" msgid="896961781287283493">"Plug in a headset to hear password keys spoken aloud."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Current text is %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"No text entered"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Key code %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift on (tap to disable)"</string>
diff --git a/java/res/values-es-rUS/strings.xml b/java/res/values-es-rUS/strings.xml
index b1a4214..42c3932 100644
--- a/java/res/values-es-rUS/strings.xml
+++ b/java/res/values-es-rUS/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Enchufa tus auriculares para escuchar en voz alta qué teclas presionas al ingresar una contraseña."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"El texto actual es %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"No se ingresó texto."</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Clave de código %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Mayús"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Se activó el modo Mayúscula (toca para desactivarlo)."</string>
diff --git a/java/res/values-es/strings.xml b/java/res/values-es/strings.xml
index 9ff9430..4d72799 100644
--- a/java/res/values-es/strings.xml
+++ b/java/res/values-es/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Conecta un auricular para escuchar las contraseñas en voz alta."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"El texto actual es %s."</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"No se ha introducido texto."</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Código del teclado: %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Mayús"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Mayúsculas activadas (tocar para inhabilitar)"</string>
diff --git a/java/res/values-et-rEE/strings-appname.xml b/java/res/values-et-rEE/strings-appname.xml
new file mode 100644
index 0000000..bbc13d2
--- /dev/null
+++ b/java/res/values-et-rEE/strings-appname.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/*
+**
+** Copyright 2013, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+ -->
+
+<resources xmlns:android="http://schemas.android.com/apk/res/android"
+ xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="english_ime_name" msgid="5940510615957428904">"Androidi klaviatuur (AOSP)"</string>
+ <string name="spell_checker_service_name" msgid="1254221805440242662">"Androidi õigekirjakontroll (AOSP)"</string>
+ <string name="english_ime_settings" msgid="5760361067176802794">"Androidi klaviatuuri seaded (AOSP)"</string>
+ <string name="android_spell_checker_settings" msgid="6123949487832861885">"Androidi õigekirjakontrolli seaded (AOSP)"</string>
+</resources>
diff --git a/java/res/values-et-rEE/strings.xml b/java/res/values-et-rEE/strings.xml
index 62197d7..5706fa1 100644
--- a/java/res/values-et-rEE/strings.xml
+++ b/java/res/values-et-rEE/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Ühendage peakomplekt, et kuulata paroole."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Praegune tekst on %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Teksti ei ole sisestatud"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Klahvi kood: %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Tõstuklahv"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Tõstuklahv sees (puudutage keelamiseks)"</string>
diff --git a/java/res/values-fa/strings.xml b/java/res/values-fa/strings.xml
index 5d89516..c03e440 100644
--- a/java/res/values-fa/strings.xml
+++ b/java/res/values-fa/strings.xml
@@ -86,6 +86,10 @@
<!-- no translation found for spoken_current_text_is (2485723011272583845) -->
<skip />
<string name="spoken_no_text_entered" msgid="7479685225597344496">"متنی وارد نشده است"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<!-- String.format failed for translation -->
<!-- no translation found for spoken_description_unknown (3197434010402179157) -->
<skip />
diff --git a/java/res/values-fi/strings.xml b/java/res/values-fi/strings.xml
index 4cdf621..744e604 100644
--- a/java/res/values-fi/strings.xml
+++ b/java/res/values-fi/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Liitä kuulokkeet, niin kuulet mitä näppäimiä painat kirjoittaessasi salasanaa."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Nykyinen teksti on %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Ei kirjoitettua tekstiä"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Näppäimen koodi %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Vaihto"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Vaihto päällä (poista käytöstä napauttamalla)"</string>
diff --git a/java/res/values-fr/strings.xml b/java/res/values-fr/strings.xml
index e096a1f..5fedd4e 100644
--- a/java/res/values-fr/strings.xml
+++ b/java/res/values-fr/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Branchez des écouteurs pour entendre l\'énoncé à haute voix des touches lors de la saisie du mot de passe."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Le texte actuel est %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Aucun texte saisi"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Code touche %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Maj"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Touche Maj activée (appuyer pour désactiver)"</string>
diff --git a/java/res/values-hi/strings.xml b/java/res/values-hi/strings.xml
index dcbbcd9..3b3228a 100644
--- a/java/res/values-hi/strings.xml
+++ b/java/res/values-hi/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"ज़ोर से बोली गई पासवर्ड कुंजियां सुनने के लिए हेडसेट प्लग इन करें."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"वर्तमान पाठ %s है"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"कोई पाठ दर्ज नहीं किया गया"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"कुंजी कोड %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"शिफ़्ट"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift चालू (अक्षम करने के लिए टैप करें)"</string>
diff --git a/java/res/values-hr/strings.xml b/java/res/values-hr/strings.xml
index 0dca64e..caef954 100644
--- a/java/res/values-hr/strings.xml
+++ b/java/res/values-hr/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Priključite slušalice da biste čuli tipke zaporke izgovorene naglas."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Trenutačni tekst je %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Nije unesen tekst"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Kôd tipke %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Uključena tipka Shift (dotaknite da onemogućite)"</string>
diff --git a/java/res/values-hu/strings.xml b/java/res/values-hu/strings.xml
index 2abed74..3b0ee47 100644
--- a/java/res/values-hu/strings.xml
+++ b/java/res/values-hu/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Csatlakoztasson egy headsetet, ha hallani szeretné a jelszót felolvasva."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"A jelenlegi szöveg: %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Szöveg nincs megadva"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Billentyűkód: %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift be van kapcsolva (érintse meg a kikapcsoláshoz)"</string>
diff --git a/java/res/values-in/strings.xml b/java/res/values-in/strings.xml
index c73c26c..bfbd2a7 100644
--- a/java/res/values-in/strings.xml
+++ b/java/res/values-in/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Pasang headset untuk mendengar tombol sandi yang diucapkan dengan keras."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Teks saat ini adalah %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Tidak ada teks yang dimasukkan"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Kode tombol %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift hidup (ketuk untuk mematikan)"</string>
diff --git a/java/res/values-it/strings.xml b/java/res/values-it/strings.xml
index eb8e1e9..6989643 100644
--- a/java/res/values-it/strings.xml
+++ b/java/res/values-it/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Collega gli auricolari per ascoltare la pronuncia dei tasti premuti per la password."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Il testo attuale è %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Nessun testo inserito"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Codice tasto %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Maiuscolo"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Maiuscolo attivo (tocca per disattivare)"</string>
diff --git a/java/res/values-iw/strings.xml b/java/res/values-iw/strings.xml
index e71bd31..b3ee014 100644
--- a/java/res/values-iw/strings.xml
+++ b/java/res/values-iw/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"חבר אוזניות כדי לשמוע הקראה של מפתחות סיסמה."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"הטקסט הנוכחי הוא %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"לא הוזן טקסט"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"קוד מקש %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift פועל (הקש כדי להשבית)"</string>
diff --git a/java/res/values-ja/strings.xml b/java/res/values-ja/strings.xml
index 4d3f155..37f0637 100644
--- a/java/res/values-ja/strings.xml
+++ b/java/res/values-ja/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"パスワードのキーが音声出力されるのでヘッドセットを接続してください。"</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"現在のテキスト:%s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"テキストが入力されていません"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"キーコード:%d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift有効（タップして解除）"</string>
diff --git a/java/res/values-ka-rGE/strings-appname.xml b/java/res/values-ka-rGE/strings-appname.xml
new file mode 100644
index 0000000..703c66a
--- /dev/null
+++ b/java/res/values-ka-rGE/strings-appname.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/*
+**
+** Copyright 2013, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+ -->
+
+<resources xmlns:android="http://schemas.android.com/apk/res/android"
+ xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="english_ime_name" msgid="5940510615957428904">"Android-ის კლავიატურა (AOSP)"</string>
+ <string name="spell_checker_service_name" msgid="1254221805440242662">"Android-ის მართლწერის შემმოწმებელი (AOSP)"</string>
+ <string name="english_ime_settings" msgid="5760361067176802794">"Android-ის კლავიატურის პარამეტრები (AOSP)"</string>
+ <string name="android_spell_checker_settings" msgid="6123949487832861885">"Android-ის მართლწერის შემმოწმებლის პარამეტრები (AOSP)"</string>
+</resources>
diff --git a/java/res/values-ka-rGE/strings.xml b/java/res/values-ka-rGE/strings.xml
index 043c14c..193f511 100644
--- a/java/res/values-ka-rGE/strings.xml
+++ b/java/res/values-ka-rGE/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"შეაერთეთ ყურსაცვამი, რათა მოისმინოთ აკრეფილი პაროლის კლავიშების სახელები."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"მიმდინარე ტექსტი არის %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"ტექსტი არ შეყვანილა"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"კლავიატურის კოდი %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift ჩართულია (შეეხეთ გამოსართავად)"</string>
diff --git a/java/res/values-kk/strings-appname.xml b/java/res/values-kk/strings-appname.xml
new file mode 100644
index 0000000..1e201fa
--- /dev/null
+++ b/java/res/values-kk/strings-appname.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/*
+**
+** Copyright 2013, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+ -->
+
+<resources xmlns:android="http://schemas.android.com/apk/res/android"
+ xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="english_ime_name" msgid="5940510615957428904">"Android пернетақтасы (AOSP)"</string>
+ <string name="spell_checker_service_name" msgid="1254221805440242662">"Android емлені тексеру құралы (AOSP)"</string>
+ <string name="english_ime_settings" msgid="5760361067176802794">"Android пернетақтасының параметрлері (AOSP)"</string>
+ <string name="android_spell_checker_settings" msgid="6123949487832861885">"Android емлені тексеру құралының параметрлері (AOSP)"</string>
+</resources>
diff --git a/java/res/values-ko/strings.xml b/java/res/values-ko/strings.xml
index a74b4f1..1497812 100644
--- a/java/res/values-ko/strings.xml
+++ b/java/res/values-ko/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"비밀번호 키를 음성으로 들으려면 헤드셋을 연결하세요."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"입력한 텍스트: %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"입력한 텍스트 없음"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"키 코드 %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"시프트 키"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift 사용(사용하지 않으려면 탭하세요.)"</string>
diff --git a/java/res/values-land/dimens.xml b/java/res/values-land/dimens.xml
index 42a746b..b874d48 100644
--- a/java/res/values-land/dimens.xml
+++ b/java/res/values-land/dimens.xml
@@ -77,7 +77,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">10%p</fraction>
<fraction name="emoji_keyboard_row_height">50%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">60%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">54%p</fraction>
<integer name="emoji_keyboard_max_key_count">20</integer>
</resources>
diff --git a/java/res/values-lt/strings.xml b/java/res/values-lt/strings.xml
index 6110858..88328db 100644
--- a/java/res/values-lt/strings.xml
+++ b/java/res/values-lt/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Prijunkite ausines, kad išgirstumėte sakomus slaptažodžio klavišus."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Dabartinis tekstas yra %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Nėra įvesto teksto"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Klavišo kodas %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Antrojo lygio klavišas"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Įjungtas antrasis lygis (palieskite, kad išjungtumėte)"</string>
diff --git a/java/res/values-lv/strings.xml b/java/res/values-lv/strings.xml
index ecd4e51..069c487 100644
--- a/java/res/values-lv/strings.xml
+++ b/java/res/values-lv/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Pievienojiet austiņas, lai dzirdētu paroles rakstzīmes."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Pašreizējais teksts ir %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Nav ievadīts teksts"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Taustiņu kods %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Pārslēgšanas taustiņš"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Pārslēgšanas taustiņš iespējots (pieskarieties, lai atspējotu)"</string>
diff --git a/java/res/values-mn-rMN/strings-appname.xml b/java/res/values-mn-rMN/strings-appname.xml
new file mode 100644
index 0000000..6c27e3d
--- /dev/null
+++ b/java/res/values-mn-rMN/strings-appname.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/*
+**
+** Copyright 2013, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+ -->
+
+<resources xmlns:android="http://schemas.android.com/apk/res/android"
+ xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="english_ime_name" msgid="5940510615957428904">"Андройд Гар (AOSP)"</string>
+ <string name="spell_checker_service_name" msgid="1254221805440242662">"Андройд Алдаа Шалгагч (AOSP)"</string>
+ <string name="english_ime_settings" msgid="5760361067176802794">"Андройд Гарын Тохиргоо (AOSP)"</string>
+ <string name="android_spell_checker_settings" msgid="6123949487832861885">"Андройд Алдаа Шалгагчийн Тохиргоо (AOSP)"</string>
+</resources>
diff --git a/java/res/values-mn-rMN/strings.xml b/java/res/values-mn-rMN/strings.xml
index 92c2bef..3b039f1 100644
--- a/java/res/values-mn-rMN/strings.xml
+++ b/java/res/values-mn-rMN/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Нууц үгний товчнуудыг чангаар уншихыг сонсохын тулд чихэвчээ залгана уу."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Одоогийн текст %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Текст оруулаагүй"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Товчийн код %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Сэлгэх"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Сэлгэхийг идэвхжүүлсэн (товшиж идэвхгүйжүүлнэ үү)"</string>
diff --git a/java/res/values-ms-rMY/strings-appname.xml b/java/res/values-ms-rMY/strings-appname.xml
new file mode 100644
index 0000000..76d1d29
--- /dev/null
+++ b/java/res/values-ms-rMY/strings-appname.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/*
+**
+** Copyright 2013, The Android Open Source Project
+**
+** Licensed under the Apache License, Version 2.0 (the "License");
+** you may not use this file except in compliance with the License.
+** You may obtain a copy of the License at
+**
+** http://www.apache.org/licenses/LICENSE-2.0
+**
+** Unless required by applicable law or agreed to in writing, software
+** distributed under the License is distributed on an "AS IS" BASIS,
+** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+** See the License for the specific language governing permissions and
+** limitations under the License.
+*/
+ -->
+
+<resources xmlns:android="http://schemas.android.com/apk/res/android"
+ xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="english_ime_name" msgid="5940510615957428904">"Papan Kekunci Android (AOSP)"</string>
+ <string name="spell_checker_service_name" msgid="1254221805440242662">"Penyemak Ejaan Android (AOSP)"</string>
+ <string name="english_ime_settings" msgid="5760361067176802794">"Tetapan Papan Kekunci Android (AOSP)"</string>
+ <string name="android_spell_checker_settings" msgid="6123949487832861885">"Tetapan Penyemak Ejaan Android (AOSP)"</string>
+</resources>
diff --git a/java/res/values-ms-rMY/strings.xml b/java/res/values-ms-rMY/strings.xml
index 49efdf1..ee241d6 100644
--- a/java/res/values-ms-rMY/strings.xml
+++ b/java/res/values-ms-rMY/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Pasangkan set kepala untuk mendengar kekunci kata laluan disebut dengan kuat."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Teks semasa adalah %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Tiada teks dimasukkan"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Kod kunci %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Kunci anjak dihidupkan (ketik untuk melumpuhkan)"</string>
diff --git a/java/res/values-nb/strings.xml b/java/res/values-nb/strings.xml
index 9b30ea3..ae43d10 100644
--- a/java/res/values-nb/strings.xml
+++ b/java/res/values-nb/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Koble til hodetelefoner for å høre opplesing av bokstavene i passordet."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Gjeldende tekst er %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Ingen tekst er skrevet inn"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Tastaturkode %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift er på (trykk for å deaktivere)"</string>
diff --git a/java/res/values-nl/strings.xml b/java/res/values-nl/strings.xml
index c61b090..de20da4 100644
--- a/java/res/values-nl/strings.xml
+++ b/java/res/values-nl/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Sluit een headset aan om wachtwoordtoetsen hardop te laten voorlezen."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Huidige tekst is %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Geen tekst ingevoerd"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Toetscode %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift aan (tik om uit te schakelen)"</string>
diff --git a/java/res/values-pl/strings.xml b/java/res/values-pl/strings.xml
index 31eace5..3908375 100644
--- a/java/res/values-pl/strings.xml
+++ b/java/res/values-pl/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Podłącz zestaw słuchawkowy, aby usłyszeć znaki hasła wypowiadane na głos."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Aktualny tekst: %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Nie wprowadzono tekstu"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Kod klawisza: %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift włączony (kliknij, by wyłączyć)"</string>
diff --git a/java/res/values-pt-rPT/strings.xml b/java/res/values-pt-rPT/strings.xml
index cd89a77..4c81479 100644
--- a/java/res/values-pt-rPT/strings.xml
+++ b/java/res/values-pt-rPT/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Ligar auscultadores com microfone integrado para ouvir as teclas da palavra-passe."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"O texto atual é %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Nenhum texto digitado"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Código da tecla %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift ativado (tocar para desativar)"</string>
diff --git a/java/res/values-pt/strings.xml b/java/res/values-pt/strings.xml
index 820f577..ef533ae 100644
--- a/java/res/values-pt/strings.xml
+++ b/java/res/values-pt/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Conecte um fone de ouvido para ouvir as chaves de senha em voz alta."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"O texto atual é %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Nenhum texto digitado"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Código de tecla %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift ativado (toque para desativar)"</string>
diff --git a/java/res/values-rm/strings.xml b/java/res/values-rm/strings.xml
index 1c509ad..c68d30c 100644
--- a/java/res/values-rm/strings.xml
+++ b/java/res/values-rm/strings.xml
@@ -139,6 +139,10 @@
<skip />
<!-- no translation found for spoken_no_text_entered (7479685225597344496) -->
<skip />
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<!-- no translation found for spoken_description_unknown (3197434010402179157) -->
<skip />
<!-- no translation found for spoken_description_shift (244197883292549308) -->
diff --git a/java/res/values-ro/strings.xml b/java/res/values-ro/strings.xml
index 3caff13..930b68b 100644
--- a/java/res/values-ro/strings.xml
+++ b/java/res/values-ro/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Conectaţi un set căşti-microfon pentru a auzi tastele apăsate când introduceţi parola."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Textul curent este %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Nu a fost introdus text"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Tasta cu codul %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Tasta Shift este activată (apăsaţi pentru a o dezactiva)"</string>
diff --git a/java/res/values-ru/strings.xml b/java/res/values-ru/strings.xml
index affe88f..8bfb011 100644
--- a/java/res/values-ru/strings.xml
+++ b/java/res/values-ru/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Подключите гарнитуру, чтобы услышать пароль."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Введенный текст: %s."</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Текст не введен"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Код клавиши:%d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Клавиша верхнего регистра"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Верхний регистр включен (нажмите, чтобы отключить)"</string>
diff --git a/java/res/values-sk/strings.xml b/java/res/values-sk/strings.xml
index 85d64c6..5c25f28 100644
--- a/java/res/values-sk/strings.xml
+++ b/java/res/values-sk/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Ak si chcete pri zadávaní hesla vypočuť nahlas vyslovené klávesy, pripojte náhlavnú súpravu."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Aktuálny text je %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Nie je zadaný žiadny text"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Kód klávesu %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Kláves Shift je zapnutý (zakážete ho klepnutím)"</string>
diff --git a/java/res/values-sl/strings.xml b/java/res/values-sl/strings.xml
index 7648d73..dde01dd 100644
--- a/java/res/values-sl/strings.xml
+++ b/java/res/values-sl/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Priključite slušalke, če želite slišati izgovorjene tipke gesla."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Trenutno besedilo je %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Ni vnesenega besedila"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Koda tipke %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift je vklopljen (dotaknite se, da onemogočite)"</string>
diff --git a/java/res/values-sr/strings.xml b/java/res/values-sr/strings.xml
index db06bce..e7f6904 100644
--- a/java/res/values-sr/strings.xml
+++ b/java/res/values-sr/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Укључите слушалице да бисте чули наглас изговорене тастере за лозинку."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Тренутни текст је %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Текст није унет"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Кôд тастера %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift је укључен (додирните да бисте га онемогућили)"</string>
diff --git a/java/res/values-sv/strings.xml b/java/res/values-sv/strings.xml
index c663516..0fcca48 100644
--- a/java/res/values-sv/strings.xml
+++ b/java/res/values-sv/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Anslut hörlurar om du vill att lösenordet ska läsas upp."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Nuvarande text är %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Ingen text har angetts"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Nyckelkod %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Skift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Skift på (knacka lätt för att inaktivera)"</string>
diff --git a/java/res/values-sw/strings.xml b/java/res/values-sw/strings.xml
index 188bbca..fbf868f 100644
--- a/java/res/values-sw/strings.xml
+++ b/java/res/values-sw/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Chomeka plagi ya kifaa cha kichwa cha kusikiza ili kusikiliza msimbo wa nenosiri inayozungumwa kwa sauti ya juu."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Maandishi ya sasa ni %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Hakuna maandishi yaliyoingizwa"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Msimbo wa kitufe %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Badilisha"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift imewashwa (gonga ili kulemaza)"</string>
diff --git a/java/res/values-sw600dp-land/dimens.xml b/java/res/values-sw600dp-land/dimens.xml
index 730b7d8..d067265 100644
--- a/java/res/values-sw600dp-land/dimens.xml
+++ b/java/res/values-sw600dp-land/dimens.xml
@@ -66,7 +66,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">10%p</fraction>
<fraction name="emoji_keyboard_row_height">33%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">95%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">85%p</fraction>
<integer name="emoji_keyboard_max_key_count">30</integer>
</resources>
diff --git a/java/res/values-sw600dp/dimens.xml b/java/res/values-sw600dp/dimens.xml
index 2bcf2fa..591355d 100644
--- a/java/res/values-sw600dp/dimens.xml
+++ b/java/res/values-sw600dp/dimens.xml
@@ -92,7 +92,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">12.5%p</fraction>
<fraction name="emoji_keyboard_row_height">33%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">85%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">76%p</fraction>
<integer name="emoji_keyboard_max_key_count">24</integer>
</resources>
diff --git a/java/res/values-sw768dp-land/dimens.xml b/java/res/values-sw768dp-land/dimens.xml
index 1e2e1c6..664630b 100644
--- a/java/res/values-sw768dp-land/dimens.xml
+++ b/java/res/values-sw768dp-land/dimens.xml
@@ -67,7 +67,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">7.69%p</fraction>
<fraction name="emoji_keyboard_row_height">33%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">75%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">68%p</fraction>
<integer name="emoji_keyboard_max_key_count">39</integer>
</resources>
diff --git a/java/res/values-sw768dp/dimens.xml b/java/res/values-sw768dp/dimens.xml
index f62a536..1fd933a 100644
--- a/java/res/values-sw768dp/dimens.xml
+++ b/java/res/values-sw768dp/dimens.xml
@@ -92,7 +92,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">10%p</fraction>
<fraction name="emoji_keyboard_row_height">33%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">85%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">76%p</fraction>
<integer name="emoji_keyboard_max_key_count">30</integer>
</resources>
diff --git a/java/res/values-th/strings.xml b/java/res/values-th/strings.xml
index 632f39a..f84b32c 100644
--- a/java/res/values-th/strings.xml
+++ b/java/res/values-th/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"เสียบชุดหูฟังเพื่อฟังเสียงเมื่อพิมพ์รหัสผ่าน"</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"ข้อความปัจจุบันคือ %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"ไม่มีข้อความ"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"รหัสคีย์ %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift เปิดอยู่ (แตะเพื่อปิดใช้งาน)"</string>
diff --git a/java/res/values-tl/strings.xml b/java/res/values-tl/strings.xml
index 3ec962b..af1c68e 100644
--- a/java/res/values-tl/strings.xml
+++ b/java/res/values-tl/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Mag-plug in ng headset upang marinig ang mga password key na binabanggit nang malakas."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Ang kasalukuyang teksto ay %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Walang tekstong inilagay"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Code ng key %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Naka-on ang shift (i-tap upang huwag paganahin)"</string>
diff --git a/java/res/values-tr/strings.xml b/java/res/values-tr/strings.xml
index 4d0a617..4c8e1a8 100644
--- a/java/res/values-tr/strings.xml
+++ b/java/res/values-tr/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Şifre tuşlarının sesli okunmasını dinlemek için mikrofonlu kulaklık takın."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Mevcut metin: %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Hiç metin girilmedi"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Tuş kodu: %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Üst Karakter"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Üst karakter açık (devre dışı bırakmak için hafifçe vurun)"</string>
diff --git a/java/res/values-uk/strings.xml b/java/res/values-uk/strings.xml
index df62f32..287bcb3 100644
--- a/java/res/values-uk/strings.xml
+++ b/java/res/values-uk/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Підключіть гарнітуру, щоб прослухати відтворені вголос символи пароля."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Поточний текст – %s."</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Текст не введено"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Код клавіші – %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Клавіша Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift увімкнено (швидко торкніться, щоб вимкнути)"</string>
diff --git a/java/res/values-vi/strings.xml b/java/res/values-vi/strings.xml
index e08d1c6..8764df5 100644
--- a/java/res/values-vi/strings.xml
+++ b/java/res/values-vi/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Cắm tai nghe để nghe mật khẩu."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Ký tự hiện tại là %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Không có ký tự nào được nhập"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Mã phím %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift đang bật (bấm để tắt)"</string>
diff --git a/java/res/values-zh-rCN/strings.xml b/java/res/values-zh-rCN/strings.xml
index 5980cbf..f7ceff4 100644
--- a/java/res/values-zh-rCN/strings.xml
+++ b/java/res/values-zh-rCN/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"需要插入耳机才能听到密码的按键声。"</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"当前文本为%s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"未输入文字"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"键码为 %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift 模式已启用（点按即可停用）"</string>
diff --git a/java/res/values-zh-rTW/strings.xml b/java/res/values-zh-rTW/strings.xml
index b14a01a..b773ad2 100644
--- a/java/res/values-zh-rTW/strings.xml
+++ b/java/res/values-zh-rTW/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"連接耳機即可聽取系統朗讀密碼按鍵。"</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"目前文字為 %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"未輸入文字"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"按鍵代碼 %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift 鍵"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift 鍵已開啟 (輕按即可停用)"</string>
diff --git a/java/res/values-zu/strings.xml b/java/res/values-zu/strings.xml
index 9ed88a0..8f722cc 100644
--- a/java/res/values-zu/strings.xml
+++ b/java/res/values-zu/strings.xml
@@ -84,6 +84,10 @@
<string name="spoken_use_headphones" msgid="896961781287283493">"Plaka ku-headset ukuze uzwe okhiye bephasiwedi ezindlebeni zakho bezwakala kakhulu."</string>
<string name="spoken_current_text_is" msgid="2485723011272583845">"Umbhalo wamanje ngu %s"</string>
<string name="spoken_no_text_entered" msgid="7479685225597344496">"Awukho umbhalo ofakiwe"</string>
+ <!-- no translation found for spoken_auto_correct (5381764628886369268) -->
+ <skip />
+ <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) -->
+ <skip />
<string name="spoken_description_unknown" msgid="3197434010402179157">"Ikhodi yokhiye %d"</string>
<string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string>
<string name="spoken_description_shift_shifted" msgid="1681877323344195035">"U-Shift uvuliwe (thepha ukuwuvimbela)"</string>
diff --git a/java/res/values/dimens.xml b/java/res/values/dimens.xml
index 4e3b2f5..2d626db 100644
--- a/java/res/values/dimens.xml
+++ b/java/res/values/dimens.xml
@@ -118,7 +118,7 @@
<!-- Emoji keyboard -->
<fraction name="emoji_keyboard_key_width">14.2857%p</fraction>
<fraction name="emoji_keyboard_row_height">33%p</fraction>
- <fraction name="emoji_keyboard_key_letter_size">90%p</fraction>
+ <fraction name="emoji_keyboard_key_letter_size">81%p</fraction>
<integer name="emoji_keyboard_max_key_count">21</integer>
<dimen name="emoji_category_page_id_height">3dp</dimen>
diff --git a/java/res/values/strings.xml b/java/res/values/strings.xml
index 69da1e8..06f4b47 100644
--- a/java/res/values/strings.xml
+++ b/java/res/values/strings.xml
@@ -174,6 +174,11 @@
<!-- Spoken description when there is no text entered -->
<string name="spoken_no_text_entered">No text entered</string>
+ <!-- Spoken description to let the user know what auto-correction will be performed when a key is pressed. -->
+ <string name="spoken_auto_correct"><xliff:g id="key" example="Space">%1$s</xliff:g> corrects <xliff:g id="original">%2$s</xliff:g> to <xliff:g id="corrected">%3$s</xliff:g></string>
+ <!-- Spoken description used during obscured (e.g. password) entry to let the user know that auto-correction will be performed when a key is pressed. -->
+ <string name="spoken_auto_correct_obscured"><xliff:g id="key" example="Space">%1$s</xliff:g> has auto-correction</string>
+
<!-- Spoken description for unknown keyboard keys. -->
<string name="spoken_description_unknown">Key code %d</string>
<!-- Spoken description for the "Shift" keyboard key when "Shift" is off. -->
diff --git a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java
index 7639432..c628c5b 100644
--- a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java
+++ b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java
@@ -35,6 +35,8 @@
import com.android.inputmethod.keyboard.Key;
import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.keyboard.KeyboardView;
+import com.android.inputmethod.latin.settings.Settings;
+import com.android.inputmethod.latin.settings.SettingsValues;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.CoordinateUtils;
@@ -285,9 +287,15 @@
private String getKeyDescription(final Key key) {
final EditorInfo editorInfo = mInputMethodService.getCurrentInputEditorInfo();
final boolean shouldObscure = mAccessibilityUtils.shouldObscureInput(editorInfo);
- final String keyDescription = mKeyCodeDescriptionMapper.getDescriptionForKey(
+ final SettingsValues currentSettings = Settings.getInstance().getCurrent();
+ final String keyCodeDescription = mKeyCodeDescriptionMapper.getDescriptionForKey(
mKeyboardView.getContext(), mKeyboardView.getKeyboard(), key, shouldObscure);
- return keyDescription;
+ if (currentSettings.isWordSeparator(key.getCode())) {
+ return mAccessibilityUtils.getAutoCorrectionDescription(
+ keyCodeDescription, shouldObscure);
+ } else {
+ return keyCodeDescription;
+ }
}
/**
diff --git a/java/src/com/android/inputmethod/accessibility/AccessibilityUtils.java b/java/src/com/android/inputmethod/accessibility/AccessibilityUtils.java
index 8929dc7..10fb9fe 100644
--- a/java/src/com/android/inputmethod/accessibility/AccessibilityUtils.java
+++ b/java/src/com/android/inputmethod/accessibility/AccessibilityUtils.java
@@ -23,6 +23,7 @@
import android.os.SystemClock;
import android.provider.Settings;
import android.support.v4.view.accessibility.AccessibilityEventCompat;
+import android.text.TextUtils;
import android.util.Log;
import android.view.MotionEvent;
import android.view.View;
@@ -34,6 +35,7 @@
import com.android.inputmethod.compat.SettingsSecureCompatUtils;
import com.android.inputmethod.latin.R;
+import com.android.inputmethod.latin.SuggestedWords;
import com.android.inputmethod.latin.utils.InputTypeUtils;
public final class AccessibilityUtils {
@@ -48,6 +50,12 @@
private AccessibilityManager mAccessibilityManager;
private AudioManager mAudioManager;
+ /** The most recent auto-correction. */
+ private String mAutoCorrectionWord;
+
+ /** The most recent typed word for auto-correction. */
+ private String mTypedWord;
+
/*
* Setting this constant to {@code false} will disable all keyboard
* accessibility code, regardless of whether Accessibility is turned on in
@@ -142,6 +150,51 @@
}
/**
+ * Sets the current auto-correction word and typed word. These may be used
+ * to provide the user with a spoken description of what auto-correction
+ * will occur when a key is typed.
+ *
+ * @param suggestedWords the list of suggested auto-correction words
+ * @param typedWord the currently typed word
+ */
+ public void setAutoCorrection(final SuggestedWords suggestedWords, final String typedWord) {
+ if (suggestedWords != null && suggestedWords.mWillAutoCorrect) {
+ mAutoCorrectionWord = suggestedWords.getWord(SuggestedWords.INDEX_OF_AUTO_CORRECTION);
+ mTypedWord = typedWord;
+ } else {
+ mAutoCorrectionWord = null;
+ mTypedWord = null;
+ }
+ }
+
+ /**
+ * Obtains a description for an auto-correction key, taking into account the
+ * currently typed word and auto-correction.
+ *
+ * @param keyCodeDescription spoken description of the key that will insert
+ * an auto-correction
+ * @param shouldObscure whether the key should be obscured
+ * @return a description including a description of the auto-correction, if
+ * needed
+ */
+ public String getAutoCorrectionDescription(
+ final String keyCodeDescription, final boolean shouldObscure) {
+ if (!TextUtils.isEmpty(mAutoCorrectionWord)) {
+ if (!TextUtils.equals(mAutoCorrectionWord, mTypedWord)) {
+ if (shouldObscure) {
+ // This should never happen, but just in case...
+ return mContext.getString(R.string.spoken_auto_correct_obscured,
+ keyCodeDescription);
+ }
+ return mContext.getString(R.string.spoken_auto_correct, keyCodeDescription,
+ mTypedWord, mAutoCorrectionWord);
+ }
+ }
+
+ return keyCodeDescription;
+ }
+
+ /**
* Sends the specified text to the {@link AccessibilityManager} to be
* spoken.
*
diff --git a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
index 4e61eda..eb48d01 100644
--- a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
+++ b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java
@@ -153,7 +153,9 @@
mCategoryNameToIdMap.put(sCategoryName[i], i);
}
addShownCategoryId(CATEGORY_ID_RECENTS);
- if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
+ if (Build.VERSION.SDK_INT > Build.VERSION_CODES.JELLY_BEAN_MR2
+ || android.os.Build.VERSION.CODENAME.equalsIgnoreCase("KeyLimePie")
+ || android.os.Build.VERSION.CODENAME.equalsIgnoreCase("KitKat")) {
addShownCategoryId(CATEGORY_ID_PEOPLE);
addShownCategoryId(CATEGORY_ID_OBJECTS);
addShownCategoryId(CATEGORY_ID_NATURE);
@@ -716,12 +718,14 @@
@Override
public void run() {
+ int repeatCount = 1;
int timeCount = 0;
while (timeCount < MAX_REPEAT_COUNT_TIME && !mAborted) {
if (timeCount > mKeyRepeatStartTimeout) {
- pressDelete();
+ pressDelete(repeatCount);
}
timeCount += mKeyRepeatInterval;
+ ++repeatCount;
try {
Thread.sleep(mKeyRepeatInterval);
} catch (InterruptedException e) {
@@ -734,9 +738,9 @@
}
}
- public void pressDelete() {
+ public void pressDelete(int repeatCount) {
mKeyboardActionListener.onPressKey(
- Constants.CODE_DELETE, 0 /* repeatCount */, true /* isSinglePointer */);
+ Constants.CODE_DELETE, repeatCount, true /* isSinglePointer */);
mKeyboardActionListener.onCodeInput(
Constants.CODE_DELETE, NOT_A_COORDINATE, NOT_A_COORDINATE);
mKeyboardActionListener.onReleaseKey(
@@ -752,7 +756,7 @@
switch(event.getAction()) {
case MotionEvent.ACTION_DOWN:
v.setBackgroundColor(mDeleteKeyPressedBackgroundColor);
- pressDelete();
+ pressDelete(0 /* repeatCount */);
startRepeat();
return true;
case MotionEvent.ACTION_UP:
diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
index cc1ffd1..74edd87 100644
--- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
+++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
@@ -315,7 +315,7 @@
}
public boolean isShowingEmojiKeyboard() {
- return mEmojiKeyboardView.getVisibility() == View.VISIBLE;
+ return mEmojiKeyboardView != null && mEmojiKeyboardView.getVisibility() == View.VISIBLE;
}
public boolean isShowingMoreKeysPanel() {
diff --git a/java/src/com/android/inputmethod/keyboard/PointerTracker.java b/java/src/com/android/inputmethod/keyboard/PointerTracker.java
index d4d0d87..ee4ac95 100644
--- a/java/src/com/android/inputmethod/keyboard/PointerTracker.java
+++ b/java/src/com/android/inputmethod/keyboard/PointerTracker.java
@@ -346,6 +346,8 @@
// true if this pointer is in a sliding key input from a modifier key,
// so that further modifier keys should be ignored.
boolean mIsInSlidingKeyInputFromModifier;
+ // if not a NOT_A_CODE, the key of this code is repeating
+ private int mCurrentRepeatingKeyCode = Constants.NOT_A_CODE;
// true if a sliding key input is allowed.
private boolean mIsAllowedSlidingKeyInput;
@@ -937,9 +939,10 @@
if (!sShouldHandleGesture) {
return;
}
- // A gesture should start only from a non-modifier key.
+ // A gesture should start only from a non-modifier key. Note that the gesture detection is
+ // disabled when the key is repeating.
mIsDetectingGesture = (mKeyboard != null) && mKeyboard.mId.isAlphabetKeyboard()
- && key != null && !key.isModifier() && !key.isRepeatable();
+ && key != null && !key.isModifier();
if (mIsDetectingGesture) {
if (getActivePointerTrackerCount() == 1) {
sGestureFirstDownTime = eventTime;
@@ -1247,6 +1250,8 @@
mIsDetectingGesture = false;
final Key currentKey = mCurrentKey;
mCurrentKey = null;
+ final int currentRepeatingKeyCode = mCurrentRepeatingKeyCode;
+ mCurrentRepeatingKeyCode = Constants.NOT_A_CODE;
// Release the last pressed key.
setReleasedKeyGraphics(currentKey);
@@ -1272,8 +1277,8 @@
if (mIsTrackingForActionDisabled) {
return;
}
- if (currentKey != null && currentKey.isRepeatable() && !isInSlidingKeyInput) {
- // Repeatable key has been registered in {@link #onDownEventInternal(int,int,long)}.
+ if (currentKey != null && currentKey.isRepeatable()
+ && (currentKey.getCode() == currentRepeatingKeyCode) && !isInSlidingKeyInput) {
return;
}
detectAndSendKey(currentKey, mKeyX, mKeyY, eventTime);
@@ -1412,7 +1417,6 @@
if (!key.isRepeatable()) return;
// Don't start key repeat when we are in sliding input mode.
if (mIsInSlidingKeyInput) return;
- detectAndSendKey(key, key.getX(), key.getY(), SystemClock.uptimeMillis());
final int startRepeatCount = 1;
mTimerProxy.startKeyRepeatTimer(this, startRepeatCount, sParams.mKeyRepeatStartTimeout);
}
@@ -1420,8 +1424,11 @@
public void onKeyRepeat(final int code, final int repeatCount) {
final Key key = getKey();
if (key == null || key.getCode() != code) {
+ mCurrentRepeatingKeyCode = Constants.NOT_A_CODE;
return;
}
+ mCurrentRepeatingKeyCode = code;
+ mIsDetectingGesture = false;
final int nextRepeatCount = repeatCount + 1;
mTimerProxy.startKeyRepeatTimer(this, nextRepeatCount, sParams.mKeyRepeatInterval);
callListenerOnPressAndCheckKeyboardLayoutChange(key, repeatCount);
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
index b3491d8..9f9fdaa 100644
--- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
+++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java
@@ -585,7 +585,7 @@
}
}
- private static boolean isSpaceCharacter(final int c) {
+ private static boolean isSpaceOrEnter(final int c) {
return c == Constants.CODE_SPACE || c == Constants.CODE_ENTER;
}
@@ -614,7 +614,12 @@
}
break;
case SWITCH_STATE_SYMBOL_BEGIN:
- if (!isSpaceCharacter(code) && (Constants.isLetterCode(code)
+ if (mIsEmojiMode) {
+ // When in the Emoji keyboard, we don't want to switch back to the main layout even
+ // after the user hits an emoji letter followed by an enter or a space.
+ break;
+ }
+ if (!isSpaceOrEnter(code) && (Constants.isLetterCode(code)
|| code == Constants.CODE_OUTPUT_TEXT)) {
mSwitchState = SWITCH_STATE_SYMBOL;
}
@@ -622,7 +627,7 @@
case SWITCH_STATE_SYMBOL:
// Switch back to alpha keyboard mode if user types one or more non-space/enter
// characters followed by a space/enter.
- if (isSpaceCharacter(code)) {
+ if (isSpaceOrEnter(code)) {
toggleAlphabetAndSymbols();
mPrevSymbolsKeyboardWasShifted = false;
}
diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
index 845a9b9..4a0ce37 100644
--- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java
@@ -25,6 +25,7 @@
import java.io.File;
import java.io.IOException;
+import java.util.Map;
// TODO: Quit extending Dictionary after implementing dynamic binary dictionary.
abstract public class AbstractDictionaryWriter extends Dictionary {
@@ -50,16 +51,16 @@
abstract public void removeBigramWords(final String word0, final String word1);
- abstract protected void writeDictionary(final DictEncoder dictEncoder)
- throws IOException, UnsupportedFormatException;
+ abstract protected void writeDictionary(final DictEncoder dictEncoder,
+ final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException;
- public void write(final String fileName) {
+ public void write(final String fileName, final Map<String, String> attributeMap) {
final String tempFileName = fileName + ".temp";
final File file = new File(mContext.getFilesDir(), fileName);
final File tempFile = new File(mContext.getFilesDir(), tempFileName);
try {
final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile);
- writeDictionary(dictEncoder);
+ writeDictionary(dictEncoder, attributeMap);
tempFile.renameTo(file);
} catch (IOException e) {
Log.e(TAG, "IO exception while writing file", e);
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index 61ccfcf..29c6c04 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -31,10 +31,12 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Locale;
+import java.util.Map;
/**
* Implements a static, compacted, binary dictionary of standard words.
*/
+// TODO: All methods which should be locked need to have a suffix "Locked".
public final class BinaryDictionary extends Dictionary {
private static final String TAG = BinaryDictionary.class.getSimpleName();
@@ -46,6 +48,11 @@
// TODO: Remove this heuristic.
private static final int SPACE_COUNT_FOR_AUTO_COMMIT = 3;
+ @UsedForTesting
+ public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
+ @UsedForTesting
+ public static final String BIGRAM_COUNT_QUERY = "BIGRAM_COUNT";
+
private long mNativeDict;
private final Locale mLocale;
private final long mDictSize;
@@ -103,10 +110,12 @@
JniUtils.loadNativeLibrary();
}
+ private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
+ String[] attributeKeyStringArray, String[] attributeValueStringArray);
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
boolean isUpdatable);
private static native void flushNative(long dict, String filePath);
- private static native boolean needsToRunGCNative(long dict);
+ private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC);
private static native void flushWithGCNative(long dict, String filePath);
private static native void closeNative(long dict);
private static native int getProbabilityNative(long dict, int[] word);
@@ -125,6 +134,21 @@
private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1);
private static native int calculateProbabilityNative(long dict, int unigramProbability,
int bigramProbability);
+ private static native String getPropertyNative(long dict, String query);
+
+ @UsedForTesting
+ public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
+ final Map<String, String> attributeMap) {
+ final String[] keyArray = new String[attributeMap.size()];
+ final String[] valueArray = new String[attributeMap.size()];
+ int index = 0;
+ for (final String key : attributeMap.keySet()) {
+ keyArray[index] = key;
+ valueArray[index] = attributeMap.get(key);
+ index++;
+ }
+ return createEmptyDictFileNative(filePath, dictVersion, keyArray, valueArray);
+ }
// TODO: Move native dict into session
private final void loadDictionary(final String path, final long startOffset,
@@ -246,7 +270,7 @@
}
private void runGCIfRequired() {
- if (needsToRunGCNative(mNativeDict)) {
+ if (needsToRunGC(true /* mindsBlockByGC */)) {
flushWithGC();
}
}
@@ -283,27 +307,34 @@
removeBigramWordsNative(mNativeDict, codePoints0, codePoints1);
}
- public void flush() {
- if (!isValidDictionary()) return;
- flushNative(mNativeDict, mDictFilePath);
- closeNative(mNativeDict);
+ private void reopen() {
+ close();
final File dictFile = new File(mDictFilePath);
mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
dictFile.length(), true /* isUpdatable */);
}
+ public void flush() {
+ if (!isValidDictionary()) return;
+ flushNative(mNativeDict, mDictFilePath);
+ reopen();
+ }
+
public void flushWithGC() {
if (!isValidDictionary()) return;
flushWithGCNative(mNativeDict, mDictFilePath);
- closeNative(mNativeDict);
- final File dictFile = new File(mDictFilePath);
- mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
- dictFile.length(), true /* isUpdatable */);
+ reopen();
}
- public boolean needsToRunGC() {
+ /**
+ * Checks whether GC is needed to run or not.
+ * @param mindsBlockByGC Whether to mind operations blocked by GC. We don't need to care about
+ * the blocking in some situations such as in idle time or just before closing.
+ * @return whether GC is needed to run or not.
+ */
+ public boolean needsToRunGC(final boolean mindsBlockByGC) {
if (!isValidDictionary()) return false;
- return needsToRunGCNative(mNativeDict);
+ return needsToRunGCNative(mNativeDict, mindsBlockByGC);
}
@UsedForTesting
@@ -312,6 +343,12 @@
return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability);
}
+ @UsedForTesting
+ public String getPropertyForTests(String query) {
+ if (!isValidDictionary()) return "";
+ return getPropertyNative(mNativeDict, query);
+ }
+
@Override
public boolean shouldAutoCommit(final SuggestedWordInfo candidate) {
// TODO: actually use the confidence rather than use this completely broken heuristic
@@ -338,21 +375,23 @@
traverseSession.close();
}
}
+ mDicTraverseSessions.clear();
}
- closeInternal();
+ closeInternalLocked();
}
- private synchronized void closeInternal() {
+ private synchronized void closeInternalLocked() {
if (mNativeDict != 0) {
closeNative(mNativeDict);
mNativeDict = 0;
}
}
+ // TODO: Manage BinaryDictionary instances without using WeakReference or something.
@Override
protected void finalize() throws Throwable {
try {
- closeInternal();
+ closeInternalLocked();
} finally {
super.finalize();
}
diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
index 5a453dd..84abfa6 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java
@@ -31,6 +31,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Map;
/**
* An in memory dictionary for memorizing entries and writing a binary dictionary.
@@ -84,8 +85,11 @@
}
@Override
- protected void writeDictionary(final DictEncoder dictEncoder)
- throws IOException, UnsupportedFormatException {
+ protected void writeDictionary(final DictEncoder dictEncoder,
+ final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException {
+ for (final Map.Entry<String, String> entry : attributeMap.entrySet()) {
+ mFusionDictionary.addOptionAttribute(entry.getKey(), entry.getValue());
+ }
dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS);
}
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index 7125202..2d1ca51 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -22,12 +22,7 @@
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
-import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
-import com.android.inputmethod.latin.makedict.FusionDictionary;
-import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
-import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder;
@@ -35,9 +30,9 @@
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
import java.io.File;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -69,8 +64,10 @@
*/
protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
- private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
- new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
+ private static final int DICTIONARY_FORMAT_VERSION = 3;
+
+ private static final String SUPPORTS_DYNAMIC_UPDATE =
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE;
/**
* A static map of update controllers, each of which records the time of accesses to a single
@@ -235,6 +232,14 @@
});
}
+ protected Map<String, String> getHeaderAttributeMap() {
+ HashMap<String, String> attributeMap = new HashMap<String, String>();
+ attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
+ SUPPORTS_DYNAMIC_UPDATE);
+ attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFilename);
+ return attributeMap;
+ }
+
protected void clear() {
getExecutor(mFilename).execute(new Runnable() {
@Override
@@ -242,17 +247,8 @@
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
mBinaryDictionary.close();
final File file = new File(mContext.getFilesDir(), mFilename);
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
- false, false));
- final DictEncoder dictEncoder = new Ver3DictEncoder(file);
- try {
- dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
- } catch (IOException e) {
- Log.e(TAG, "Exception in creating new dictionary file.", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Exception in creating new dictionary file.", e);
- }
+ BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
+ DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
} else {
mDictionaryWriter.clear();
}
@@ -501,31 +497,22 @@
if (needsToReloadBeforeWriting()) {
mDictionaryWriter.clear();
loadDictionaryAsync();
- mDictionaryWriter.write(mFilename);
+ mDictionaryWriter.write(mFilename, getHeaderAttributeMap());
} else {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
final File file = new File(mContext.getFilesDir(), mFilename);
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
- false, false));
- final DictEncoder dictEncoder = new Ver3DictEncoder(file);
- try {
- dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
- } catch (IOException e) {
- Log.e(TAG, "Exception in creating new dictionary file.", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Exception in creating new dictionary file.", e);
- }
+ BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
+ DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
} else {
- if (mBinaryDictionary.needsToRunGC()) {
+ if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
mBinaryDictionary.flushWithGC();
} else {
mBinaryDictionary.flush();
}
}
} else {
- mDictionaryWriter.write(mFilename);
+ mDictionaryWriter.write(mFilename, getHeaderAttributeMap());
}
}
}
@@ -674,47 +661,6 @@
}
}
- /**
- * Dynamically adds a word unigram to the dictionary for testing with blocking-lock.
- */
- @UsedForTesting
- protected void addWordDynamicallyForTests(final String word, final String shortcutTarget,
- final int frequency, final boolean isNotAWord) {
- getExecutor(mFilename).executePrioritized(new Runnable() {
- @Override
- public void run() {
- addWordDynamically(word, shortcutTarget, frequency, isNotAWord);
- }
- });
- }
-
- /**
- * Dynamically adds a word bigram in the dictionary for testing with blocking-lock.
- */
- @UsedForTesting
- protected void addBigramDynamicallyForTests(final String word0, final String word1,
- final int frequency, final boolean isValid) {
- getExecutor(mFilename).executePrioritized(new Runnable() {
- @Override
- public void run() {
- addBigramDynamically(word0, word1, frequency, isValid);
- }
- });
- }
-
- /**
- * Dynamically remove a word bigram in the dictionary for testing with blocking-lock.
- */
- @UsedForTesting
- protected void removeBigramDynamicallyForTests(final String word0, final String word1) {
- getExecutor(mFilename).executePrioritized(new Runnable() {
- @Override
- public void run() {
- removeBigramDynamically(word0, word1);
- }
- });
- }
-
// TODO: Implement native binary methods once the dynamic dictionary implementation is done.
@UsedForTesting
public boolean isInDictionaryForTests(final String word) {
diff --git a/java/src/com/android/inputmethod/latin/InputAttributes.java b/java/src/com/android/inputmethod/latin/InputAttributes.java
index 21b103e..8caf6f1 100644
--- a/java/src/com/android/inputmethod/latin/InputAttributes.java
+++ b/java/src/com/android/inputmethod/latin/InputAttributes.java
@@ -103,6 +103,10 @@
}
}
+ public boolean isTypeNull() {
+ return InputType.TYPE_NULL == mInputType;
+ }
+
public boolean isSameInputType(final EditorInfo editorInfo) {
return editorInfo.inputType == mInputType;
}
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index 270dc4c..96e16de 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -81,7 +81,7 @@
import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister;
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary;
-import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
+import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.settings.SettingsActivity;
import com.android.inputmethod.latin.settings.SettingsValues;
@@ -179,7 +179,7 @@
private boolean mIsMainDictionaryAvailable;
private UserBinaryDictionary mUserDictionary;
- private UserHistoryPredictionDictionary mUserHistoryPredictionDictionary;
+ private UserHistoryDictionary mUserHistoryDictionary;
private PersonalizationPredictionDictionary mPersonalizationPredictionDictionary;
private PersonalizationDictionary mPersonalizationDictionary;
private boolean mIsUserDictionaryAvailable;
@@ -623,9 +623,9 @@
final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
- mUserHistoryPredictionDictionary = PersonalizationHelper
- .getUserHistoryPredictionDictionary(this, localeStr, prefs);
- newSuggest.setUserHistoryPredictionDictionary(mUserHistoryPredictionDictionary);
+ mUserHistoryDictionary = PersonalizationHelper.getUserHistoryDictionary(
+ this, localeStr, prefs);
+ newSuggest.setUserHistoryDictionary(mUserHistoryDictionary);
mPersonalizationDictionary = PersonalizationHelper
.getPersonalizationDictionary(this, localeStr, prefs);
newSuggest.setPersonalizationDictionary(mPersonalizationDictionary);
@@ -1521,7 +1521,7 @@
mSubtypeState.switchSubtype(token, mRichImm);
}
- private void sendDownUpKeyEventForBackwardCompatibility(final int code) {
+ private void sendDownUpKeyEvent(final int code) {
final long eventTime = SystemClock.uptimeMillis();
mConnection.sendKeyEvent(new KeyEvent(eventTime, eventTime,
KeyEvent.ACTION_DOWN, code, 0, 0, KeyCharacterMap.VIRTUAL_KEYBOARD, 0,
@@ -1538,7 +1538,7 @@
// TODO: Remove this special handling of digit letters.
// For backward compatibility. See {@link InputMethodService#sendKeyChar(char)}.
if (code >= '0' && code <= '9') {
- sendDownUpKeyEventForBackwardCompatibility(code - '0' + KeyEvent.KEYCODE_0);
+ sendDownUpKeyEvent(code - '0' + KeyEvent.KEYCODE_0);
return;
}
@@ -1547,7 +1547,7 @@
// a hardware keyboard event on pressing enter or delete. This is bad for many
// reasons (there are race conditions with commits) but some applications are
// relying on this behavior so we continue to support it for older apps.
- sendDownUpKeyEventForBackwardCompatibility(KeyEvent.KEYCODE_ENTER);
+ sendDownUpKeyEvent(KeyEvent.KEYCODE_ENTER);
} else {
final String text = new String(new int[] { code }, 0, 1);
mConnection.commitText(text, text.length());
@@ -2104,12 +2104,16 @@
}
final int lengthToDelete = Character.isSupplementaryCodePoint(
mConnection.getCodePointBeforeCursor()) ? 2 : 1;
- if (mAppWorkAroundsUtils.isBeforeJellyBean()) {
- // Backward compatibility mode. Before Jelly bean, the keyboard would simulate
- // a hardware keyboard event on pressing enter or delete. This is bad for many
- // reasons (there are race conditions with commits) but some applications are
- // relying on this behavior so we continue to support it for older apps.
- sendDownUpKeyEventForBackwardCompatibility(KeyEvent.KEYCODE_DEL);
+ if (mAppWorkAroundsUtils.isBeforeJellyBean() ||
+ currentSettings.mInputAttributes.isTypeNull()) {
+ // There are two possible reasons to send a key event: either the field has
+ // type TYPE_NULL, in which case the keyboard should send events, or we are
+ // running in backward compatibility mode. Before Jelly bean, the keyboard
+ // would simulate a hardware keyboard event on pressing enter or delete. This
+ // is bad for many reasons (there are race conditions with commits) but some
+ // applications are relying on this behavior so we continue to support it for
+ // older apps, so we retain this behavior if the app has target SDK < JellyBean.
+ sendDownUpKeyEvent(KeyEvent.KEYCODE_DEL);
} else {
mConnection.deleteSurroundingText(lengthToDelete, 0);
}
@@ -2553,6 +2557,8 @@
private void showSuggestionStripWithTypedWord(final SuggestedWords suggestedWords,
final String typedWord) {
if (suggestedWords.isEmpty()) {
+ // No auto-correction is available, clear the cached values.
+ AccessibilityUtils.getInstance().setAutoCorrection(null, null);
clearSuggestionStrip();
return;
}
@@ -2561,6 +2567,9 @@
setSuggestedWords(suggestedWords, isAutoCorrection);
setAutoCorrectionIndicator(isAutoCorrection);
setSuggestionStripShown(isSuggestionsStripVisible());
+ // An auto-correction is available, cache it in accessibility code so
+ // we can be speak it if the user touches a key that will insert it.
+ AccessibilityUtils.getInstance().setAutoCorrection(suggestedWords, typedWord);
}
private void showSuggestionStrip(final SuggestedWords suggestedWords) {
@@ -2746,9 +2755,8 @@
final SettingsValues currentSettings = mSettings.getCurrent();
if (!currentSettings.mCorrectionEnabled) return null;
- final UserHistoryPredictionDictionary userHistoryPredictionDictionary =
- mUserHistoryPredictionDictionary;
- if (userHistoryPredictionDictionary == null) return null;
+ final UserHistoryDictionary userHistoryDictionary = mUserHistoryDictionary;
+ if (userHistoryDictionary == null) return null;
final String prevWord = mConnection.getNthPreviousWord(currentSettings.mWordSeparators, 2);
final String secondWord;
@@ -2762,8 +2770,7 @@
final int maxFreq = AutoCorrectionUtils.getMaxFrequency(
suggest.getUnigramDictionaries(), suggestion);
if (maxFreq == 0) return null;
- userHistoryPredictionDictionary
- .addToPersonalizationPredictionDictionary(prevWord, secondWord, maxFreq > 0);
+ userHistoryDictionary.addToDictionary(prevWord, secondWord, maxFreq > 0);
return prevWord;
}
@@ -2949,7 +2956,7 @@
}
mConnection.deleteSurroundingText(deleteLength, 0);
if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) {
- mUserHistoryPredictionDictionary.cancelAddingUserHistory(previousWord, committedWord);
+ mUserHistoryDictionary.cancelAddingUserHistory(previousWord, committedWord);
}
final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString;
if (mSettings.getCurrent().mCurrentLanguageHasSpaces) {
diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java
index 925381b..8580a6e 100644
--- a/java/src/com/android/inputmethod/latin/RichInputConnection.java
+++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java
@@ -30,6 +30,7 @@
import com.android.inputmethod.latin.settings.SettingsValues;
import com.android.inputmethod.latin.utils.CapsModeUtils;
import com.android.inputmethod.latin.utils.DebugLogUtils;
+import com.android.inputmethod.latin.utils.SpannableStringUtils;
import com.android.inputmethod.latin.utils.StringUtils;
import com.android.inputmethod.latin.utils.TextRange;
import com.android.inputmethod.research.ResearchLogger;
@@ -610,8 +611,9 @@
// We don't use TextUtils#concat because it copies all spans without respect to their
// nature. If the text includes a PARAGRAPH span and it has been split, then
// TextUtils#concat will crash when it tries to concat both sides of it.
- return new TextRange(StringUtils.concatWithNonParagraphSuggestionSpansOnly(before, after),
- startIndexInBefore, before.length() + endIndexInAfter, before.length());
+ return new TextRange(
+ SpannableStringUtils.concatWithNonParagraphSuggestionSpansOnly(before, after),
+ startIndexInBefore, before.length() + endIndexInAfter, before.length());
}
public boolean isCursorTouchingWord(final SettingsValues settingsValues) {
diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java
index 6c18c94..9fd1f53 100644
--- a/java/src/com/android/inputmethod/latin/Suggest.java
+++ b/java/src/com/android/inputmethod/latin/Suggest.java
@@ -26,7 +26,7 @@
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary;
-import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
+import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.AutoCorrectionUtils;
import com.android.inputmethod.latin.utils.BoundedTreeSet;
@@ -190,10 +190,8 @@
addOrReplaceDictionaryInternal(Dictionary.TYPE_CONTACTS, contactsDictionary);
}
- public void setUserHistoryPredictionDictionary(
- final UserHistoryPredictionDictionary userHistoryPredictionDictionary) {
- addOrReplaceDictionaryInternal(Dictionary.TYPE_USER_HISTORY,
- userHistoryPredictionDictionary);
+ public void setUserHistoryDictionary(final UserHistoryDictionary userHistoryDictionary) {
+ addOrReplaceDictionaryInternal(Dictionary.TYPE_USER_HISTORY, userHistoryDictionary);
}
public void setPersonalizationPredictionDictionary(
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
index 4dba8e5..af61f29 100644
--- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
@@ -225,6 +225,26 @@
return position;
}
+ static void writeUIntToStream(final OutputStream stream, final int value, final int size)
+ throws IOException {
+ switch(size) {
+ case 4:
+ stream.write((value >> 24) & 0xFF);
+ /* fall through */
+ case 3:
+ stream.write((value >> 16) & 0xFF);
+ /* fall through */
+ case 2:
+ stream.write((value >> 8) & 0xFF);
+ /* fall through */
+ case 1:
+ stream.write(value & 0xFF);
+ break;
+ default:
+ /* nop */
+ }
+ }
+
// End utility methods
// This method is responsible for finding a nice ordering of the nodes that favors run-time
@@ -365,12 +385,14 @@
nodeSize + size, ptNode.mChildren));
}
nodeSize += getShortcutListSize(ptNode.mShortcutTargets);
- if (null != ptNode.mBigrams) {
- for (WeightedString bigram : ptNode.mBigrams) {
- final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray,
- nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE,
- FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord));
- nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE;
+ if (formatOptions.mVersion < FormatSpec.FIRST_VERSION_WITH_TERMINAL_ID) {
+ if (null != ptNode.mBigrams) {
+ for (WeightedString bigram : ptNode.mBigrams) {
+ final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray,
+ nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE,
+ FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord));
+ nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE;
+ }
}
}
ptNode.mCachedSize = nodeSize;
@@ -882,8 +904,9 @@
* @param destination the stream to write the file header to.
* @param dict the dictionary to write.
* @param formatOptions file format options.
+ * @return the size of the header.
*/
- /* package */ static void writeDictionaryHeader(final OutputStream destination,
+ /* package */ static int writeDictionaryHeader(final OutputStream destination,
final FusionDictionary dict, final FormatOptions formatOptions)
throws IOException, UnsupportedFormatException {
final int version = formatOptions.mVersion;
@@ -932,5 +955,6 @@
destination.write(bytes);
headerBuffer.close();
+ return size;
}
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
index bf3d191..411e265 100644
--- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
@@ -77,7 +77,7 @@
* @param newParentAddress the absolute address of the parent.
* @param formatOptions file format options.
*/
- public static void updateParentAddress(final DictBuffer dictBuffer,
+ private static void updateParentAddress(final DictBuffer dictBuffer,
final int ptNodeOriginAddress, final int newParentAddress,
final FormatOptions formatOptions) {
final int originalPosition = dictBuffer.position();
@@ -109,7 +109,7 @@
* @param newParentAddress the address to be written.
* @param formatOptions file format options.
*/
- public static void updateParentAddresses(final DictBuffer dictBuffer,
+ private static void updateParentAddresses(final DictBuffer dictBuffer,
final int ptNodeOriginAddress, final int newParentAddress,
final FormatOptions formatOptions) {
final int originalPosition = dictBuffer.position();
@@ -136,7 +136,7 @@
* @param newChildrenAddress the absolute address of the child.
* @param formatOptions file format options.
*/
- public static void updateChildrenAddress(final DictBuffer dictBuffer,
+ private static void updateChildrenAddress(final DictBuffer dictBuffer,
final int ptNodeOriginAddress, final int newChildrenAddress,
final FormatOptions formatOptions) {
final int originalPosition = dictBuffer.position();
diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
index 51b89a0..9481a8c 100644
--- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
+++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
@@ -263,7 +263,14 @@
// These values are used only by version 4 or later.
static final String TRIE_FILE_EXTENSION = ".trie";
static final String FREQ_FILE_EXTENSION = ".freq";
+ // tat = Terminal Address Table
+ static final String TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat";
+ static final String BIGRAM_FILE_EXTENSION = ".bigram";
+ static final String BIGRAM_LOOKUP_TABLE_FILE_EXTENSION = ".bigram_lookup";
+ static final String BIGRAM_ADDRESS_TABLE_FILE_EXTENSION = ".bigram_index";
static final int FREQUENCY_AND_FLAGS_SIZE = 2;
+ static final int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3;
+ static final int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4;
static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE;
static final int NO_PARENT_ADDRESS = 0;
@@ -322,9 +329,15 @@
public final int mHeaderSize;
public final DictionaryOptions mDictionaryOptions;
public final FormatOptions mFormatOptions;
- private static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
- private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
- private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";
+ // Note that these are corresponding definitions in native code in latinime::HeaderPolicy
+ // and latinime::HeaderReadWriteUtils.
+ public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE";
+ public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE";
+ public static final String ATTRIBUTE_VALUE_TRUE = "1";
+
+ public static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
+ public static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
+ public static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";
private static final String DICTIONARY_DESCRIPTION_ATTRIBUTE = "description";
public FileHeader(final int headerSize, final DictionaryOptions dictionaryOptions,
final FormatOptions formatOptions) {
diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java
new file mode 100644
index 0000000..96d057a
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.makedict;
+
+import com.android.inputmethod.annotations.UsedForTesting;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+
+/**
+ * SparseTable is an extensible map from integer to integer.
+ * This holds one value for every mBlockSize keys, so it uses 1/mBlockSize'th of the full index
+ * memory.
+ */
+@UsedForTesting
+public class SparseTable {
+
+ /**
+ * mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize
+ * terminals.
+ * It contains at index i = j / mBlockSize the index in mContentsTable where the values for
+ * terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized integer array.
+ */
+ private final ArrayList<Integer> mLookupTable;
+ private final ArrayList<Integer> mContentTable;
+
+ private final int mBlockSize;
+ public static final int NOT_EXIST = -1;
+
+ @UsedForTesting
+ public SparseTable(final int initialCapacity, final int blockSize) {
+ mBlockSize = blockSize;
+ final int lookupTableSize = initialCapacity / mBlockSize
+ + (initialCapacity % mBlockSize > 0 ? 1 : 0);
+ mLookupTable = new ArrayList<Integer>(Collections.nCopies(lookupTableSize, NOT_EXIST));
+ mContentTable = new ArrayList<Integer>();
+ }
+
+ @UsedForTesting
+ public SparseTable(final int[] lookupTable, final int[] contentTable, final int blockSize) {
+ mBlockSize = blockSize;
+ mLookupTable = new ArrayList<Integer>(lookupTable.length);
+ for (int i = 0; i < lookupTable.length; ++i) {
+ mLookupTable.add(lookupTable[i]);
+ }
+ mContentTable = new ArrayList<Integer>(contentTable.length);
+ for (int i = 0; i < contentTable.length; ++i) {
+ mContentTable.add(contentTable[i]);
+ }
+ }
+
+ /**
+ * Converts an byte array to an int array considering each set of 4 bytes is an int stored in
+ * big-endian.
+ * The length of byteArray must be a multiple of four.
+ * Otherwise, IndexOutOfBoundsException will be raised.
+ */
+ @UsedForTesting
+ private static void convertByteArrayToIntegerArray(final byte[] byteArray,
+ final ArrayList<Integer> integerArray) {
+ for (int i = 0; i < byteArray.length; i += 4) {
+ int value = 0;
+ for (int j = i; j < i + 4; ++j) {
+ value <<= 8;
+ value |= byteArray[j] & 0xFF;
+ }
+ integerArray.add(value);
+ }
+ }
+
+ @UsedForTesting
+ public SparseTable(final byte[] lookupTable, final byte[] contentTable, final int blockSize) {
+ mBlockSize = blockSize;
+ mLookupTable = new ArrayList<Integer>(lookupTable.length / 4);
+ mContentTable = new ArrayList<Integer>(contentTable.length / 4);
+ convertByteArrayToIntegerArray(lookupTable, mLookupTable);
+ convertByteArrayToIntegerArray(contentTable, mContentTable);
+ }
+
+ @UsedForTesting
+ public int get(final int index) {
+ if (index < 0 || index / mBlockSize >= mLookupTable.size()
+ || mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
+ return NOT_EXIST;
+ }
+ return mContentTable.get(mLookupTable.get(index / mBlockSize) + (index % mBlockSize));
+ }
+
+ @UsedForTesting
+ public void set(final int index, final int value) {
+ if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
+ mLookupTable.set(index / mBlockSize, mContentTable.size());
+ for (int i = 0; i < mBlockSize; ++i) {
+ mContentTable.add(NOT_EXIST);
+ }
+ }
+ mContentTable.set(mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value);
+ }
+
+ public void remove(final int index) {
+ set(index, NOT_EXIST);
+ }
+
+ @UsedForTesting
+ public int size() {
+ return mLookupTable.size() * mBlockSize;
+ }
+
+ @UsedForTesting
+ /* package */ int getContentTableSize() {
+ return mContentTable.size();
+ }
+
+ @UsedForTesting
+ /* package */ int getLookupTableSize() {
+ return mLookupTable.size();
+ }
+
+ public boolean contains(final int index) {
+ return get(index) != NOT_EXIST;
+ }
+
+ @UsedForTesting
+ public void write(final OutputStream lookupOutStream, final OutputStream contentOutStream)
+ throws IOException {
+ for (final int index : mLookupTable) {
+ BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, 4);
+ }
+
+ for (final int index : mContentTable) {
+ BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4);
+ }
+ }
+
+ @UsedForTesting
+ public void writeToFiles(final File lookupTableFile, final File contentFile)
+ throws IOException {
+ FileOutputStream lookupTableOutStream = null;
+ FileOutputStream contentOutStream = null;
+ try {
+ lookupTableOutStream = new FileOutputStream(lookupTableFile);
+ contentOutStream = new FileOutputStream(contentFile);
+ write(lookupTableOutStream, contentOutStream);
+ } finally {
+ if (lookupTableOutStream != null) {
+ lookupTableOutStream.close();
+ }
+ if (contentOutStream != null) {
+ contentOutStream.close();
+ }
+ }
+ }
+
+ private static byte[] readFileToByteArray(final File file) throws IOException {
+ final byte[] contents = new byte[(int) file.length()];
+ FileInputStream inStream = null;
+ try {
+ inStream = new FileInputStream(file);
+ inStream.read(contents);
+ } finally {
+ if (inStream != null) {
+ inStream.close();
+ }
+ }
+ return contents;
+ }
+
+ @UsedForTesting
+ public static SparseTable readFromFiles(final File lookupTableFile, final File contentFile,
+ final int blockSize) throws IOException {
+ final byte[] lookupTable = readFileToByteArray(lookupTableFile);
+ final byte[] content = readFileToByteArray(contentFile);
+ return new SparseTable(lookupTable, content, blockSize);
+ }
+}
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
index 36c5a27..0aa4319 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
@@ -41,11 +41,16 @@
private static final int FILETYPE_TRIE = 1;
private static final int FILETYPE_FREQUENCY = 2;
+ private static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3;
+ private static final int FILETYPE_BIGRAM = 4;
private final File mDictDirectory;
private final DictionaryBufferFactory mBufferFactory;
private DictBuffer mDictBuffer;
private DictBuffer mFrequencyBuffer;
+ private DictBuffer mTerminalAddressTableBuffer;
+ private DictBuffer mBigramBuffer;
+ private SparseTable mBigramAddressTable;
@UsedForTesting
/* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) {
@@ -77,6 +82,12 @@
} else if (fileType == FILETYPE_FREQUENCY) {
return new File(mDictDirectory,
mDictDirectory.getName() + FormatSpec.FREQ_FILE_EXTENSION);
+ } else if (fileType == FILETYPE_TERMINAL_ADDRESS_TABLE) {
+ return new File(mDictDirectory,
+ mDictDirectory.getName() + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
+ } else if (fileType == FILETYPE_BIGRAM) {
+ return new File(mDictDirectory,
+ mDictDirectory.getName() + FormatSpec.BIGRAM_FILE_EXTENSION);
} else {
throw new RuntimeException("Unsupported kind of file : " + fileType);
}
@@ -87,6 +98,10 @@
final String filename = mDictDirectory.getName();
mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE));
mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY));
+ mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer(
+ getFile(FILETYPE_TERMINAL_ADDRESS_TABLE));
+ mBigramBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_BIGRAM));
+ loadBigramAddressSparseTable();
}
@Override
@@ -111,6 +126,15 @@
return header;
}
+ private void loadBigramAddressSparseTable() throws IOException {
+ final File lookupIndexFile = new File(mDictDirectory,
+ mDictDirectory.getName() + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION);
+ final File contentFile = new File(mDictDirectory,
+ mDictDirectory.getName() + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION);
+ mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, contentFile,
+ FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE);
+ }
+
protected static class PtNodeReader extends DictDecoder.PtNodeReader {
protected static int readFrequency(final DictBuffer frequencyBuffer, final int terminalId) {
frequencyBuffer.position(terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE + 1);
@@ -184,8 +208,21 @@
final ArrayList<PendingAttribute> bigrams;
if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
bigrams = new ArrayList<PendingAttribute>();
- addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams,
- addressPointer);
+ final int posOfBigrams = mBigramAddressTable.get(terminalId);
+ mBigramBuffer.position(posOfBigrams);
+ while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
+ // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE,
+ // remaining bigram entries are ignored.
+ final int bigramFlags = mBigramBuffer.readUnsignedByte();
+ final int targetTerminalId = mBigramBuffer.readUnsignedInt24();
+ mTerminalAddressTableBuffer.position(
+ targetTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE);
+ final int targetAddress = mTerminalAddressTableBuffer.readUnsignedInt24();
+ bigrams.add(new PendingAttribute(
+ bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY,
+ targetAddress));
+ if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break;
+ }
if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
MakedictLog.d("too many bigrams in a node.");
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
index 75b75ae..4c25faf 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
@@ -26,6 +26,7 @@
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -41,10 +42,15 @@
public class Ver4DictEncoder implements DictEncoder {
private final File mDictPlacedDir;
private byte[] mTrieBuf;
- private byte[] mFreqBuf;
private int mTriePos;
+ private int mHeaderSize;
+ private SparseTable mBigramAddressTable;
private OutputStream mTrieOutStream;
private OutputStream mFreqOutStream;
+ private OutputStream mTerminalAddressTableOutStream;
+ private OutputStream mBigramOutStream;
+ private File mDictDir;
+ private String mBaseFilename;
@UsedForTesting
public Ver4DictEncoder(final File dictPlacedDir) {
@@ -54,18 +60,25 @@
private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions)
throws FileNotFoundException, IOException {
final FileHeader header = new FileHeader(0, dictOptions, formatOptions);
- final String filename = header.getId() + "." + header.getVersion();
- final File mDictDir = new File(mDictPlacedDir, filename);
- final File trieFile = new File(mDictDir, filename + FormatSpec.TRIE_FILE_EXTENSION);
- final File freqFile = new File(mDictDir, filename + FormatSpec.FREQ_FILE_EXTENSION);
+ mBaseFilename = header.getId() + "." + header.getVersion();
+ mDictDir = new File(mDictPlacedDir, mBaseFilename);
+ final File trieFile = new File(mDictDir, mBaseFilename + FormatSpec.TRIE_FILE_EXTENSION);
+ final File freqFile = new File(mDictDir, mBaseFilename + FormatSpec.FREQ_FILE_EXTENSION);
+ final File terminalAddressTableFile = new File(mDictDir,
+ mBaseFilename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
+ final File bigramFile = new File(mDictDir,
+ mBaseFilename + FormatSpec.BIGRAM_FILE_EXTENSION);
if (!mDictDir.isDirectory()) {
if (mDictDir.exists()) mDictDir.delete();
mDictDir.mkdirs();
}
if (!trieFile.exists()) trieFile.createNewFile();
if (!freqFile.exists()) freqFile.createNewFile();
+ if (!terminalAddressTableFile.exists()) terminalAddressTableFile.createNewFile();
mTrieOutStream = new FileOutputStream(trieFile);
mFreqOutStream = new FileOutputStream(freqFile);
+ mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile);
+ mBigramOutStream = new FileOutputStream(bigramFile);
}
private void close() throws IOException {
@@ -76,9 +89,17 @@
if (mFreqOutStream != null) {
mFreqOutStream.close();
}
+ if (mTerminalAddressTableOutStream != null) {
+ mTerminalAddressTableOutStream.close();
+ }
+ if (mBigramOutStream != null) {
+ mBigramOutStream.close();
+ }
} finally {
mTrieOutStream = null;
mFreqOutStream = null;
+ mTerminalAddressTableOutStream = null;
+ mBigramOutStream = null;
}
}
@@ -97,7 +118,8 @@
openStreams(formatOptions, dict.mOptions);
}
- BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict, formatOptions);
+ mHeaderSize = BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict,
+ formatOptions);
MakedictLog.i("Flattening the tree...");
ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray);
@@ -112,10 +134,15 @@
BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions);
if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes);
+ writeTerminalData(flatNodes, terminalCount);
+ mBigramAddressTable = new SparseTable(terminalCount,
+ FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE);
+ writeBigrams(flatNodes, dict);
+ writeBigramAddressSparseTable();
+
final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize;
mTrieBuf = new byte[bufferSize];
- mFreqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE];
MakedictLog.i("Writing file...");
for (PtNodeArray nodeArray : flatNodes) {
@@ -126,7 +153,6 @@
MakedictLog.i("has " + terminalCount + " terminals.");
}
mTrieOutStream.write(mTrieBuf);
- mFreqOutStream.write(mFreqBuf);
MakedictLog.i("Done");
close();
@@ -185,12 +211,6 @@
FormatSpec.PTNODE_TERMINAL_ID_SIZE);
}
- private void writeFrequency(final int frequency, final int terminalId) {
- final int freqPos = terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE;
- BinaryDictEncoderUtils.writeUIntToBuffer(mFreqBuf, freqPos, frequency,
- FormatSpec.FREQUENCY_AND_FLAGS_SIZE);
- }
-
private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
if (formatOptions.mSupportsDynamicUpdate) {
@@ -226,24 +246,41 @@
shortcutByteSize, FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE);
}
- private void writeBigrams(ArrayList<WeightedString> bigrams, FusionDictionary dict) {
- if (bigrams == null) return;
+ private void writeBigrams(final ArrayList<PtNodeArray> flatNodes, final FusionDictionary dict)
+ throws IOException {
+ final ByteArrayOutputStream bigramBuffer = new ByteArrayOutputStream();
- final Iterator<WeightedString> bigramIterator = bigrams.iterator();
- while (bigramIterator.hasNext()) {
- final WeightedString bigram = bigramIterator.next();
- final PtNode target =
- FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord);
- final int addressOfBigram = target.mCachedAddressAfterUpdate;
- final int unigramFrequencyForThisWord = target.mFrequency;
- final int offset = addressOfBigram
- - (mTriePos + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
- int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(),
- offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
- mTrieBuf[mTriePos++] = (byte) bigramFlags;
- mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf,
- mTriePos, Math.abs(offset));
+ for (final PtNodeArray nodeArray : flatNodes) {
+ for (final PtNode ptNode : nodeArray.mData) {
+ if (ptNode.mBigrams != null) {
+ final int startPos = bigramBuffer.size();
+ mBigramAddressTable.set(ptNode.mTerminalId, startPos);
+ final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator();
+ while (bigramIterator.hasNext()) {
+ final WeightedString bigram = bigramIterator.next();
+ final PtNode target =
+ FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord);
+ final int unigramFrequencyForThisWord = target.mFrequency;
+ final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(
+ bigramIterator.hasNext(), 0, bigram.mFrequency,
+ unigramFrequencyForThisWord, bigram.mWord);
+ BinaryDictEncoderUtils.writeUIntToStream(bigramBuffer, bigramFlags,
+ FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
+ BinaryDictEncoderUtils.writeUIntToStream(bigramBuffer, target.mTerminalId,
+ FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE);
+ }
+ }
+ }
}
+ bigramBuffer.writeTo(mBigramOutStream);
+ }
+
+ private void writeBigramAddressSparseTable() throws IOException {
+ final File lookupIndexFile =
+ new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION);
+ final File contentFile =
+ new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION);
+ mBigramAddressTable.writeToFiles(lookupIndexFile, contentFile);
}
@Override
@@ -260,10 +297,30 @@
writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
if (ptNode.isTerminal()) {
writeTerminalId(ptNode.mTerminalId);
- writeFrequency(ptNode.mFrequency, ptNode.mTerminalId);
}
writeChildrenPosition(ptNode, formatOptions);
writeShortcuts(ptNode.mShortcutTargets);
- writeBigrams(ptNode.mBigrams, dict);
+ }
+
+ private void writeTerminalData(final ArrayList<PtNodeArray> flatNodes,
+ final int terminalCount) throws IOException {
+ final byte[] freqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE];
+ final byte[] terminalAddressTableBuf =
+ new byte[terminalCount * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE];
+ for (final PtNodeArray nodeArray : flatNodes) {
+ for (final PtNode ptNode : nodeArray.mData) {
+ if (ptNode.isTerminal()) {
+ BinaryDictEncoderUtils.writeUIntToBuffer(freqBuf,
+ ptNode.mTerminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE,
+ ptNode.mFrequency, FormatSpec.FREQUENCY_AND_FLAGS_SIZE);
+ BinaryDictEncoderUtils.writeUIntToBuffer(terminalAddressTableBuf,
+ ptNode.mTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE,
+ ptNode.mCachedAddressAfterUpdate + mHeaderSize,
+ FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE);
+ }
+ }
+ }
+ mFreqOutStream.write(freqBuf);
+ mTerminalAddressTableOutStream.write(terminalAddressTableBuf);
}
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
similarity index 80%
rename from java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
rename to java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
index 075d7e3..7cf4f0c 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
@@ -22,6 +22,7 @@
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
+import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.makedict.DictDecoder;
@@ -34,12 +35,15 @@
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
/**
- * This class is a base class of a dictionary for the personalized prediction language model.
+ * This class is a base class of a dictionary that supports decaying for the personalized language
+ * model.
*/
-public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary {
- private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName();
+public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary {
+ private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName();
public static final boolean DBG_SAVE_RESTORE = false;
private static final boolean DBG_STRESS_TEST = false;
private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
@@ -47,6 +51,9 @@
/** Any pair being typed or picked */
public static final int FREQUENCY_FOR_TYPED = 2;
+ public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED;
+ public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY;
+
/** Locale for which this user history dictionary is storing words */
private final String mLocale;
@@ -60,8 +67,9 @@
// Should always be false except when we use this class for test
@UsedForTesting boolean mIsTest = false;
- /* package */ DynamicPredictionDictionaryBase(final Context context, final String locale,
- final SharedPreferences sp, final String dictionaryType, final String fileName) {
+ /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
+ final String locale, final SharedPreferences sp, final String dictionaryType,
+ final String fileName) {
super(context, fileName, dictionaryType, true);
mLocale = locale;
mFileName = fileName;
@@ -84,6 +92,18 @@
}
@Override
+ protected Map<String, String> getHeaderAttributeMap() {
+ HashMap<String, String> attributeMap = new HashMap<String, String>();
+ attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+ attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+ attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName);
+ attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale);
+ return attributeMap;
+ }
+
+ @Override
protected boolean hasContentChanged() {
return false;
}
@@ -103,27 +123,29 @@
}
/**
- * Pair will be added to the personalization prediction dictionary.
+ * Pair will be added to the decaying dictionary.
*
* The first word may be null. That means we don't know the context, in other words,
* it's only a unigram. The first word may also be an empty string : this means start
* context, as in beginning of a sentence for example.
* The second word may not be null (a NullPointerException would be thrown).
*/
- public void addToPersonalizationPredictionDictionary(
- final String word0, final String word1, final boolean isValid) {
+ public void addToDictionary(final String word0, final String word1, final boolean isValid) {
if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
return;
}
- addWordDynamically(word1, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED,
+ final int frequency = ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
+ (isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS) :
+ FREQUENCY_FOR_TYPED;
+ addWordDynamically(word1, null /* the "shortcut" parameter is null */, frequency,
false /* isNotAWord */);
// Do not insert a word as a bigram of itself
if (word1.equals(word0)) {
return;
}
if (null != word0) {
- addBigramDynamically(word0, word1, FREQUENCY_FOR_TYPED, isValid);
+ addBigramDynamically(word0, word1, frequency, isValid);
}
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
index 3050885..039b253 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
@@ -36,6 +36,7 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Map;
// Currently this class is used to implement dynamic prodiction dictionary.
// TODO: Move to native code.
@@ -113,8 +114,8 @@
}
@Override
- protected void writeDictionary(final DictEncoder dictEncoder)
- throws IOException, UnsupportedFormatException {
+ protected void writeDictionary(final DictEncoder dictEncoder,
+ final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException {
UserHistoryDictIOUtils.writeDictionary(dictEncoder,
new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams),
mBigramList, FORMAT_OPTIONS);
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
index ab3de80..a86f6e5 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
@@ -46,7 +46,7 @@
// TODO: Use a dynamic binary dictionary instead
public WeakReference<PersonalizationDictionary> mDictionary;
- public WeakReference<DynamicPredictionDictionaryBase> mPredictionDictionary;
+ public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary;
public final String mSystemLocale;
public PersonalizationDictionaryUpdateSession(String locale) {
mSystemLocale = locale;
@@ -60,15 +60,16 @@
mDictionary = new WeakReference<PersonalizationDictionary>(dictionary);
}
- public void setPredictionDictionary(DynamicPredictionDictionaryBase dictionary) {
- mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary);
+ public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) {
+ mPredictionDictionary =
+ new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary);
}
protected PersonalizationDictionary getDictionary() {
return mDictionary == null ? null : mDictionary.get();
}
- protected DynamicPredictionDictionaryBase getPredictionDictionary() {
+ protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() {
return mPredictionDictionary == null ? null : mPredictionDictionary.get();
}
@@ -81,7 +82,7 @@
}
private void unsetPredictionDictionary() {
- final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
+ final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) {
return;
}
@@ -89,7 +90,7 @@
}
public void clearAndFlushPredictionDictionary(Context context) {
- final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
+ final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) {
return;
}
@@ -105,24 +106,23 @@
// TODO: Support multi locale to add bigram
public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,
int frequency) {
- final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
+ final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) {
return;
}
- dictionary.addToPersonalizationPredictionDictionary(word0, word1, isValid);
+ dictionary.addToDictionary(word0, word1, isValid);
}
// Bulk import
// TODO: Support multi locale to add bigram
public void addBigramsToPersonalizationDictionary(
final ArrayList<PersonalizationLanguageModelParam> lmParams) {
- final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
+ final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) {
return;
}
for (final PersonalizationLanguageModelParam lmParam : lmParams) {
- dictionary.addToPersonalizationPredictionDictionary(
- lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid);
+ dictionary.addToDictionary(lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid);
}
}
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
index 5f702ee..8c9484b 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
@@ -30,7 +30,7 @@
private static final String TAG = PersonalizationHelper.class.getSimpleName();
private static final boolean DEBUG = false;
- private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>>
+ private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>>
sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap();
private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>>
@@ -41,25 +41,23 @@
sLangPersonalizationPredictionDictCache =
CollectionUtils.newConcurrentHashMap();
- public static UserHistoryPredictionDictionary getUserHistoryPredictionDictionary(
+ public static UserHistoryDictionary getUserHistoryDictionary(
final Context context, final String locale, final SharedPreferences sp) {
synchronized (sLangUserHistoryDictCache) {
if (sLangUserHistoryDictCache.containsKey(locale)) {
- final SoftReference<UserHistoryPredictionDictionary> ref =
+ final SoftReference<UserHistoryDictionary> ref =
sLangUserHistoryDictCache.get(locale);
- final UserHistoryPredictionDictionary dict = ref == null ? null : ref.get();
+ final UserHistoryDictionary dict = ref == null ? null : ref.get();
if (dict != null) {
if (DEBUG) {
- Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale);
+ Log.w(TAG, "Use cached UserHistoryDictionary for " + locale);
}
dict.reloadDictionaryIfRequired();
return dict;
}
}
- final UserHistoryPredictionDictionary dict =
- new UserHistoryPredictionDictionary(context, locale, sp);
- sLangUserHistoryDictCache.put(
- locale, new SoftReference<UserHistoryPredictionDictionary>(dict));
+ final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale, sp);
+ sLangUserHistoryDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict));
return dict;
}
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
index e80953c..4329544 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
@@ -22,7 +22,7 @@
import android.content.Context;
import android.content.SharedPreferences;
-public class PersonalizationPredictionDictionary extends DynamicPredictionDictionaryBase {
+public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName();
/* package */ PersonalizationPredictionDictionary(final Context context, final String locale,
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
similarity index 85%
rename from java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
rename to java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
index b140c91..a60226d 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
@@ -26,10 +26,10 @@
* Locally gathers stats about the words user types and various other signals like auto-correction
* cancellation or manual picks. This allows the keyboard to adapt to the typist over time.
*/
-public class UserHistoryPredictionDictionary extends DynamicPredictionDictionaryBase {
+public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase {
/* package for tests */ static final String NAME =
- UserHistoryPredictionDictionary.class.getSimpleName();
- /* package */ UserHistoryPredictionDictionary(final Context context, final String locale,
+ UserHistoryDictionary.class.getSimpleName();
+ /* package */ UserHistoryDictionary(final Context context, final String locale,
final SharedPreferences sp) {
super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale));
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
index 4c1803b..55a90ee 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
@@ -54,7 +54,7 @@
* Called when loaded from the SQL DB.
*/
public void addBigram(String word1, String word2, byte fcValue) {
- if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) {
+ if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) {
Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue);
}
final HashMap<String, Byte> map;
@@ -74,7 +74,7 @@
* Called when inserted to the SQL DB.
*/
public void updateBigram(String word1, String word2, byte fcValue) {
- if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) {
+ if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) {
Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue);
}
final HashMap<String, Byte> map;
diff --git a/java/src/com/android/inputmethod/latin/utils/SpannableStringUtils.java b/java/src/com/android/inputmethod/latin/utils/SpannableStringUtils.java
new file mode 100644
index 0000000..b51fd93
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/utils/SpannableStringUtils.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.utils;
+
+import android.text.Spannable;
+import android.text.SpannableString;
+import android.text.Spanned;
+import android.text.SpannedString;
+import android.text.TextUtils;
+import android.text.style.SuggestionSpan;
+
+public final class SpannableStringUtils {
+ /**
+ * Copies the spans from the region <code>start...end</code> in
+ * <code>source</code> to the region
+ * <code>destoff...destoff+end-start</code> in <code>dest</code>.
+ * Spans in <code>source</code> that begin before <code>start</code>
+ * or end after <code>end</code> but overlap this range are trimmed
+ * as if they began at <code>start</code> or ended at <code>end</code>.
+ * Only SuggestionSpans that don't have the SPAN_PARAGRAPH span are copied.
+ *
+ * This code is almost entirely taken from {@link TextUtils#copySpansFrom}, except for the
+ * kind of span that is copied.
+ *
+ * @throws IndexOutOfBoundsException if any of the copied spans
+ * are out of range in <code>dest</code>.
+ */
+ public static void copyNonParagraphSuggestionSpansFrom(Spanned source, int start, int end,
+ Spannable dest, int destoff) {
+ Object[] spans = source.getSpans(start, end, SuggestionSpan.class);
+
+ for (int i = 0; i < spans.length; i++) {
+ int fl = source.getSpanFlags(spans[i]);
+ if (0 != (fl & Spannable.SPAN_PARAGRAPH)) continue;
+
+ int st = source.getSpanStart(spans[i]);
+ int en = source.getSpanEnd(spans[i]);
+
+ if (st < start)
+ st = start;
+ if (en > end)
+ en = end;
+
+ dest.setSpan(spans[i], st - start + destoff, en - start + destoff,
+ fl);
+ }
+ }
+
+ /**
+ * Returns a CharSequence concatenating the specified CharSequences, retaining their
+ * SuggestionSpans that don't have the PARAGRAPH flag, but not other spans.
+ *
+ * This code is almost entirely taken from {@link TextUtils#concat(CharSequence...)}, except
+ * it calls copyNonParagraphSuggestionSpansFrom instead of {@link TextUtils#copySpansFrom}.
+ */
+ public static CharSequence concatWithNonParagraphSuggestionSpansOnly(CharSequence... text) {
+ if (text.length == 0) {
+ return "";
+ }
+
+ if (text.length == 1) {
+ return text[0];
+ }
+
+ boolean spanned = false;
+ for (int i = 0; i < text.length; i++) {
+ if (text[i] instanceof Spanned) {
+ spanned = true;
+ break;
+ }
+ }
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < text.length; i++) {
+ sb.append(text[i]);
+ }
+
+ if (!spanned) {
+ return sb.toString();
+ }
+
+ SpannableString ss = new SpannableString(sb);
+ int off = 0;
+ for (int i = 0; i < text.length; i++) {
+ int len = text[i].length();
+
+ if (text[i] instanceof Spanned) {
+ copyNonParagraphSuggestionSpansFrom((Spanned) text[i], 0, len, ss, off);
+ }
+
+ off += len;
+ }
+
+ return new SpannedString(ss);
+ }
+}
diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
index 327780a..121aecf 100644
--- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
@@ -20,12 +20,7 @@
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.settings.SettingsValues;
-import android.text.Spannable;
-import android.text.SpannableString;
-import android.text.Spanned;
-import android.text.SpannedString;
import android.text.TextUtils;
-import android.text.style.SuggestionSpan;
import android.util.JsonReader;
import android.util.JsonWriter;
import android.util.Log;
@@ -467,88 +462,4 @@
}
return "";
}
-
- /**
- * Copies the spans from the region <code>start...end</code> in
- * <code>source</code> to the region
- * <code>destoff...destoff+end-start</code> in <code>dest</code>.
- * Spans in <code>source</code> that begin before <code>start</code>
- * or end after <code>end</code> but overlap this range are trimmed
- * as if they began at <code>start</code> or ended at <code>end</code>.
- * Only SuggestionSpans that don't have the SPAN_PARAGRAPH span are copied.
- *
- * This code is almost entirely taken from {@link TextUtils#copySpansFrom}, except for the
- * kind of span that is copied.
- *
- * @throws IndexOutOfBoundsException if any of the copied spans
- * are out of range in <code>dest</code>.
- */
- public static void copyNonParagraphSuggestionSpansFrom(Spanned source, int start, int end,
- Spannable dest, int destoff) {
- Object[] spans = source.getSpans(start, end, SuggestionSpan.class);
-
- for (int i = 0; i < spans.length; i++) {
- int fl = source.getSpanFlags(spans[i]);
- if (0 != (fl & Spannable.SPAN_PARAGRAPH)) continue;
-
- int st = source.getSpanStart(spans[i]);
- int en = source.getSpanEnd(spans[i]);
-
- if (st < start)
- st = start;
- if (en > end)
- en = end;
-
- dest.setSpan(spans[i], st - start + destoff, en - start + destoff,
- fl);
- }
- }
-
- /**
- * Returns a CharSequence concatenating the specified CharSequences, retaining their
- * SuggestionSpans that don't have the PARAGRAPH flag, but not other spans.
- *
- * This code is almost entirely taken from {@link TextUtils#concat(CharSequence...)}, except
- * it calls copyNonParagraphSuggestionSpansFrom instead of {@link TextUtils#copySpansFrom}.
- */
- public static CharSequence concatWithNonParagraphSuggestionSpansOnly(CharSequence... text) {
- if (text.length == 0) {
- return "";
- }
-
- if (text.length == 1) {
- return text[0];
- }
-
- boolean spanned = false;
- for (int i = 0; i < text.length; i++) {
- if (text[i] instanceof Spanned) {
- spanned = true;
- break;
- }
- }
-
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < text.length; i++) {
- sb.append(text[i]);
- }
-
- if (!spanned) {
- return sb.toString();
- }
-
- SpannableString ss = new SpannableString(sb);
- int off = 0;
- for (int i = 0; i < text.length; i++) {
- int len = text[i].length();
-
- if (text[i] instanceof Spanned) {
- copyNonParagraphSuggestionSpansFrom((Spanned) text[i], 0, len, ss, off);
- }
-
- off += len;
- }
-
- return new SpannedString(ss);
- }
}
diff --git a/native/jni/Android.mk b/native/jni/Android.mk
index c207032..36afea5 100644
--- a/native/jni/Android.mk
+++ b/native/jni/Android.mk
@@ -85,6 +85,8 @@
$(addprefix suggest/policyimpl/dictionary/utils/, \
buffer_with_extendable_buffer.cpp \
byte_array_utils.cpp \
+ decaying_utils.cpp \
+ dict_file_writing_utils.cpp \
format_utils.cpp) \
suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
$(addprefix suggest/policyimpl/typing/, \
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 7f47493..c5ef264 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -26,12 +26,55 @@
#include "suggest/core/dictionary/dictionary.h"
#include "suggest/core/suggest_options.h"
#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
+#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "utils/autocorrection_threshold_utils.h"
namespace latinime {
class ProximityInfo;
+// TODO: Move to makedict.
+static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclass clazz,
+ jstring filePath, jlong dictVersion, jobjectArray attributeKeyStringArray,
+ jobjectArray attributeValueStringArray) {
+ const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
+ char filePathChars[filePathUtf8Length + 1];
+ env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
+ filePathChars[filePathUtf8Length] = '\0';
+
+ const int keyCount = env->GetArrayLength(attributeKeyStringArray);
+ const int valueCount = env->GetArrayLength(attributeValueStringArray);
+ if (keyCount != valueCount) {
+ return false;
+ }
+
+ HeaderReadWriteUtils::AttributeMap attributeMap;
+ for (int i = 0; i < keyCount; i++) {
+ jstring keyString = static_cast<jstring>(
+ env->GetObjectArrayElement(attributeKeyStringArray, i));
+ const jsize keyUtf8Length = env->GetStringUTFLength(keyString);
+ char keyChars[keyUtf8Length + 1];
+ env->GetStringUTFRegion(keyString, 0, env->GetStringLength(keyString), keyChars);
+ keyChars[keyUtf8Length] = '\0';
+ HeaderReadWriteUtils::AttributeMap::key_type key;
+ HeaderReadWriteUtils::insertCharactersIntoVector(keyChars, &key);
+
+ jstring valueString = static_cast<jstring>(
+ env->GetObjectArrayElement(attributeValueStringArray, i));
+ const jsize valueUtf8Length = env->GetStringUTFLength(valueString);
+ char valueChars[valueUtf8Length + 1];
+ env->GetStringUTFRegion(valueString, 0, env->GetStringLength(valueString), valueChars);
+ valueChars[valueUtf8Length] = '\0';
+ HeaderReadWriteUtils::AttributeMap::mapped_type value;
+ HeaderReadWriteUtils::insertCharactersIntoVector(valueChars, &value);
+
+ attributeMap[key] = value;
+ }
+
+ return DictFileWritingUtils::createEmptyDictFile(filePathChars, static_cast<int>(dictVersion),
+ &attributeMap);
+}
+
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
jlong dictOffset, jlong dictSize, jboolean isUpdatable) {
PROF_OPEN;
@@ -70,10 +113,10 @@
}
static bool latinime_BinaryDictionary_needsToRunGC(JNIEnv *env, jclass clazz,
- jlong dict) {
+ jlong dict, jboolean mindsBlockByGC) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return false;
- return dictionary->needsToRunGC();
+ return dictionary->needsToRunGC(mindsBlockByGC == JNI_TRUE);
}
static void latinime_BinaryDictionary_flushWithGC(JNIEnv *env, jclass clazz, jlong dict,
@@ -280,8 +323,31 @@
bigramProbability);
}
+static jstring latinime_BinaryDictionary_getProperty(JNIEnv *env, jclass clazz, jlong dict,
+ jstring query) {
+ Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+ if (!dictionary) {
+ return env->NewStringUTF("");
+ }
+ const jsize queryUtf8Length = env->GetStringUTFLength(query);
+ char queryChars[queryUtf8Length + 1];
+ env->GetStringUTFRegion(query, 0, env->GetStringLength(query), queryChars);
+ queryChars[queryUtf8Length] = '\0';
+ static const int GET_PROPERTY_RESULT_LENGTH = 100;
+ char resultChars[GET_PROPERTY_RESULT_LENGTH];
+ resultChars[0] = '\0';
+ dictionary->getDictionaryStructurePolicy()->getProperty(queryChars, resultChars,
+ GET_PROPERTY_RESULT_LENGTH);
+ return env->NewStringUTF(resultChars);
+}
+
static const JNINativeMethod sMethods[] = {
{
+ const_cast<char *>("createEmptyDictFileNative"),
+ const_cast<char *>("(Ljava/lang/String;J[Ljava/lang/String;[Ljava/lang/String;)Z"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_createEmptyDictFile)
+ },
+ {
const_cast<char *>("openNative"),
const_cast<char *>("(Ljava/lang/String;JJZ)J"),
reinterpret_cast<void *>(latinime_BinaryDictionary_open)
@@ -298,7 +364,7 @@
},
{
const_cast<char *>("needsToRunGCNative"),
- const_cast<char *>("(J)Z"),
+ const_cast<char *>("(JZ)Z"),
reinterpret_cast<void *>(latinime_BinaryDictionary_needsToRunGC)
},
{
@@ -350,6 +416,11 @@
const_cast<char *>("calculateProbabilityNative"),
const_cast<char *>("(JII)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_calculateProbabilityNative)
+ },
+ {
+ const_cast<char *>("getPropertyNative"),
+ const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_getProperty)
}
};
diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h
index 89dfa39..c2aa8ba 100644
--- a/native/jni/src/defines.h
+++ b/native/jni/src/defines.h
@@ -375,7 +375,7 @@
CT_TERMINAL,
CT_TERMINAL_INSERTION,
// Create new word with space omission
- CT_NEW_WORD_SPACE_OMITTION,
+ CT_NEW_WORD_SPACE_OMISSION,
// Create new word with space substitution
CT_NEW_WORD_SPACE_SUBSTITUTION,
} CorrectionType;
diff --git a/native/jni/src/suggest/core/dicnode/dic_node.h b/native/jni/src/suggest/core/dicnode/dic_node.h
index 41ef9d2..9099e82 100644
--- a/native/jni/src/suggest/core/dicnode/dic_node.h
+++ b/native/jni/src/suggest/core/dicnode/dic_node.h
@@ -38,10 +38,10 @@
INTS_TO_CHARS(mDicNodeState.mDicNodeStatePrevWord.mPrevWord, \
mDicNodeState.mDicNodeStatePrevWord.getPrevWordLength(), prevWordCharBuf, \
NELEMS(prevWordCharBuf)); \
- AKLOGI("#%8s, %5f, %5f, %5f, %5f, %s, %s, %d,,", header, \
+ AKLOGI("#%8s, %5f, %5f, %5f, %5f, %s, %s, %d, %5f,", header, \
getSpatialDistanceForScoring(), getLanguageDistanceForScoring(), \
getNormalizedCompoundDistance(), getRawLength(), prevWordCharBuf, charBuf, \
- getInputIndex(0)); \
+ getInputIndex(0), getNormalizedCompoundDistanceAfterFirstWord()); \
} while (0)
#else
#define LOGI_SHOW_ADD_COST_PROP
@@ -434,6 +434,13 @@
return mDicNodeState.mDicNodeStateScoring.getLanguageDistance();
}
+ // For space-aware gestures, we store the normalized distance at the char index
+ // that ends the first word of the suggestion. We call this the distance after
+ // first word.
+ float getNormalizedCompoundDistanceAfterFirstWord() const {
+ return mDicNodeState.mDicNodeStateScoring.getNormalizedCompoundDistanceAfterFirstWord();
+ }
+
float getLanguageDistanceRatePerWordForScoring() const {
const float langDist = getLanguageDistanceForScoring();
const float totalWordCount =
@@ -565,6 +572,12 @@
inputSize, getTotalInputIndex(), errorType);
}
+ // Saves the current normalized compound distance for space-aware gestures.
+ // See getNormalizedCompoundDistanceAfterFirstWord for details.
+ AK_FORCE_INLINE void saveNormalizedCompoundDistanceAfterFirstWordIfNoneYet() {
+ mDicNodeState.mDicNodeStateScoring.saveNormalizedCompoundDistanceAfterFirstWordIfNoneYet();
+ }
+
// Caveat: Must not be called outside Weighting
// This restriction is guaranteed by "friend"
AK_FORCE_INLINE void forwardInputIndex(const int pointerId, const int count,
diff --git a/native/jni/src/suggest/core/dicnode/internal/dic_node_state_scoring.h b/native/jni/src/suggest/core/dicnode/internal/dic_node_state_scoring.h
index 4c88422..3c85d0e 100644
--- a/native/jni/src/suggest/core/dicnode/internal/dic_node_state_scoring.h
+++ b/native/jni/src/suggest/core/dicnode/internal/dic_node_state_scoring.h
@@ -31,7 +31,8 @@
mDigraphIndex(DigraphUtils::NOT_A_DIGRAPH_INDEX),
mEditCorrectionCount(0), mProximityCorrectionCount(0),
mNormalizedCompoundDistance(0.0f), mSpatialDistance(0.0f), mLanguageDistance(0.0f),
- mRawLength(0.0f), mExactMatch(true) {
+ mRawLength(0.0f), mExactMatch(true),
+ mNormalizedCompoundDistanceAfterFirstWord(MAX_VALUE_FOR_WEIGHTING) {
}
virtual ~DicNodeStateScoring() {}
@@ -45,6 +46,7 @@
mRawLength = 0.0f;
mDoubleLetterLevel = NOT_A_DOUBLE_LETTER;
mDigraphIndex = DigraphUtils::NOT_A_DIGRAPH_INDEX;
+ mNormalizedCompoundDistanceAfterFirstWord = MAX_VALUE_FOR_WEIGHTING;
mExactMatch = true;
}
@@ -58,6 +60,8 @@
mDoubleLetterLevel = scoring->mDoubleLetterLevel;
mDigraphIndex = scoring->mDigraphIndex;
mExactMatch = scoring->mExactMatch;
+ mNormalizedCompoundDistanceAfterFirstWord =
+ scoring->mNormalizedCompoundDistanceAfterFirstWord;
}
void addCost(const float spatialCost, const float languageCost, const bool doNormalization,
@@ -86,6 +90,17 @@
}
}
+ // Saves the current normalized distance for space-aware gestures.
+ // See getNormalizedCompoundDistanceAfterFirstWord for details.
+ void saveNormalizedCompoundDistanceAfterFirstWordIfNoneYet() {
+ // We get called here after each word. We only want to store the distance after
+ // the first word, so if we already have a distance we skip saving -- hence "IfNoneYet"
+ // in the method name.
+ if (mNormalizedCompoundDistanceAfterFirstWord >= MAX_VALUE_FOR_WEIGHTING) {
+ mNormalizedCompoundDistanceAfterFirstWord = getNormalizedCompoundDistance();
+ }
+ }
+
void addRawLength(const float rawLength) {
mRawLength += rawLength;
}
@@ -102,6 +117,13 @@
return mNormalizedCompoundDistance;
}
+ // For space-aware gestures, we store the normalized distance at the char index
+ // that ends the first word of the suggestion. We call this the distance after
+ // first word.
+ float getNormalizedCompoundDistanceAfterFirstWord() const {
+ return mNormalizedCompoundDistanceAfterFirstWord;
+ }
+
float getSpatialDistance() const {
return mSpatialDistance;
}
@@ -178,6 +200,7 @@
float mLanguageDistance;
float mRawLength;
bool mExactMatch;
+ float mNormalizedCompoundDistanceAfterFirstWord;
AK_FORCE_INLINE void addDistance(float spatialDistance, float languageDistance,
bool doNormalization, int inputSize, int totalInputIndex) {
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp
index ec1b63a..b1d01ed 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp
@@ -33,6 +33,8 @@
namespace latinime {
+const int Dictionary::HEADER_ATTRIBUTE_BUFFER_SIZE = 32;
+
Dictionary::Dictionary(JNIEnv *env,
DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy)
: mDictionaryStructureWithBufferPolicy(dictionaryStructureWithBufferPolicy),
@@ -121,32 +123,37 @@
mDictionaryStructureWithBufferPolicy->flushWithGC(filePath);
}
-bool Dictionary::needsToRunGC() {
- return mDictionaryStructureWithBufferPolicy->needsToRunGC();
+bool Dictionary::needsToRunGC(const bool mindsBlockByGC) {
+ return mDictionaryStructureWithBufferPolicy->needsToRunGC(mindsBlockByGC);
+}
+
+void Dictionary::getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const {
+ return mDictionaryStructureWithBufferPolicy->getProperty(query, outResult, maxResultLength);
}
void Dictionary::logDictionaryInfo(JNIEnv *const env) const {
- const int BUFFER_SIZE = 16;
- int dictionaryIdCodePointBuffer[BUFFER_SIZE];
- int versionStringCodePointBuffer[BUFFER_SIZE];
- int dateStringCodePointBuffer[BUFFER_SIZE];
+ int dictionaryIdCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
+ int versionStringCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
+ int dateStringCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
const DictionaryHeaderStructurePolicy *const headerPolicy =
getDictionaryStructurePolicy()->getHeaderStructurePolicy();
headerPolicy->readHeaderValueOrQuestionMark("dictionary", dictionaryIdCodePointBuffer,
- BUFFER_SIZE);
+ HEADER_ATTRIBUTE_BUFFER_SIZE);
headerPolicy->readHeaderValueOrQuestionMark("version", versionStringCodePointBuffer,
- BUFFER_SIZE);
- headerPolicy->readHeaderValueOrQuestionMark("date", dateStringCodePointBuffer, BUFFER_SIZE);
+ HEADER_ATTRIBUTE_BUFFER_SIZE);
+ headerPolicy->readHeaderValueOrQuestionMark("date", dateStringCodePointBuffer,
+ HEADER_ATTRIBUTE_BUFFER_SIZE);
- char dictionaryIdCharBuffer[BUFFER_SIZE];
- char versionStringCharBuffer[BUFFER_SIZE];
- char dateStringCharBuffer[BUFFER_SIZE];
- intArrayToCharArray(dictionaryIdCodePointBuffer, BUFFER_SIZE,
- dictionaryIdCharBuffer, BUFFER_SIZE);
- intArrayToCharArray(versionStringCodePointBuffer, BUFFER_SIZE,
- versionStringCharBuffer, BUFFER_SIZE);
- intArrayToCharArray(dateStringCodePointBuffer, BUFFER_SIZE,
- dateStringCharBuffer, BUFFER_SIZE);
+ char dictionaryIdCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
+ char versionStringCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
+ char dateStringCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
+ intArrayToCharArray(dictionaryIdCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE,
+ dictionaryIdCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE);
+ intArrayToCharArray(versionStringCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE,
+ versionStringCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE);
+ intArrayToCharArray(dateStringCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE,
+ dateStringCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE);
LogUtils::logToJava(env,
"Dictionary info: dictionary = %s ; version = %s ; date = %s",
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h
index 9744474..d8a0f3e 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.h
+++ b/native/jni/src/suggest/core/dictionary/dictionary.h
@@ -81,7 +81,10 @@
void flushWithGC(const char *const filePath);
- bool needsToRunGC();
+ bool needsToRunGC(const bool mindsBlockByGC);
+
+ void getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const;
const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
return mDictionaryStructureWithBufferPolicy;
@@ -92,6 +95,8 @@
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
+ static const int HEADER_ATTRIBUTE_BUFFER_SIZE;
+
DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy;
const BigramDictionary *const mBigramDictionary;
const SuggestInterface *const mGestureSuggest;
diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
index b95488e..c7ffef0 100644
--- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
+++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
@@ -78,7 +78,10 @@
virtual void flushWithGC(const char *const filePath) = 0;
- virtual bool needsToRunGC() const = 0;
+ virtual bool needsToRunGC(const bool mindsBlockByGC) const = 0;
+
+ virtual void getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const = 0;
protected:
DictionaryStructureWithBufferPolicy() {}
diff --git a/native/jni/src/suggest/core/policy/weighting.cpp b/native/jni/src/suggest/core/policy/weighting.cpp
index f9b777d..0c40168 100644
--- a/native/jni/src/suggest/core/policy/weighting.cpp
+++ b/native/jni/src/suggest/core/policy/weighting.cpp
@@ -38,7 +38,7 @@
case CT_SUBSTITUTION:
PROF_SUBSTITUTION(node->mProfiler);
return;
- case CT_NEW_WORD_SPACE_OMITTION:
+ case CT_NEW_WORD_SPACE_OMISSION:
PROF_NEW_WORD(node->mProfiler);
return;
case CT_MATCH:
@@ -93,6 +93,11 @@
}
dicNode->addCost(spatialCost, languageCost, weighting->needsToNormalizeCompoundDistance(),
inputSize, errorType);
+ if (CT_NEW_WORD_SPACE_OMISSION == correctionType) {
+ // When we are on a terminal, we save the current distance for evaluating
+ // when to auto-commit partial suggestions.
+ dicNode->saveNormalizedCompoundDistanceAfterFirstWordIfNoneYet();
+ }
}
/* static */ float Weighting::getSpatialCost(const Weighting *const weighting,
@@ -108,7 +113,7 @@
case CT_SUBSTITUTION:
// only used for typing
return weighting->getSubstitutionCost();
- case CT_NEW_WORD_SPACE_OMITTION:
+ case CT_NEW_WORD_SPACE_OMISSION:
return weighting->getNewWordSpatialCost(traverseSession, dicNode, inputStateG);
case CT_MATCH:
return weighting->getMatchedCost(traverseSession, dicNode, inputStateG);
@@ -138,7 +143,7 @@
return 0.0f;
case CT_SUBSTITUTION:
return 0.0f;
- case CT_NEW_WORD_SPACE_OMITTION:
+ case CT_NEW_WORD_SPACE_OMISSION:
return weighting->getNewWordBigramLanguageCost(
traverseSession, parentDicNode, multiBigramMap);
case CT_MATCH:
@@ -173,7 +178,7 @@
return 0; /* 0 because CT_MATCH will be called */
case CT_SUBSTITUTION:
return 0; /* 0 because CT_MATCH will be called */
- case CT_NEW_WORD_SPACE_OMITTION:
+ case CT_NEW_WORD_SPACE_OMISSION:
return 0;
case CT_MATCH:
return 1;
diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp
index b1340e1..e20bc49 100644
--- a/native/jni/src/suggest/core/suggest.cpp
+++ b/native/jni/src/suggest/core/suggest.cpp
@@ -574,7 +574,7 @@
DicNodeUtils::initAsRootWithPreviousWord(
traverseSession->getDictionaryStructurePolicy(), dicNode, &newDicNode);
const CorrectionType correctionType = spaceSubstitution ?
- CT_NEW_WORD_SPACE_SUBSTITUTION : CT_NEW_WORD_SPACE_OMITTION;
+ CT_NEW_WORD_SPACE_SUBSTITUTION : CT_NEW_WORD_SPACE_OMISSION;
Weighting::addCostAndForwardInputIndex(WEIGHTING, correctionType, traverseSession, dicNode,
&newDicNode, traverseSession->getMultiBigramMap());
if (newDicNode.getCompoundDistance() < static_cast<float>(MAX_VALUE_FOR_WEIGHTING)) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
index bc2f5ee..67a085d 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
@@ -17,10 +17,10 @@
#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h"
#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
-#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+#include "suggest/policyimpl/dictionary/utils/decaying_utils.h"
namespace latinime {
@@ -41,9 +41,14 @@
if (usesAdditionalBuffer && originalBigramPos != NOT_A_DICT_POS) {
originalBigramPos += mBuffer->getOriginalBufferSize();
}
- *outBigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos);
*outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(bigramFlags);
*outHasNext = BigramListReadWriteUtils::hasNext(bigramFlags);
+ if (mIsDecayingDict && !DecayingUtils::isValidBigram(*outProbability)) {
+ // This bigram is too weak to output.
+ *outBigramPos = NOT_A_DICT_POS;
+ } else {
+ *outBigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos);
+ }
if (usesAdditionalBuffer) {
*bigramEntryPos += mBuffer->getOriginalBufferSize();
}
@@ -119,7 +124,7 @@
// Finding useless bigram entries and remove them. Bigram entry is useless when the target PtNode
// has been deleted or is not a valid terminal.
bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(
- int *const bigramListPos) {
+ int *const bigramListPos, int *const outValidBigramEntryCount) {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
if (usesAdditionalBuffer) {
*bigramListPos -= mBuffer->getOriginalBufferSize();
@@ -153,14 +158,22 @@
const int bigramTargetNodePos =
followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos);
nodeReader.fetchNodeInfoInBufferFromPtNodePos(bigramTargetNodePos);
- // TODO: Update probability for supporting probability decaying.
if (nodeReader.isDeleted() || !nodeReader.isTerminal()
|| bigramTargetNodePos == NOT_A_DICT_POS) {
// The target is no longer valid terminal. Invalidate the current bigram entry.
if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, bigramFlags,
- NOT_A_DICT_POS /* targetOffset */, &bigramEntryPos)) {
+ NOT_A_DICT_POS /* targetPtNodePos */, &bigramEntryPos)) {
return false;
}
+ continue;
+ }
+ bool isRemoved = false;
+ if (!updateProbabilityForDecay(bigramFlags, bigramTargetNodePos, &bigramEntryPos,
+ &isRemoved)) {
+ return false;
+ }
+ if (!isRemoved) {
+ (*outValidBigramEntryCount) += 1;
}
} while(BigramListReadWriteUtils::hasNext(bigramFlags));
return true;
@@ -169,7 +182,7 @@
// Updates bigram target PtNode positions in the list after the placing step in GC.
bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos,
const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const
- ptNodePositionRelocationMap) {
+ ptNodePositionRelocationMap, int *const outBigramEntryCount) {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
if (usesAdditionalBuffer) {
*bigramListPos -= mBuffer->getOriginalBufferSize();
@@ -211,11 +224,12 @@
return false;
}
} while(BigramListReadWriteUtils::hasNext(bigramFlags));
+ (*outBigramEntryCount) = bigramEntryCount;
return true;
}
bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTargetPos,
- const int probability, int *const bigramListPos) {
+ const int probability, int *const bigramListPos, bool *const outAddedNewBigram) {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
if (usesAdditionalBuffer) {
*bigramListPos -= mBuffer->getOriginalBufferSize();
@@ -243,8 +257,15 @@
}
if (followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos) == bigramTargetPos) {
// Update this bigram entry.
+ *outAddedNewBigram = false;
+ const int originalProbability = BigramListReadWriteUtils::getProbabilityFromFlags(
+ bigramFlags);
+ const int probabilityToWrite = mIsDecayingDict ?
+ DecayingUtils::getUpdatedBigramProbabilityDelta(
+ originalProbability, probability) : probability;
const BigramListReadWriteUtils::BigramFlags updatedFlags =
- BigramListReadWriteUtils::setProbabilityInFlags(bigramFlags, probability);
+ BigramListReadWriteUtils::setProbabilityInFlags(bigramFlags,
+ probabilityToWrite);
return BigramListReadWriteUtils::writeBigramEntry(mBuffer, updatedFlags,
originalBigramPos, &entryPos);
}
@@ -254,12 +275,14 @@
// The current last entry is found.
// First, update the flags of the last entry.
if (!BigramListReadWriteUtils::setHasNextFlag(mBuffer, true /* hasNext */, entryPos)) {
+ *outAddedNewBigram = false;
return false;
}
if (usesAdditionalBuffer) {
*bigramListPos += mBuffer->getOriginalBufferSize();
}
// Then, add a new entry after the last entry.
+ *outAddedNewBigram = true;
return writeNewBigramEntry(bigramTargetPos, probability, bigramListPos);
} while(BigramListReadWriteUtils::hasNext(bigramFlags));
// We return directly from the while loop.
@@ -270,8 +293,11 @@
bool DynamicBigramListPolicy::writeNewBigramEntry(const int bigramTargetPos, const int probability,
int *const writingPos) {
// hasNext is false because we are adding a new bigram entry at the end of the bigram list.
+ const int probabilityToWrite = mIsDecayingDict ?
+ DecayingUtils::getUpdatedBigramProbabilityDelta(NOT_A_PROBABILITY, probability) :
+ probability;
return BigramListReadWriteUtils::createAndWriteBigramEntry(mBuffer, bigramTargetPos,
- probability, false /* hasNext */, writingPos);
+ probabilityToWrite, false /* hasNext */, writingPos);
}
bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int bigramTargetPos) {
@@ -325,7 +351,7 @@
nodeReader.fetchNodeInfoInBufferFromPtNodePos(currentPos);
bigramLinkCount++;
if (bigramLinkCount > CONTINUING_BIGRAM_LINK_COUNT_LIMIT) {
- AKLOGE("Bigram link is invalid. start position: %d", bigramPos);
+ AKLOGE("Bigram link is invalid. start position: %d", originalBigramPos);
ASSERT(false);
return NOT_A_DICT_POS;
}
@@ -333,4 +359,33 @@
return currentPos;
}
+bool DynamicBigramListPolicy::updateProbabilityForDecay(
+ BigramListReadWriteUtils::BigramFlags bigramFlags, const int targetPtNodePos,
+ int *const bigramEntryPos, bool *const outRemoved) const {
+ *outRemoved = false;
+ if (mIsDecayingDict) {
+ // Update bigram probability for decaying.
+ const int newProbability = DecayingUtils::getBigramProbabilityDeltaToSave(
+ BigramListReadWriteUtils::getProbabilityFromFlags(bigramFlags));
+ if (DecayingUtils::isValidBigram(newProbability)) {
+ // Write new probability.
+ const BigramListReadWriteUtils::BigramFlags updatedBigramFlags =
+ BigramListReadWriteUtils::setProbabilityInFlags(
+ bigramFlags, newProbability);
+ if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, updatedBigramFlags,
+ targetPtNodePos, bigramEntryPos)) {
+ return false;
+ }
+ } else {
+ // Remove current bigram entry.
+ *outRemoved = true;
+ if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, bigramFlags,
+ NOT_A_DICT_POS /* targetPtNodePos */, bigramEntryPos)) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
index 8ea318a..b358b4e 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
@@ -21,6 +21,7 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
+#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"
namespace latinime {
@@ -34,8 +35,9 @@
class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {
public:
DynamicBigramListPolicy(BufferWithExtendableBuffer *const buffer,
- const DictionaryShortcutsStructurePolicy *const shortcutPolicy)
- : mBuffer(buffer), mShortcutPolicy(shortcutPolicy) {}
+ const DictionaryShortcutsStructurePolicy *const shortcutPolicy,
+ const bool isDecayingDict)
+ : mBuffer(buffer), mShortcutPolicy(shortcutPolicy), mIsDecayingDict(isDecayingDict) {}
~DynamicBigramListPolicy() {}
@@ -50,19 +52,20 @@
bool copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite, int *const fromPos,
int *const toPos, int *const outBigramsCount) const;
- bool updateAllBigramEntriesAndDeleteUselessEntries(int *const bigramListPos);
+ bool updateAllBigramEntriesAndDeleteUselessEntries(int *const bigramListPos,
+ int *const outBigramEntryCount);
bool updateAllBigramTargetPtNodePositions(int *const bigramListPos,
const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const
- ptNodePositionRelocationMap);
+ ptNodePositionRelocationMap, int *const outValidBigramEntryCount);
bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability,
- int *const bigramListPos);
+ int *const bigramListPos, bool *const outAddedNewBigram);
bool writeNewBigramEntry(const int bigramTargetPos, const int probability,
int *const writingPos);
- // Return if targetBigramPos is found or not.
+ // Return whether or not targetBigramPos is found.
bool removeBigram(const int bigramListPos, const int bigramTargetPos);
private:
@@ -73,9 +76,13 @@
BufferWithExtendableBuffer *const mBuffer;
const DictionaryShortcutsStructurePolicy *const mShortcutPolicy;
+ const bool mIsDecayingDict;
// Follow bigram link and return the position of bigram target PtNode that is currently valid.
int followBigramLinkAndGetCurrentBigramPtNodePos(const int originalBigramPos) const;
+
+ bool updateProbabilityForDecay(BigramListReadWriteUtils::BigramFlags bigramFlags,
+ const int targetPtNodePos, int *const bigramEntryPos, bool *const outRemoved) const;
};
} // namespace latinime
#endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
index c60e458..081163a 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp
@@ -16,6 +16,8 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h"
+#include "suggest/policyimpl/dictionary/utils/decaying_utils.h"
+
namespace latinime {
bool DynamicPatriciaTrieGcEventListeners
@@ -25,6 +27,19 @@
// PtNode is useless when the PtNode is not a terminal and doesn't have any not useless
// children.
bool isUselessPtNode = !node->isTerminal();
+ if (node->isTerminal() && mIsDecayingDict) {
+ const int newProbability =
+ DecayingUtils::getUnigramProbabilityToSave(node->getProbability());
+ int writingPos = node->getProbabilityFieldPos();
+ // Update probability.
+ if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(
+ mBuffer, newProbability, &writingPos)) {
+ return false;
+ }
+ if (!DecayingUtils::isValidUnigram(newProbability)) {
+ isUselessPtNode = false;
+ }
+ }
if (mChildrenValue > 0) {
isUselessPtNode = false;
} else if (node->isTerminal()) {
@@ -41,7 +56,27 @@
return false;
}
} else {
- valueStack.back() += 1;
+ mValueStack.back() += 1;
+ if (node->isTerminal()) {
+ mValidUnigramCount += 1;
+ }
+ }
+ return true;
+}
+
+bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability
+ ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
+ const int *const nodeCodePoints) {
+ if (!node->isDeleted()) {
+ int pos = node->getBigramsPos();
+ if (pos != NOT_A_DICT_POS) {
+ int bigramEntryCount = 0;
+ if (!mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&pos,
+ &bigramEntryCount)) {
+ return false;
+ }
+ mValidBigramEntryCount += bigramEntryCount;
+ }
}
return true;
}
@@ -137,10 +172,15 @@
// Updates bigram target PtNode positions in the bigram list.
int bigramsPos = node->getBigramsPos();
if (bigramsPos != NOT_A_DICT_POS) {
+ int bigramEntryCount;
if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos,
- &mDictPositionRelocationMap->mPtNodePositionRelocationMap)) {
+ &mDictPositionRelocationMap->mPtNodePositionRelocationMap, &bigramEntryCount)) {
return false;
}
+ mBigramCount += bigramEntryCount;
+ }
+ if (node->isTerminal()) {
+ mUnigramCount++;
}
return true;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
index 4256f22..463715a 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h
@@ -39,23 +39,23 @@
public:
TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
DynamicPatriciaTrieWritingHelper *const writingHelper,
- BufferWithExtendableBuffer *const buffer)
- : mWritingHelper(writingHelper), mBuffer(buffer), valueStack(),
- mChildrenValue(0) {}
+ BufferWithExtendableBuffer *const buffer, const bool isDecayingDict)
+ : mWritingHelper(writingHelper), mBuffer(buffer), mIsDecayingDict(isDecayingDict),
+ mValueStack(), mChildrenValue(0), mValidUnigramCount(0) {}
~TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted() {};
bool onAscend() {
- if (valueStack.empty()) {
+ if (mValueStack.empty()) {
return false;
}
- mChildrenValue = valueStack.back();
- valueStack.pop_back();
+ mChildrenValue = mValueStack.back();
+ mValueStack.pop_back();
return true;
}
bool onDescend(const int ptNodeArrayPos) {
- valueStack.push_back(0);
+ mValueStack.push_back(0);
return true;
}
@@ -64,14 +64,20 @@
bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
const int *const nodeCodePoints);
+ int getValidUnigramCount() const {
+ return mValidUnigramCount;
+ }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(
TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted);
DynamicPatriciaTrieWritingHelper *const mWritingHelper;
BufferWithExtendableBuffer *const mBuffer;
- std::vector<int> valueStack;
+ const int mIsDecayingDict;
+ std::vector<int> mValueStack;
int mChildrenValue;
+ int mValidUnigramCount;
};
// Updates all bigram entries that are held by valid PtNodes. This removes useless bigram
@@ -80,7 +86,7 @@
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
public:
TraversePolicyToUpdateBigramProbability(DynamicBigramListPolicy *const bigramPolicy)
- : mBigramPolicy(bigramPolicy) {}
+ : mBigramPolicy(bigramPolicy), mValidBigramEntryCount(0) {}
bool onAscend() { return true; }
@@ -89,22 +95,17 @@
bool onReadingPtNodeArrayTail() { return true; }
bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
- const int *const nodeCodePoints) {
- if (!node->isDeleted()) {
- int pos = node->getBigramsPos();
- if (pos != NOT_A_DICT_POS) {
- if (!mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&pos)) {
- return false;
- }
- }
- }
- return true;
+ const int *const nodeCodePoints);
+
+ int getValidBigramEntryCount() const {
+ return mValidBigramEntryCount;
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability);
DynamicBigramListPolicy *const mBigramPolicy;
+ int mValidBigramEntryCount;
};
class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
@@ -150,7 +151,8 @@
dictPositionRelocationMap)
: mWritingHelper(writingHelper), mBigramPolicy(bigramPolicy),
mBufferToWrite(bufferToWrite),
- mDictPositionRelocationMap(dictPositionRelocationMap) {};
+ mDictPositionRelocationMap(dictPositionRelocationMap), mUnigramCount(0),
+ mBigramCount(0) {};
bool onAscend() { return true; }
@@ -161,6 +163,14 @@
bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,
const int *const nodeCodePoints);
+ int getUnigramCount() const {
+ return mUnigramCount;
+ }
+
+ int getBigramCount() const {
+ return mBigramCount;
+ }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields);
@@ -169,6 +179,8 @@
BufferWithExtendableBuffer *const mBufferToWrite;
const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const
mDictPositionRelocationMap;
+ int mUnigramCount;
+ int mBigramCount;
};
private:
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
index 42397c1..0d8c927 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp
@@ -16,6 +16,10 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h"
+#include <cstdio>
+#include <cstring>
+#include <ctime>
+
#include "defines.h"
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_vector.h"
@@ -24,10 +28,18 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
+#include "suggest/policyimpl/dictionary/utils/decaying_utils.h"
#include "suggest/policyimpl/dictionary/utils/probability_utils.h"
namespace latinime {
+const char *const DynamicPatriciaTriePolicy::UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
+const char *const DynamicPatriciaTriePolicy::BIGRAM_COUNT_QUERY = "BIGRAM_COUNT";
+const int DynamicPatriciaTriePolicy::MAX_DICT_EXTENDED_REGION_SIZE = 1024 * 1024;
+const int DynamicPatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS =
+ DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE - 1024;
+const int DynamicPatriciaTriePolicy::MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING = 2 * 60 * 60;
+
void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode,
DicNodeVector *const childDicNodes) const {
if (!dicNode->hasChildren()) {
@@ -137,14 +149,17 @@
int DynamicPatriciaTriePolicy::getProbability(const int unigramProbability,
const int bigramProbability) const {
- // TODO: check mHeaderPolicy.usesForgettingCurve();
- if (unigramProbability == NOT_A_PROBABILITY) {
- return NOT_A_PROBABILITY;
- } else if (bigramProbability == NOT_A_PROBABILITY) {
- return ProbabilityUtils::backoff(unigramProbability);
+ if (mHeaderPolicy.isDecayingDict()) {
+ return DecayingUtils::getProbability(unigramProbability, bigramProbability);
} else {
- return ProbabilityUtils::computeProbabilityForBigram(unigramProbability,
- bigramProbability);
+ if (unigramProbability == NOT_A_PROBABILITY) {
+ return NOT_A_PROBABILITY;
+ } else if (bigramProbability == NOT_A_PROBABILITY) {
+ return ProbabilityUtils::backoff(unigramProbability);
+ } else {
+ return ProbabilityUtils::computeProbabilityForBigram(unigramProbability,
+ bigramProbability);
+ }
}
}
@@ -193,12 +208,26 @@
AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
return false;
}
+ if (mBufferWithExtendableBuffer.getTailPosition()
+ >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
+ AKLOGE("The dictionary is too large to dynamically update.");
+ return false;
+ }
DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer,
getBigramsStructurePolicy(), getShortcutsStructurePolicy());
readingHelper.initWithPtNodeArrayPos(getRootPosition());
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
- &mBigramListPolicy, &mShortcutListPolicy);
- return writingHelper.addUnigramWord(&readingHelper, word, length, probability);
+ &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict());
+ bool addedNewUnigram = false;
+ if (writingHelper.addUnigramWord(&readingHelper, word, length, probability,
+ &addedNewUnigram)) {
+ if (addedNewUnigram) {
+ mUnigramCount++;
+ }
+ return true;
+ } else {
+ return false;
+ }
}
bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int length0,
@@ -207,6 +236,11 @@
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
return false;
}
+ if (mBufferWithExtendableBuffer.getTailPosition()
+ >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
+ AKLOGE("The dictionary is too large to dynamically update.");
+ return false;
+ }
const int word0Pos = getTerminalNodePositionOfWord(word0, length0,
false /* forceLowerCaseSearch */);
if (word0Pos == NOT_A_DICT_POS) {
@@ -218,8 +252,16 @@
return false;
}
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
- &mBigramListPolicy, &mShortcutListPolicy);
- return writingHelper.addBigramWords(word0Pos, word1Pos, probability);
+ &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict());
+ bool addedNewBigram = false;
+ if (writingHelper.addBigramWords(word0Pos, word1Pos, probability, &addedNewBigram)) {
+ if (addedNewBigram) {
+ mBigramCount++;
+ }
+ return true;
+ } else {
+ return false;
+ }
}
bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0,
@@ -228,6 +270,11 @@
AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary.");
return false;
}
+ if (mBufferWithExtendableBuffer.getTailPosition()
+ >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
+ AKLOGE("The dictionary is too large to dynamically update.");
+ return false;
+ }
const int word0Pos = getTerminalNodePositionOfWord(word0, length0,
false /* forceLowerCaseSearch */);
if (word0Pos == NOT_A_DICT_POS) {
@@ -239,8 +286,13 @@
return false;
}
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
- &mBigramListPolicy, &mShortcutListPolicy);
- return writingHelper.removeBigramWords(word0Pos, word1Pos);
+ &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict());
+ if (writingHelper.removeBigramWords(word0Pos, word1Pos)) {
+ mBigramCount--;
+ return true;
+ } else {
+ return false;
+ }
}
void DynamicPatriciaTriePolicy::flush(const char *const filePath) {
@@ -249,8 +301,8 @@
return;
}
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
- &mBigramListPolicy, &mShortcutListPolicy);
- writingHelper.writeToDictFile(filePath, &mHeaderPolicy);
+ &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict());
+ writingHelper.writeToDictFile(filePath, &mHeaderPolicy, mUnigramCount, mBigramCount);
}
void DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) {
@@ -259,17 +311,51 @@
return;
}
DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
- &mBigramListPolicy, &mShortcutListPolicy);
+ &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict());
writingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy);
}
-bool DynamicPatriciaTriePolicy::needsToRunGC() const {
+bool DynamicPatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
if (!mBuffer->isUpdatable()) {
AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
return false;
}
- // TODO: Implement more properly.
- return mBufferWithExtendableBuffer.isNearSizeLimit();
+ if (mBufferWithExtendableBuffer.isNearSizeLimit()) {
+ // Additional buffer size is near the limit.
+ return true;
+ } else if (mHeaderPolicy.getExtendedRegionSize()
+ + mBufferWithExtendableBuffer.getUsedAdditionalBufferSize()
+ > MAX_DICT_EXTENDED_REGION_SIZE) {
+ // Total extended region size exceeds the limit.
+ return true;
+ } else if (mBufferWithExtendableBuffer.getTailPosition()
+ >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS
+ && mBufferWithExtendableBuffer.getUsedAdditionalBufferSize() > 0) {
+ // Needs to reduce dictionary size.
+ return true;
+ } else if (mHeaderPolicy.isDecayingDict()) {
+ if (mUnigramCount >= DecayingUtils::MAX_UNIGRAM_COUNT) {
+ // Unigram count exceeds the limit.
+ return true;
+ } else if (mBigramCount >= DecayingUtils::MAX_BIGRAM_COUNT) {
+ // Bigram count exceeds the limit.
+ return true;
+ } else if (mindsBlockByGC && mHeaderPolicy.getLastUpdatedTime()
+ + MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING < time(0)) {
+ // Time to update probabilities for decaying.
+ return true;
+ }
+ }
+ return false;
+}
+
+void DynamicPatriciaTriePolicy::getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const {
+ if (strncmp(query, UNIGRAM_COUNT_QUERY, maxResultLength) == 0) {
+ snprintf(outResult, maxResultLength, "%d", mUnigramCount);
+ } else if (strncmp(query, BIGRAM_COUNT_QUERY, maxResultLength) == 0) {
+ snprintf(outResult, maxResultLength, "%d", mBigramCount);
+ }
}
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
index 06d8095..d3150c6 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h
@@ -37,7 +37,10 @@
mBufferWithExtendableBuffer(mBuffer->getBuffer() + mHeaderPolicy.getSize(),
mBuffer->getBufferSize() - mHeaderPolicy.getSize()),
mShortcutListPolicy(&mBufferWithExtendableBuffer),
- mBigramListPolicy(&mBufferWithExtendableBuffer, &mShortcutListPolicy) {}
+ mBigramListPolicy(&mBufferWithExtendableBuffer, &mShortcutListPolicy,
+ mHeaderPolicy.isDecayingDict()),
+ mUnigramCount(mHeaderPolicy.getUnigramCount()),
+ mBigramCount(mHeaderPolicy.getBigramCount()) {}
~DynamicPatriciaTriePolicy() {
delete mBuffer;
@@ -89,16 +92,27 @@
void flushWithGC(const char *const filePath);
- bool needsToRunGC() const;
+ bool needsToRunGC(const bool mindsBlockByGC) const;
+
+ void getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);
+ static const char*const UNIGRAM_COUNT_QUERY;
+ static const char*const BIGRAM_COUNT_QUERY;
+ static const int MAX_DICT_EXTENDED_REGION_SIZE;
+ static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS;
+ static const int MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING;
+
const MmappedBuffer *const mBuffer;
const HeaderPolicy mHeaderPolicy;
BufferWithExtendableBuffer mBufferWithExtendableBuffer;
DynamicShortcutListPolicy mShortcutListPolicy;
DynamicBigramListPolicy mBigramListPolicy;
+ int mUnigramCount;
+ int mBigramCount;
};
} // namespace latinime
#endif // LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h
index 154590f..512a4d8 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h
@@ -264,7 +264,7 @@
AK_FORCE_INLINE void pushReadingStateToStack() {
if (mReadingStateStack.size() > MAX_READING_STATE_STACK_SIZE) {
- AKLOGI("Reading state stack overflow. Max size: %d", MAX_READING_STATE_STACK_SIZE);
+ AKLOGI("Reading state stack overflow. Max size: %zd", MAX_READING_STATE_STACK_SIZE);
ASSERT(false);
mIsError = true;
mReadingState.mPos = NOT_A_DICT_POS;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
index a51ae5e..28124d2 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
@@ -16,9 +16,6 @@
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"
-#include <cstdio>
-#include <cstring>
-
#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h"
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"
@@ -28,19 +25,20 @@
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h"
+#include "suggest/policyimpl/dictionary/utils/decaying_utils.h"
+#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "utils/hash_map_compat.h"
namespace latinime {
const int DynamicPatriciaTrieWritingHelper::CHILDREN_POSITION_FIELD_SIZE = 3;
-const char *const DynamicPatriciaTrieWritingHelper::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
- ".tmp";
// TODO: Make MAX_DICTIONARY_SIZE 8MB.
const size_t DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;
bool DynamicPatriciaTrieWritingHelper::addUnigramWord(
DynamicPatriciaTrieReadingHelper *const readingHelper,
- const int *const wordCodePoints, const int codePointCount, const int probability) {
+ const int *const wordCodePoints, const int codePointCount, const int probability,
+ bool *const outAddedNewUnigram) {
int parentPos = NOT_A_DICT_POS;
while (!readingHelper->isEnd()) {
const int matchedCodePointCount = readingHelper->getPrevTotalCodePointCount();
@@ -58,8 +56,11 @@
const int nextIndex = matchedCodePointCount + j;
if (nextIndex >= codePointCount || !readingHelper->isMatchedCodePoint(j,
wordCodePoints[matchedCodePointCount + j])) {
+ *outAddedNewUnigram = true;
return reallocatePtNodeAndAddNewPtNodes(nodeReader,
- readingHelper->getMergedNodeCodePoints(), j, probability,
+ readingHelper->getMergedNodeCodePoints(), j,
+ getUpdatedProbability(NOT_A_PROBABILITY /* originalProbability */,
+ probability),
wordCodePoints + matchedCodePointCount,
codePointCount - matchedCodePointCount);
}
@@ -67,10 +68,12 @@
// All characters are matched.
if (codePointCount == readingHelper->getTotalCodePointCount()) {
return setPtNodeProbability(nodeReader, probability,
- readingHelper->getMergedNodeCodePoints());
+ readingHelper->getMergedNodeCodePoints(), outAddedNewUnigram);
}
if (!nodeReader->hasChildren()) {
- return createChildrenPtNodeArrayAndAChildPtNode(nodeReader, probability,
+ *outAddedNewUnigram = true;
+ return createChildrenPtNodeArrayAndAChildPtNode(nodeReader,
+ getUpdatedProbability(NOT_A_PROBABILITY /* originalProbability */, probability),
wordCodePoints + readingHelper->getTotalCodePointCount(),
codePointCount - readingHelper->getTotalCodePointCount());
}
@@ -83,14 +86,15 @@
return false;
}
int pos = readingHelper->getPosOfLastForwardLinkField();
+ *outAddedNewUnigram = true;
return createAndInsertNodeIntoPtNodeArray(parentPos,
wordCodePoints + readingHelper->getPrevTotalCodePointCount(),
codePointCount - readingHelper->getPrevTotalCodePointCount(),
- probability, &pos);
+ getUpdatedProbability(NOT_A_PROBABILITY /* originalProbability */, probability), &pos);
}
bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const int word1Pos,
- const int probability) {
+ const int probability, bool *const outAddedNewBigram) {
int mMergedNodeCodePoints[MAX_WORD_LENGTH];
DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy);
nodeReader.fetchNodeInfoInBufferFromPtNodePosAndGetNodeCodePoints(word0Pos, MAX_WORD_LENGTH,
@@ -111,9 +115,11 @@
if (nodeReader.getBigramsPos() != NOT_A_DICT_POS) {
// Insert a new bigram entry into the existing bigram list.
int bigramListPos = nodeReader.getBigramsPos();
- return mBigramPolicy->addNewBigramEntryToBigramList(word1Pos, probability, &bigramListPos);
+ return mBigramPolicy->addNewBigramEntryToBigramList(word1Pos, probability, &bigramListPos,
+ outAddedNewBigram);
} else {
// The PtNode doesn't have a bigram list.
+ *outAddedNewBigram = true;
// First, Write a bigram entry at the tail position of the PtNode.
if (!mBigramPolicy->writeNewBigramEntry(word1Pos, probability, &writingPos)) {
return false;
@@ -142,26 +148,32 @@
}
void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileName,
- const HeaderPolicy *const headerPolicy) {
+ const HeaderPolicy *const headerPolicy, const int unigramCount, const int bigramCount) {
BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
- if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) {
+ const int extendedRegionSize = headerPolicy->getExtendedRegionSize() +
+ mBuffer->getUsedAdditionalBufferSize();
+ if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */,
+ unigramCount, bigramCount, extendedRegionSize)) {
return;
}
- flushAllToFile(fileName, &headerBuffer, mBuffer);
+ DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, mBuffer);
}
void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
const char *const fileName, const HeaderPolicy *const headerPolicy) {
- BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
- if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */)) {
- return;
- }
BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */,
MAX_DICTIONARY_SIZE);
- if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) {
+ int unigramCount = 0;
+ int bigramCount = 0;
+ if (!runGC(rootPtNodeArrayPos, &newDictBuffer, &unigramCount, &bigramCount)) {
return;
}
- flushAllToFile(fileName, &headerBuffer, &newDictBuffer);
+ BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
+ if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */,
+ unigramCount, bigramCount, 0 /* extendedRegionSize */)) {
+ return;
+ }
+ DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, &newDictBuffer);
}
bool DynamicPatriciaTrieWritingHelper::markNodeAsDeleted(
@@ -339,23 +351,28 @@
bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability(
const DynamicPatriciaTrieNodeReader *const originalPtNode, const int probability,
- const int *const codePoints) {
+ const int *const codePoints, bool *const outAddedNewUnigram) {
if (originalPtNode->isTerminal()) {
// Overwrites the probability.
+ *outAddedNewUnigram = false;
+ const int probabilityToWrite = getUpdatedProbability(originalPtNode->getProbability(),
+ probability);
int probabilityFieldPos = originalPtNode->getProbabilityFieldPos();
if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(mBuffer,
- probability, &probabilityFieldPos)) {
+ probabilityToWrite, &probabilityFieldPos)) {
return false;
}
} else {
// Make the node terminal and write the probability.
+ *outAddedNewUnigram = true;
int movedPos = mBuffer->getTailPosition();
if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) {
return false;
}
if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, originalPtNode,
originalPtNode->getParentPos(), codePoints, originalPtNode->getCodePointCount(),
- probability, &movedPos)) {
+ getUpdatedProbability(NOT_A_PROBABILITY /* originalProbability */, probability),
+ &movedPos)) {
return false;
}
}
@@ -463,72 +480,23 @@
return true;
}
-// TODO: Create a struct which contains header, body and etc... and use here as an argument.
-void DynamicPatriciaTrieWritingHelper::flushAllToFile(const char *const fileName,
- BufferWithExtendableBuffer *const dictHeader,
- BufferWithExtendableBuffer *const dictBody) const {
- const int tmpFileNameBufSize = strlen(fileName)
- + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */;
- // Name of a temporary file used for writing that is a connected string of original name and
- // TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE.
- char tmpFileName[tmpFileNameBufSize];
- snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName,
- TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
- FILE *const file = fopen(tmpFileName, "wb");
- if (!file) {
- AKLOGI("Dictionary file %s cannnot be opened.", tmpFileName);
- ASSERT(false);
- return;
- }
- // Write the dictionary header.
- if (!writeBufferToFilePointer(file, dictHeader)) {
- remove(tmpFileName);
- AKLOGI("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition());
- ASSERT(false);
- return;
- }
- // Write the dictionary body.
- if (!writeBufferToFilePointer(file, dictBody)) {
- remove(tmpFileName);
- AKLOGI("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition());
- ASSERT(false);
- return;
- }
- fclose(file);
- rename(tmpFileName, fileName);
-}
-
-// This closes file pointer when an error is caused and returns whether the writing was succeeded
-// or not.
-bool DynamicPatriciaTrieWritingHelper::writeBufferToFilePointer(FILE *const file,
- const BufferWithExtendableBuffer *const buffer) const {
- const int originalBufSize = buffer->getOriginalBufferSize();
- if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */),
- originalBufSize, 1, file) < 1) {
- fclose(file);
- return false;
- }
- const int additionalBufSize = buffer->getTailPosition() - buffer->getOriginalBufferSize();
- if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */),
- additionalBufSize, 1, file) < 1) {
- fclose(file);
- return false;
- }
- return true;
-}
-
bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
- BufferWithExtendableBuffer *const bufferToWrite) {
+ BufferWithExtendableBuffer *const bufferToWrite, int *const outUnigramCount,
+ int *const outBigramCount) {
DynamicPatriciaTrieReadingHelper readingHelper(mBuffer, mBigramPolicy, mShortcutPolicy);
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners
::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
- this, mBuffer);
+ this, mBuffer, mIsDecayingDict);
if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner(
&traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted)) {
return false;
}
+ if (mIsDecayingDict && traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
+ .getValidUnigramCount() > DecayingUtils::MAX_UNIGRAM_COUNT_AFTER_GC) {
+ // TODO: Remove more unigrams.
+ }
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability
@@ -538,6 +506,11 @@
return false;
}
+ if (mIsDecayingDict && traversePolicyToUpdateBigramProbability.getValidBigramEntryCount()
+ > DecayingUtils::MAX_BIGRAM_COUNT_AFTER_GC) {
+ // TODO: Remove more bigrams.
+ }
+
// Mapping from positions in mBuffer to positions in bufferToWrite.
DictPositionRelocationMap dictPositionRelocationMap;
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
@@ -551,7 +524,8 @@
// Create policy instance for the GCed dictionary.
DynamicShortcutListPolicy newDictShortcutPolicy(bufferToWrite);
- DynamicBigramListPolicy newDictBigramPolicy(bufferToWrite, &newDictShortcutPolicy);
+ DynamicBigramListPolicy newDictBigramPolicy(bufferToWrite, &newDictShortcutPolicy,
+ mIsDecayingDict);
// Create reading helper for the GCed dictionary.
DynamicPatriciaTrieReadingHelper newDictReadingHelper(bufferToWrite, &newDictBigramPolicy,
&newDictShortcutPolicy);
@@ -563,7 +537,18 @@
&traversePolicyToUpdateAllPositionFields)) {
return false;
}
+ *outUnigramCount = traversePolicyToUpdateAllPositionFields.getUnigramCount();
+ *outBigramCount = traversePolicyToUpdateAllPositionFields.getBigramCount();
return true;
}
+int DynamicPatriciaTrieWritingHelper::getUpdatedProbability(const int originalProbability,
+ const int newProbability) {
+ if (mIsDecayingDict) {
+ return DecayingUtils::getUpdatedUnigramProbability(originalProbability, newProbability);
+ } else {
+ return newProbability;
+ }
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
index 028fa60..ecee2cd 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
@@ -17,7 +17,6 @@
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H
-#include <cstdio>
#include <stdint.h>
#include "defines.h"
@@ -48,24 +47,30 @@
DISALLOW_COPY_AND_ASSIGN(DictPositionRelocationMap);
};
+ static const size_t MAX_DICTIONARY_SIZE;
+
DynamicPatriciaTrieWritingHelper(BufferWithExtendableBuffer *const buffer,
DynamicBigramListPolicy *const bigramPolicy,
- DynamicShortcutListPolicy *const shortcutPolicy)
- : mBuffer(buffer), mBigramPolicy(bigramPolicy), mShortcutPolicy(shortcutPolicy) {}
+ DynamicShortcutListPolicy *const shortcutPolicy, const bool isDecayingDict)
+ : mBuffer(buffer), mBigramPolicy(bigramPolicy), mShortcutPolicy(shortcutPolicy),
+ mIsDecayingDict(isDecayingDict) {}
~DynamicPatriciaTrieWritingHelper() {}
// Add a word to the dictionary. If the word already exists, update the probability.
bool addUnigramWord(DynamicPatriciaTrieReadingHelper *const readingHelper,
- const int *const wordCodePoints, const int codePointCount, const int probability);
+ const int *const wordCodePoints, const int codePointCount, const int probability,
+ bool *const outAddedNewUnigram);
// Add a bigram relation from word0Pos to word1Pos.
- bool addBigramWords(const int word0Pos, const int word1Pos, const int probability);
+ bool addBigramWords(const int word0Pos, const int word1Pos, const int probability,
+ bool *const outAddedNewBigram);
// Remove a bigram relation from word0Pos to word1Pos.
bool removeBigramWords(const int word0Pos, const int word1Pos);
- void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy);
+ void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy,
+ const int unigramCount, const int bigramCount);
void writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const fileName,
const HeaderPolicy *const headerPolicy);
@@ -85,12 +90,11 @@
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper);
static const int CHILDREN_POSITION_FIELD_SIZE;
- static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE;
- static const size_t MAX_DICTIONARY_SIZE;
BufferWithExtendableBuffer *const mBuffer;
DynamicBigramListPolicy *const mBigramPolicy;
DynamicShortcutListPolicy *const mShortcutPolicy;
+ const bool mIsDecayingDict;
bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate,
const int movedPos, const int bigramLinkedNodePos);
@@ -109,7 +113,7 @@
const int nodeCodePointCount, const int probability, int *const forwardLinkFieldPos);
bool setPtNodeProbability(const DynamicPatriciaTrieNodeReader *const originalNode,
- const int probability, const int *const codePoints);
+ const int probability, const int *const codePoints, bool *const outAddedNewUnigram);
bool createChildrenPtNodeArrayAndAChildPtNode(
const DynamicPatriciaTrieNodeReader *const parentNode, const int probability,
@@ -124,14 +128,10 @@
const int probabilityOfNewPtNode, const int *const newNodeCodePoints,
const int newNodeCodePointCount);
- void flushAllToFile(const char *const fileName,
- BufferWithExtendableBuffer *const dictHeader,
- BufferWithExtendableBuffer *const dictBody) const;
+ bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite,
+ int *const outUnigramCount, int *const outBigramCount);
- bool writeBufferToFilePointer(FILE *const file,
- const BufferWithExtendableBuffer *const buffer) const;
-
- bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite);
+ int getUpdatedProbability(const int originalProbability, const int newProbability);
};
} // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp
index 5a39837..30ff10c 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp
@@ -36,6 +36,16 @@
const int DynamicPatriciaTrieWritingUtils::PROBABILITY_FIELD_SIZE = 1;
const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1;
+/* static */ bool DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(
+ BufferWithExtendableBuffer *const buffer, const int rootPos) {
+ int writingPos = rootPos;
+ if (!writePtNodeArraySizeAndAdvancePosition(buffer, 0 /* arraySize */, &writingPos)) {
+ return false;
+ }
+ return writeForwardLinkPositionAndAdvancePosition(buffer, NOT_A_DICT_POS /* forwardLinkPos */,
+ &writingPos);
+}
+
/* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int forwardLinkPos,
int *const forwardLinkFieldPos) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h
index a37e9fb..af76bc6 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h
@@ -30,6 +30,8 @@
public:
static const int NODE_FLAG_FIELD_SIZE;
+ static bool writeEmptyDictionary(BufferWithExtendableBuffer *const buffer, const int rootPos);
+
static bool writeForwardLinkPositionAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int forwardLinkPos,
int *const forwardLinkFieldPos);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
index 47ace23..9ce9994 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
@@ -16,16 +16,17 @@
#include "suggest/policyimpl/dictionary/header/header_policy.h"
-#include <cstddef>
-#include <cstdio>
-#include <ctime>
-
namespace latinime {
+// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE";
-const char *const HeaderPolicy::USES_FORGETTING_CURVE_KEY = "USES_FORGETTING_CURVE";
+// TODO: Change attribute string to "IS_DECAYING_DICT".
+const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE";
const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date";
-const float HeaderPolicy::DEFAULT_MULTIPLE_WORD_COST_MULTIPLIER = 1.0f;
+const char *const HeaderPolicy::UNIGRAM_COUNT_KEY = "UNIGRAM_COUNT";
+const char *const HeaderPolicy::BIGRAM_COUNT_KEY = "BIGRAM_COUNT";
+const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE";
+const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100;
const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f;
// Used for logging. Question mark is used to indicate that the key is not found.
@@ -37,7 +38,7 @@
return;
}
std::vector<int> keyCodePointVector;
- insertCharactersIntoVector(key, &keyCodePointVector);
+ HeaderReadWriteUtils::insertCharactersIntoVector(key, &keyCodePointVector);
HeaderReadWriteUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyCodePointVector);
if (it == mAttributeMap.end()) {
// The key was not found.
@@ -53,51 +54,17 @@
}
float HeaderPolicy::readMultipleWordCostMultiplier() const {
- int attributeValue = 0;
- if (getAttributeValueAsInt(MULTIPLE_WORDS_DEMOTION_RATE_KEY, &attributeValue)) {
- if (attributeValue <= 0) {
- return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
- }
- return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(attributeValue);
- } else {
- return DEFAULT_MULTIPLE_WORD_COST_MULTIPLIER;
+ const int demotionRate = HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ MULTIPLE_WORDS_DEMOTION_RATE_KEY, DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE);
+ if (demotionRate <= 0) {
+ return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
}
-}
-
-bool HeaderPolicy::readUsesForgettingCurveFlag() const {
- int attributeValue = 0;
- if (getAttributeValueAsInt(USES_FORGETTING_CURVE_KEY, &attributeValue)) {
- return attributeValue != 0;
- } else {
- return false;
- }
-}
-
-// Returns S_INT_MIN when the key is not found or the value is invalid.
-int HeaderPolicy::readLastUpdatedTime() const {
- int attributeValue = 0;
- if (getAttributeValueAsInt(LAST_UPDATED_TIME_KEY, &attributeValue)) {
- return attributeValue;
- } else {
- return S_INT_MIN;
- }
-}
-
-// Returns whether the key is found or not and stores the found value into outValue.
-bool HeaderPolicy::getAttributeValueAsInt(const char *const key, int *const outValue) const {
- std::vector<int> keyVector;
- insertCharactersIntoVector(key, &keyVector);
- HeaderReadWriteUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyVector);
- if (it == mAttributeMap.end()) {
- // The key was not found.
- return false;
- }
- *outValue = parseIntAttributeValue(&(it->second));
- return true;
+ return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(demotionRate);
}
bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
- const bool updatesLastUpdatedTime) const {
+ const bool updatesLastUpdatedTime, const int unigramCount, const int bigramCount,
+ const int extendedRegionSize) const {
int writingPos = 0;
if (!HeaderReadWriteUtils::writeDictionaryVersion(bufferToWrite, mDictFormatVersion,
&writingPos)) {
@@ -113,26 +80,19 @@
&writingPos)) {
return false;
}
+ HeaderReadWriteUtils::AttributeMap attributeMapTowrite(mAttributeMap);
+ HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, UNIGRAM_COUNT_KEY, unigramCount);
+ HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, BIGRAM_COUNT_KEY, bigramCount);
+ HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, EXTENDED_REGION_SIZE_KEY,
+ extendedRegionSize);
if (updatesLastUpdatedTime) {
// Set current time as a last updated time.
- HeaderReadWriteUtils::AttributeMap attributeMapTowrite(mAttributeMap);
- std::vector<int> updatedTimekey;
- insertCharactersIntoVector(LAST_UPDATED_TIME_KEY, &updatedTimekey);
- const time_t currentTime = time(NULL);
- std::vector<int> updatedTimeValue;
- char charBuf[LARGEST_INT_DIGIT_COUNT + 1];
- snprintf(charBuf, LARGEST_INT_DIGIT_COUNT + 1, "%ld", currentTime);
- insertCharactersIntoVector(charBuf, &updatedTimeValue);
- attributeMapTowrite[updatedTimekey] = updatedTimeValue;
- if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &attributeMapTowrite,
- &writingPos)) {
- return false;
- }
- } else {
- if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &mAttributeMap,
- &writingPos)) {
- return false;
- }
+ HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, LAST_UPDATED_TIME_KEY,
+ time(0));
+ }
+ if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &attributeMapTowrite,
+ &writingPos)) {
+ return false;
}
// Writes an actual header size.
if (!HeaderReadWriteUtils::writeDictionaryHeaderSize(bufferToWrite, writingPos,
@@ -149,30 +109,4 @@
return attributeMap;
}
-/* static */ int HeaderPolicy::parseIntAttributeValue(
- const std::vector<int> *const attributeValue) {
- int value = 0;
- bool isNegative = false;
- for (size_t i = 0; i < attributeValue->size(); ++i) {
- if (i == 0 && attributeValue->at(i) == '-') {
- isNegative = true;
- } else {
- if (!isdigit(attributeValue->at(i))) {
- // If not a number, return S_INT_MIN
- return S_INT_MIN;
- }
- value *= 10;
- value += attributeValue->at(i) - '0';
- }
- }
- return isNegative ? -value : value;
-}
-
-/* static */ void HeaderPolicy::insertCharactersIntoVector(const char *const characters,
- std::vector<int> *const vector) {
- for (int i = 0; characters[i]; ++i) {
- vector->push_back(characters[i]);
- }
-}
-
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
index 6b396f3..4261667 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
@@ -17,7 +17,7 @@
#ifndef LATINIME_HEADER_POLICY_H
#define LATINIME_HEADER_POLICY_H
-#include <cctype>
+#include <ctime>
#include <stdint.h>
#include "defines.h"
@@ -29,15 +29,36 @@
class HeaderPolicy : public DictionaryHeaderStructurePolicy {
public:
- explicit HeaderPolicy(const uint8_t *const dictBuf, const int dictSize)
- : mDictBuf(dictBuf),
- mDictFormatVersion(FormatUtils::detectFormatVersion(dictBuf, dictSize)),
+ // Reads information from existing dictionary buffer.
+ HeaderPolicy(const uint8_t *const dictBuf, const int dictSize)
+ : mDictFormatVersion(FormatUtils::detectFormatVersion(dictBuf, dictSize)),
mDictionaryFlags(HeaderReadWriteUtils::getFlags(dictBuf)),
mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)),
- mAttributeMap(createAttributeMapAndReadAllAttributes(mDictBuf)),
+ mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
- mUsesForgettingCurve(readUsesForgettingCurveFlag()),
- mLastUpdatedTime(readLastUpdatedTime()) {}
+ mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
+ IS_DECAYING_DICT_KEY, false /* defaultValue */)),
+ mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)),
+ mUnigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ UNIGRAM_COUNT_KEY, 0 /* defaultValue */)),
+ mBigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ BIGRAM_COUNT_KEY, 0 /* defaultValue */)),
+ mExtendedRegionSize(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ EXTENDED_REGION_SIZE_KEY, 0 /* defaultValue */)) {}
+
+ // Constructs header information using an attribute map.
+ HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion,
+ const HeaderReadWriteUtils::AttributeMap *const attributeMap)
+ : mDictFormatVersion(dictFormatVersion),
+ mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
+ attributeMap)), mSize(0), mAttributeMap(*attributeMap),
+ mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
+ mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
+ IS_DECAYING_DICT_KEY, false /* defaultValue */)),
+ mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
+ LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)),
+ mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0) {}
~HeaderPolicy() {}
@@ -61,53 +82,60 @@
return mMultiWordCostMultiplier;
}
- AK_FORCE_INLINE bool usesForgettingCurve() const {
- return mUsesForgettingCurve;
+ AK_FORCE_INLINE bool isDecayingDict() const {
+ return mIsDecayingDict;
}
AK_FORCE_INLINE int getLastUpdatedTime() const {
return mLastUpdatedTime;
}
+ AK_FORCE_INLINE int getUnigramCount() const {
+ return mUnigramCount;
+ }
+
+ AK_FORCE_INLINE int getBigramCount() const {
+ return mBigramCount;
+ }
+
+ AK_FORCE_INLINE int getExtendedRegionSize() const {
+ return mExtendedRegionSize;
+ }
+
void readHeaderValueOrQuestionMark(const char *const key,
int *outValue, int outValueSize) const;
bool writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
- const bool updatesLastUpdatedTime) const;
+ const bool updatesLastUpdatedTime, const int unigramCount,
+ const int bigramCount, const int extendedRegionSize) const;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy);
static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
- static const char *const USES_FORGETTING_CURVE_KEY;
+ static const char *const IS_DECAYING_DICT_KEY;
static const char *const LAST_UPDATED_TIME_KEY;
- static const float DEFAULT_MULTIPLE_WORD_COST_MULTIPLIER;
+ static const char *const UNIGRAM_COUNT_KEY;
+ static const char *const BIGRAM_COUNT_KEY;
+ static const char *const EXTENDED_REGION_SIZE_KEY;
+ static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE;
static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE;
- const uint8_t *const mDictBuf;
const FormatUtils::FORMAT_VERSION mDictFormatVersion;
const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags;
const int mSize;
HeaderReadWriteUtils::AttributeMap mAttributeMap;
const float mMultiWordCostMultiplier;
- const bool mUsesForgettingCurve;
+ const bool mIsDecayingDict;
const int mLastUpdatedTime;
+ const int mUnigramCount;
+ const int mBigramCount;
+ const int mExtendedRegionSize;
float readMultipleWordCostMultiplier() const;
- bool readUsesForgettingCurveFlag() const;
-
- int readLastUpdatedTime() const;
-
- bool getAttributeValueAsInt(const char *const key, int *const outValue) const;
-
static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes(
const uint8_t *const dictBuf);
-
- static int parseIntAttributeValue(const std::vector<int> *const attributeValue);
-
- static void insertCharactersIntoVector(
- const char *const characters, std::vector<int> *const vector);
};
} // namespace latinime
#endif /* LATINIME_HEADER_POLICY_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
index 80fe886..2694ce8 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
@@ -16,6 +16,8 @@
#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h"
+#include <cctype>
+#include <cstdio>
#include <vector>
#include "defines.h"
@@ -43,6 +45,13 @@
const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
+// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
+const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE";
+const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
+ "REQUIRES_GERMAN_UMLAUT_PROCESSING";
+const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY =
+ "REQUIRES_FRENCH_LIGATURE_PROCESSING";
+
/* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) {
// See the format of the header in the comment in
// BinaryDictionaryFormatUtils::detectFormatVersion()
@@ -56,6 +65,22 @@
HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE);
}
+/* static */ HeaderReadWriteUtils::DictionaryFlags
+ HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
+ const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
+ const bool requiresGermanUmlautProcessing = readBoolAttributeValue(attributeMap,
+ REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false /* defaultValue */);
+ const bool requiresFrenchLigatureProcessing = readBoolAttributeValue(attributeMap,
+ REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY, false /* defaultValue */);
+ const bool supportsDynamicUpdate = readBoolAttributeValue(attributeMap,
+ SUPPORTS_DYNAMIC_UPDATE_KEY, false /* defaultValue */);
+ DictionaryFlags dictflags = NO_FLAGS;
+ dictflags |= requiresGermanUmlautProcessing ? GERMAN_UMLAUT_PROCESSING_FLAG : 0;
+ dictflags |= requiresFrenchLigatureProcessing ? FRENCH_LIGATURE_PROCESSING_FLAG : 0;
+ dictflags |= supportsDynamicUpdate ? SUPPORTS_DYNAMIC_UPDATE_FLAG : 0;
+ return dictflags;
+}
+
/* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf,
AttributeMap *const headerAttributes) {
const int headerSize = getHeaderSize(dictBuf);
@@ -128,4 +153,72 @@
return true;
}
+/* static */ void HeaderReadWriteUtils::setBoolAttribute(AttributeMap *const headerAttributes,
+ const char *const key, const bool value) {
+ setIntAttribute(headerAttributes, key, value ? 1 : 0);
+}
+
+/* static */ void HeaderReadWriteUtils::setIntAttribute(AttributeMap *const headerAttributes,
+ const char *const key, const int value) {
+ AttributeMap::key_type keyVector;
+ insertCharactersIntoVector(key, &keyVector);
+ setIntAttributeInner(headerAttributes, &keyVector, value);
+}
+
+/* static */ void HeaderReadWriteUtils::setIntAttributeInner(AttributeMap *const headerAttributes,
+ const AttributeMap::key_type *const key, const int value) {
+ AttributeMap::mapped_type valueVector;
+ char charBuf[LARGEST_INT_DIGIT_COUNT + 1];
+ snprintf(charBuf, LARGEST_INT_DIGIT_COUNT + 1, "%d", value);
+ insertCharactersIntoVector(charBuf, &valueVector);
+ (*headerAttributes)[*key] = valueVector;
+}
+
+/* static */ bool HeaderReadWriteUtils::readBoolAttributeValue(
+ const AttributeMap *const headerAttributes, const char *const key,
+ const bool defaultValue) {
+ const int intDefaultValue = defaultValue ? 1 : 0;
+ const int intValue = readIntAttributeValue(headerAttributes, key, intDefaultValue);
+ return intValue != 0;
+}
+
+/* static */ int HeaderReadWriteUtils::readIntAttributeValue(
+ const AttributeMap *const headerAttributes, const char *const key,
+ const int defaultValue) {
+ AttributeMap::key_type keyVector;
+ insertCharactersIntoVector(key, &keyVector);
+ return readIntAttributeValueInner(headerAttributes, &keyVector, defaultValue);
+}
+
+/* static */ int HeaderReadWriteUtils::readIntAttributeValueInner(
+ const AttributeMap *const headerAttributes, const AttributeMap::key_type *const key,
+ const int defaultValue) {
+ AttributeMap::const_iterator it = headerAttributes->find(*key);
+ if (it != headerAttributes->end()) {
+ int value = 0;
+ bool isNegative = false;
+ for (size_t i = 0; i < it->second.size(); ++i) {
+ if (i == 0 && it->second.at(i) == '-') {
+ isNegative = true;
+ } else {
+ if (!isdigit(it->second.at(i))) {
+ // If not a number.
+ return defaultValue;
+ }
+ value *= 10;
+ value += it->second.at(i) - '0';
+ }
+ }
+ return isNegative ? -value : value;
+ }
+ return defaultValue;
+}
+
+/* static */ void HeaderReadWriteUtils::insertCharactersIntoVector(const char *const characters,
+ std::vector<int> *const vector) {
+ for (int i = 0; characters[i]; ++i) {
+ vector->push_back(characters[i]);
+ }
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h
index 6cce733..2259683 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h
@@ -54,6 +54,9 @@
+ HEADER_SIZE_FIELD_SIZE;
}
+ static DictionaryFlags createAndGetDictionaryFlagsUsingAttributeMap(
+ const HeaderReadWriteUtils::AttributeMap *const attributeMap);
+
static void fetchAllHeaderAttributes(const uint8_t *const dictBuf,
AttributeMap *const headerAttributes);
@@ -69,6 +72,24 @@
static bool writeHeaderAttributes(BufferWithExtendableBuffer *const buffer,
const AttributeMap *const headerAttributes, int *const writingPos);
+ /**
+ * Methods for header attributes.
+ */
+ static void setBoolAttribute(AttributeMap *const headerAttributes,
+ const char *const key, const bool value);
+
+ static void setIntAttribute(AttributeMap *const headerAttributes,
+ const char *const key, const int value);
+
+ static bool readBoolAttributeValue(const AttributeMap *const headerAttributes,
+ const char *const key, const bool defaultValue);
+
+ static int readIntAttributeValue(const AttributeMap *const headerAttributes,
+ const char *const key, const int defaultValue);
+
+ static void insertCharactersIntoVector(const char *const characters,
+ AttributeMap::key_type *const key);
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderReadWriteUtils);
@@ -87,7 +108,16 @@
static const DictionaryFlags GERMAN_UMLAUT_PROCESSING_FLAG;
static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG;
static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG;
- static const DictionaryFlags CONTAINS_BIGRAMS_FLAG;
+
+ static const char *const SUPPORTS_DYNAMIC_UPDATE_KEY;
+ static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY;
+ static const char *const REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY;
+
+ static void setIntAttributeInner(AttributeMap *const headerAttributes,
+ const AttributeMap::key_type *const key, const int value);
+
+ static int readIntAttributeValueInner(const AttributeMap *const headerAttributes,
+ const AttributeMap::key_type *const key, const int defaultValue);
};
}
#endif /* LATINIME_HEADER_READ_WRITE_UTILS_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp
index 5269795..8a84bd2 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp
@@ -31,9 +31,21 @@
return;
}
int nextPos = dicNode->getChildrenPos();
+ if (nextPos < 0 || nextPos >= mDictBufferSize) {
+ AKLOGE("Children PtNode array position is invalid. pos: %d, dict size: %d",
+ nextPos, mDictBufferSize);
+ ASSERT(false);
+ return;
+ }
const int childCount = PatriciaTrieReadingUtils::getPtNodeArraySizeAndAdvancePosition(
mDictRoot, &nextPos);
for (int i = 0; i < childCount; i++) {
+ if (nextPos < 0 || nextPos >= mDictBufferSize) {
+ AKLOGE("Child PtNode position is invalid. pos: %d, dict size: %d, childCount: %d / %d",
+ nextPos, mDictBufferSize, i, childCount);
+ ASSERT(false);
+ return;
+ }
nextPos = createAndGetLeavingChildNode(dicNode, nextPos, childDicNodes);
}
}
@@ -404,6 +416,11 @@
if (PatriciaTrieReadingUtils::hasBigrams(flags)) {
getBigramsStructurePolicy()->skipAllBigrams(&pos);
}
+ if (mergedNodeCodePointCount <= 0) {
+ AKLOGE("Empty PtNode is not allowed. Code point count: %d", mergedNodeCodePointCount);
+ ASSERT(false);
+ return pos;
+ }
childDicNodes->pushLeavingChild(dicNode, ptNodePos, childrenPos, probability,
PatriciaTrieReadingUtils::isTerminal(flags),
PatriciaTrieReadingUtils::hasChildrenInFlags(flags),
diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
index 19155f9..8d88c68 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h
@@ -36,6 +36,7 @@
PatriciaTriePolicy(const MmappedBuffer *const buffer)
: mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer(), buffer->getBufferSize()),
mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()),
+ mDictBufferSize(mBuffer->getBufferSize() - mHeaderPolicy.getSize()),
mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}
~PatriciaTriePolicy() {
@@ -106,18 +107,27 @@
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
}
- bool needsToRunGC() const {
+ bool needsToRunGC(const bool mindsBlockByGC) const {
// This method should not be called for non-updatable dictionary.
AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
return false;
}
+ void getProperty(const char *const query, char *const outResult,
+ const int maxResultLength) const {
+ // getProperty is not supported for this class.
+ if (maxResultLength > 0) {
+ outResult[0] = '\0';
+ }
+ }
+
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy);
const MmappedBuffer *const mBuffer;
const HeaderPolicy mHeaderPolicy;
const uint8_t *const mDictRoot;
+ const int mDictBufferSize;
const BigramListPolicy mBigramListPolicy;
const ShortcutListPolicy mShortcutListPolicy;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp
index 1316b42..7df5581 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp
@@ -71,8 +71,17 @@
length = ByteArrayUtils::readStringAndAdvancePosition(buffer, maxLength, outBuffer,
pos);
} else {
- if (maxLength > 0) {
- outBuffer[0] = getCodePointAndAdvancePosition(buffer, pos);
+ const int codePoint = getCodePointAndAdvancePosition(buffer, pos);
+ if (codePoint == NOT_A_CODE_POINT) {
+ // CAVEAT: codePoint == NOT_A_CODE_POINT means the code point is
+ // CHARACTER_ARRAY_TERMINATOR. The code point must not be CHARACTER_ARRAY_TERMINATOR
+ // when the PtNode has a single code point.
+ length = 0;
+ AKLOGE("codePoint is NOT_A_CODE_POINT. pos: %d, codePoint: 0x%x, buffer[pos - 1]: 0x%x",
+ *pos - 1, codePoint, buffer[*pos - 1]);
+ ASSERT(false);
+ } else if (maxLength > 0) {
+ outBuffer[0] = codePoint;
length = 1;
}
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
index 17d2e39..9dc3482 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
@@ -42,6 +42,10 @@
return mOriginalBufferSize + mUsedAdditionalBufferSize;
}
+ AK_FORCE_INLINE int getUsedAdditionalBufferSize() const {
+ return mUsedAdditionalBufferSize;
+ }
+
/**
* For reading.
*/
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.cpp
new file mode 100644
index 0000000..942a742
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.cpp
@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/policyimpl/dictionary/utils/decaying_utils.h"
+
+#include "suggest/policyimpl/dictionary/utils/probability_utils.h"
+
+namespace latinime {
+
+const int DecayingUtils::MAX_UNIGRAM_COUNT = 12000;
+const int DecayingUtils::MAX_UNIGRAM_COUNT_AFTER_GC = 10000;
+const int DecayingUtils::MAX_BIGRAM_COUNT = 12000;
+const int DecayingUtils::MAX_BIGRAM_COUNT_AFTER_GC = 10000;
+
+const int DecayingUtils::MAX_COMPUTED_PROBABILITY = 127;
+const int DecayingUtils::MAX_UNIGRAM_PROBABILITY = 120;
+const int DecayingUtils::MIN_VALID_UNIGRAM_PROBABILITY = 24;
+const int DecayingUtils::UNIGRAM_PROBABILITY_STEP = 8;
+const int DecayingUtils::MAX_BIGRAM_PROBABILITY_DELTA = 15;
+const int DecayingUtils::MIN_VALID_BIGRAM_PROBABILITY_DELTA = 3;
+const int DecayingUtils::BIGRAM_PROBABILITY_DELTA_STEP = 1;
+
+/* static */ int DecayingUtils::getProbability(const int encodedUnigramProbability,
+ const int encodedBigramProbabilityDelta) {
+ if (encodedUnigramProbability == NOT_A_PROBABILITY) {
+ return NOT_A_PROBABILITY;
+ } else if (encodedBigramProbabilityDelta == NOT_A_PROBABILITY) {
+ const int rawProbability = ProbabilityUtils::backoff(decodeUnigramProbability(
+ encodedUnigramProbability));
+ return min(getDecayedProbability(rawProbability), MAX_COMPUTED_PROBABILITY);
+ } else {
+ const int rawProbability = ProbabilityUtils::computeProbabilityForBigram(
+ decodeUnigramProbability(encodedUnigramProbability),
+ decodeBigramProbabilityDelta(encodedBigramProbabilityDelta));
+ return min(getDecayedProbability(rawProbability), MAX_COMPUTED_PROBABILITY);
+ }
+}
+
+/* static */ int DecayingUtils::getUpdatedUnigramProbability(const int originalEncodedProbability,
+ const int newProbability) {
+ if (originalEncodedProbability == NOT_A_PROBABILITY) {
+ // The unigram is not in this dictionary.
+ if (newProbability == NOT_A_PROBABILITY) {
+ // The unigram is not in other dictionaries.
+ return 0;
+ } else {
+ return MIN_VALID_UNIGRAM_PROBABILITY;
+ }
+ } else {
+ if (newProbability != NOT_A_PROBABILITY
+ && originalEncodedProbability < MIN_VALID_UNIGRAM_PROBABILITY) {
+ return MIN_VALID_UNIGRAM_PROBABILITY;
+ }
+ return min(originalEncodedProbability + UNIGRAM_PROBABILITY_STEP, MAX_UNIGRAM_PROBABILITY);
+ }
+}
+
+/* static */ int DecayingUtils::getUnigramProbabilityToSave(const int encodedProbability) {
+ return max(encodedProbability - UNIGRAM_PROBABILITY_STEP, 0);
+}
+
+/* static */ int DecayingUtils::getBigramProbabilityDeltaToSave(const int encodedProbabilityDelta) {
+ return max(encodedProbabilityDelta - BIGRAM_PROBABILITY_DELTA_STEP, 0);
+}
+
+/* static */ int DecayingUtils::getUpdatedBigramProbabilityDelta(
+ const int originalEncodedProbabilityDelta, const int newProbability) {
+ if (originalEncodedProbabilityDelta == NOT_A_PROBABILITY) {
+ // The bigram relation is not in this dictionary.
+ if (newProbability == NOT_A_PROBABILITY) {
+ // The bigram target is not in other dictionaries.
+ return 0;
+ } else {
+ return MIN_VALID_BIGRAM_PROBABILITY_DELTA;
+ }
+ } else {
+ if (newProbability != NOT_A_PROBABILITY
+ && originalEncodedProbabilityDelta < MIN_VALID_BIGRAM_PROBABILITY_DELTA) {
+ return MIN_VALID_BIGRAM_PROBABILITY_DELTA;
+ }
+ return min(originalEncodedProbabilityDelta + BIGRAM_PROBABILITY_DELTA_STEP,
+ MAX_BIGRAM_PROBABILITY_DELTA);
+ }
+}
+
+/* static */ int DecayingUtils::isValidUnigram(const int encodedUnigramProbability) {
+ return encodedUnigramProbability >= MIN_VALID_UNIGRAM_PROBABILITY;
+}
+
+/* static */ int DecayingUtils::isValidBigram(const int encodedBigramProbabilityDelta) {
+ return encodedBigramProbabilityDelta >= MIN_VALID_BIGRAM_PROBABILITY_DELTA;
+}
+
+/* static */ int DecayingUtils::decodeUnigramProbability(const int encodedProbability) {
+ const int probability = encodedProbability - MIN_VALID_UNIGRAM_PROBABILITY;
+ if (probability < 0) {
+ return NOT_A_PROBABILITY;
+ } else {
+ return min(probability, MAX_UNIGRAM_PROBABILITY);
+ }
+}
+
+/* static */ int DecayingUtils::decodeBigramProbabilityDelta(const int encodedProbabilityDelta) {
+ const int probabilityDelta = encodedProbabilityDelta - MIN_VALID_BIGRAM_PROBABILITY_DELTA;
+ if (probabilityDelta < 0) {
+ return NOT_A_PROBABILITY;
+ } else {
+ return min(probabilityDelta, MAX_BIGRAM_PROBABILITY_DELTA);
+ }
+}
+
+/* static */ int DecayingUtils::getDecayedProbability(const int rawProbability) {
+ return rawProbability;
+}
+
+} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.h
new file mode 100644
index 0000000..1ca0391
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_DECAYING_UTILS_H
+#define LATINIME_DECAYING_UTILS_H
+
+#include "defines.h"
+
+namespace latinime {
+
+// TODO: Check the elapsed time and decrease the probability depending on the time. Time field is
+// required to introduced to each terminal PtNode and bigram entry.
+// TODO: Quit using bigram probability to indicate the delta.
+// TODO: Quit using bigram probability delta.
+class DecayingUtils {
+ public:
+ static const int MAX_UNIGRAM_COUNT;
+ static const int MAX_UNIGRAM_COUNT_AFTER_GC;
+ static const int MAX_BIGRAM_COUNT;
+ static const int MAX_BIGRAM_COUNT_AFTER_GC;
+
+ static int getProbability(const int encodedUnigramProbability,
+ const int encodedBigramProbabilityDelta);
+
+ static int getUpdatedUnigramProbability(const int originalEncodedProbability,
+ const int newProbability);
+
+ static int getUpdatedBigramProbabilityDelta(const int originalEncodedProbabilityDelta,
+ const int newProbability);
+
+ static int isValidUnigram(const int encodedUnigramProbability);
+
+ static int isValidBigram(const int encodedProbabilityDelta);
+
+ static int getUnigramProbabilityToSave(const int encodedProbability);
+
+ static int getBigramProbabilityDeltaToSave(const int encodedProbabilityDelta);
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(DecayingUtils);
+
+ static const int MAX_COMPUTED_PROBABILITY;
+ static const int MAX_UNIGRAM_PROBABILITY;
+ static const int MIN_VALID_UNIGRAM_PROBABILITY;
+ static const int UNIGRAM_PROBABILITY_STEP;
+ static const int MAX_BIGRAM_PROBABILITY_DELTA;
+ static const int MIN_VALID_BIGRAM_PROBABILITY_DELTA;
+ static const int BIGRAM_PROBABILITY_DELTA_STEP;
+
+ static int decodeUnigramProbability(const int encodedProbability);
+
+ static int decodeBigramProbabilityDelta(const int encodedProbability);
+
+ static int getDecayedProbability(const int rawProbability);
+};
+} // namespace latinime
+#endif /* LATINIME_DECAYING_UTILS_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
new file mode 100644
index 0000000..f22e94c
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
+
+#include <cstdio>
+#include <cstring>
+
+#include "suggest/policyimpl/dictionary/header/header_policy.h"
+#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h"
+#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+#include "suggest/policyimpl/dictionary/utils/format_utils.h"
+
+namespace latinime {
+
+const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = ".tmp";
+
+/* static */ bool DictFileWritingUtils::createEmptyDictFile(const char *const filePath,
+ const int dictVersion, const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
+ switch (dictVersion) {
+ case 3:
+ return createEmptyV3DictFile(filePath, attributeMap);
+ default:
+ // Only version 3 dictionary is supported for now.
+ return false;
+ }
+}
+
+/* static */ bool DictFileWritingUtils::createEmptyV3DictFile(const char *const filePath,
+ const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
+ BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
+ HeaderPolicy headerPolicy(FormatUtils::VERSION_3, attributeMap);
+ headerPolicy.writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */,
+ 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);
+ BufferWithExtendableBuffer bodyBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
+ if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(&bodyBuffer, 0 /* rootPos */)) {
+ return false;
+ }
+ return flushAllHeaderAndBodyToFile(filePath, &headerBuffer, &bodyBuffer);
+}
+
+/* static */ bool DictFileWritingUtils::flushAllHeaderAndBodyToFile(const char *const filePath,
+ BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody) {
+ const int tmpFileNameBufSize = strlen(filePath)
+ + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */;
+ // Name of a temporary file used for writing that is a connected string of original name and
+ // TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE.
+ char tmpFileName[tmpFileNameBufSize];
+ snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", filePath,
+ TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
+ FILE *const file = fopen(tmpFileName, "wb");
+ if (!file) {
+ AKLOGE("Dictionary file %s cannnot be opened.", tmpFileName);
+ ASSERT(false);
+ return false;
+ }
+ // Write the dictionary header.
+ if (!writeBufferToFile(file, dictHeader)) {
+ remove(tmpFileName);
+ AKLOGE("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition());
+ ASSERT(false);
+ return false;
+ }
+ // Write the dictionary body.
+ if (!writeBufferToFile(file, dictBody)) {
+ remove(tmpFileName);
+ AKLOGE("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition());
+ ASSERT(false);
+ return false;
+ }
+ fclose(file);
+ rename(tmpFileName, filePath);
+ return true;
+}
+
+// This closes file pointer when an error is caused and returns whether the writing was succeeded
+// or not.
+/* static */ bool DictFileWritingUtils::writeBufferToFile(FILE *const file,
+ const BufferWithExtendableBuffer *const buffer) {
+ const int originalBufSize = buffer->getOriginalBufferSize();
+ if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */),
+ originalBufSize, 1, file) < 1) {
+ fclose(file);
+ return false;
+ }
+ const int additionalBufSize = buffer->getUsedAdditionalBufferSize();
+ if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */),
+ additionalBufSize, 1, file) < 1) {
+ fclose(file);
+ return false;
+ }
+ return true;
+}
+
+} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
new file mode 100644
index 0000000..bd4ac66
--- /dev/null
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_DICT_FILE_WRITING_UTILS_H
+#define LATINIME_DICT_FILE_WRITING_UTILS_H
+
+#include <cstdio>
+
+#include "defines.h"
+#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h"
+
+namespace latinime {
+
+class BufferWithExtendableBuffer;
+
+class DictFileWritingUtils {
+ public:
+ static bool createEmptyDictFile(const char *const filePath, const int dictVersion,
+ const HeaderReadWriteUtils::AttributeMap *const attributeMap);
+
+ static bool flushAllHeaderAndBodyToFile(const char *const filePath,
+ BufferWithExtendableBuffer *const dictHeader,
+ BufferWithExtendableBuffer *const dictBody);
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils);
+
+ static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE;
+
+ static bool createEmptyV3DictFile(const char *const filePath,
+ const HeaderReadWriteUtils::AttributeMap *const attributeMap);
+
+ static bool writeBufferToFile(FILE *const file,
+ const BufferWithExtendableBuffer *const buffer);
+};
+} // namespace latinime
+#endif /* LATINIME_DICT_FILE_WRITING_UTILS_H */
diff --git a/native/jni/src/suggest/policyimpl/typing/typing_weighting.cpp b/native/jni/src/suggest/policyimpl/typing/typing_weighting.cpp
index 408b12a..5b6b5e8 100644
--- a/native/jni/src/suggest/policyimpl/typing/typing_weighting.cpp
+++ b/native/jni/src/suggest/policyimpl/typing/typing_weighting.cpp
@@ -47,7 +47,7 @@
case CT_TERMINAL_INSERTION:
case CT_TRANSPOSITION:
return ET_EDIT_CORRECTION;
- case CT_NEW_WORD_SPACE_OMITTION:
+ case CT_NEW_WORD_SPACE_OMISSION:
case CT_NEW_WORD_SPACE_SUBSTITUTION:
return ET_NEW_WORD;
case CT_TERMINAL:
diff --git a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h
index b6aa858..9f0a331 100644
--- a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h
+++ b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h
@@ -74,7 +74,8 @@
// Note: min() required since length can be MAX_POINT_TO_KEY_LENGTH for characters not on
// the keyboard (like accented letters)
const float normalizedSquaredLength = traverseSession->getProximityInfoState(0)
- ->getPointToKeyLength(pointIndex, dicNode->getNodeCodePoint());
+ ->getPointToKeyLength(pointIndex,
+ CharUtils::toBaseLowerCase(dicNode->getNodeCodePoint()));
const float normalizedDistance = TouchPositionCorrectionUtils::getSweetSpotFactor(
traverseSession->isTouchPositionCorrectionEnabled(), normalizedSquaredLength);
const float weightedDistance = ScoringParams::DISTANCE_WEIGHT_LENGTH * normalizedDistance;
@@ -113,10 +114,10 @@
const int16_t parentPointIndex = parentDicNode->getInputIndex(0);
const int prevCodePoint = parentDicNode->getNodeCodePoint();
const float distance1 = traverseSession->getProximityInfoState(0)->getPointToKeyLength(
- parentPointIndex + 1, prevCodePoint);
+ parentPointIndex + 1, CharUtils::toBaseLowerCase(prevCodePoint));
const int codePoint = dicNode->getNodeCodePoint();
const float distance2 = traverseSession->getProximityInfoState(0)->getPointToKeyLength(
- parentPointIndex, codePoint);
+ parentPointIndex, CharUtils::toBaseLowerCase(codePoint));
const float distance = distance1 + distance2;
const float weightedLengthDistance =
distance * ScoringParams::DISTANCE_WEIGHT_LENGTH;
@@ -133,7 +134,7 @@
const bool existsAdjacentProximityChars = traverseSession->getProximityInfoState(0)
->existsAdjacentProximityChars(insertedPointIndex);
const float dist = traverseSession->getProximityInfoState(0)->getPointToKeyLength(
- insertedPointIndex + 1, dicNode->getNodeCodePoint());
+ insertedPointIndex + 1, CharUtils::toBaseLowerCase(dicNode->getNodeCodePoint()));
const float weightedDistance = dist * ScoringParams::DISTANCE_WEIGHT_LENGTH;
const bool singleChar = dicNode->getNodeCodePointCount() == 1;
float cost = (singleChar ? ScoringParams::INSERTION_COST_FIRST_CHAR : 0.0f);
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
new file mode 100644
index 0000000..cf85d97
--- /dev/null
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin;
+
+import android.test.AndroidTestCase;
+import android.test.suitebuilder.annotation.LargeTest;
+
+import com.android.inputmethod.latin.makedict.FormatSpec;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+@LargeTest
+public class BinaryDictionaryDecayingTests extends AndroidTestCase {
+ private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
+ private static final String TEST_LOCALE = "test";
+
+ private static final int DUMMY_PROBABILITY = 0;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ private void forcePassingShortTime(final BinaryDictionary binaryDictionary) {
+ binaryDictionary.flushWithGC();
+ }
+
+ private void forcePassingLongTime(final BinaryDictionary binaryDictionary) {
+ // Currently, probabilities are decayed when GC is run. All entries that have never been
+ // typed in 32 GCs are removed.
+ final int count = 32;
+ for (int i = 0; i < count; i++) {
+ binaryDictionary.flushWithGC();
+ }
+ }
+
+ private File createEmptyDictionaryAndGetFile(final String filename) throws IOException {
+ final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION,
+ getContext().getCacheDir());
+ Map<String, String> attributeMap = new HashMap<String, String>();
+ attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+ attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+ if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
+ 3 /* dictVersion */, attributeMap)) {
+ return file;
+ } else {
+ throw new IOException("Empty dictionary cannot be created.");
+ }
+ }
+
+ public void testAddValidAndInvalidWords() {
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ }
+ BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+
+ binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY);
+ assertFalse(binaryDictionary.isValidWord("a"));
+ binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY);
+ assertFalse(binaryDictionary.isValidWord("a"));
+ binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY);
+ assertFalse(binaryDictionary.isValidWord("a"));
+ binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY);
+ assertTrue(binaryDictionary.isValidWord("a"));
+
+ binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
+ assertTrue(binaryDictionary.isValidWord("b"));
+
+ final int unigramProbability = binaryDictionary.getFrequency("a");
+ binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY);
+ assertFalse(binaryDictionary.isValidBigram("a", "b"));
+ binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY);
+ assertFalse(binaryDictionary.isValidBigram("a", "b"));
+ binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY);
+ assertFalse(binaryDictionary.isValidBigram("a", "b"));
+ binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY);
+ assertTrue(binaryDictionary.isValidBigram("a", "b"));
+
+ binaryDictionary.addUnigramWord("c", DUMMY_PROBABILITY);
+ binaryDictionary.addBigramWords("a", "c", DUMMY_PROBABILITY);
+ assertTrue(binaryDictionary.isValidBigram("a", "c"));
+
+ binaryDictionary.close();
+ dictFile.delete();
+ }
+
+ // TODO: Add large tests.
+ public void testDecayingProbability() {
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ }
+ BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+
+ binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ assertTrue(binaryDictionary.isValidWord("a"));
+ forcePassingShortTime(binaryDictionary);
+ assertFalse(binaryDictionary.isValidWord("a"));
+
+ binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ forcePassingShortTime(binaryDictionary);
+ assertTrue(binaryDictionary.isValidWord("a"));
+ forcePassingLongTime(binaryDictionary);
+ assertFalse(binaryDictionary.isValidWord("a"));
+
+ binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
+ binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY);
+ assertTrue(binaryDictionary.isValidBigram("a", "b"));
+ forcePassingShortTime(binaryDictionary);
+ assertFalse(binaryDictionary.isValidBigram("a", "b"));
+
+ binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
+ binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
+ binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
+ binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
+ binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY);
+ assertTrue(binaryDictionary.isValidBigram("a", "b"));
+ forcePassingShortTime(binaryDictionary);
+ assertTrue(binaryDictionary.isValidBigram("a", "b"));
+ forcePassingLongTime(binaryDictionary);
+ assertFalse(binaryDictionary.isValidBigram("a", "b"));
+
+ binaryDictionary.close();
+ dictFile.delete();
+ }
+}
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
index 96a2217..6a21522 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
@@ -21,24 +21,19 @@
import android.util.Pair;
import com.android.inputmethod.latin.makedict.CodePointUtils;
-import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
-import com.android.inputmethod.latin.makedict.FusionDictionary;
-import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
-import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Locale;
+import java.util.Map;
import java.util.Random;
@LargeTest
public class BinaryDictionaryTests extends AndroidTestCase {
- private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
- new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
private static final String TEST_LOCALE = "test";
@@ -52,15 +47,18 @@
super.tearDown();
}
- private File createEmptyDictionaryAndGetFile(final String filename) throws IOException,
- UnsupportedFormatException {
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false));
+ private File createEmptyDictionaryAndGetFile(final String filename) throws IOException {
final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION,
getContext().getCacheDir());
- final DictEncoder dictEncoder = new Ver3DictEncoder(file);
- dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
- return file;
+ Map<String, String> attributeMap = new HashMap<String, String>();
+ attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+ if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
+ 3 /* dictVersion */, attributeMap)) {
+ return file;
+ } else {
+ throw new IOException("Empty dictionary cannot be created.");
+ }
}
public void testIsValidDictionary() {
@@ -69,8 +67,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@@ -95,8 +91,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@@ -139,8 +133,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@@ -169,8 +161,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@@ -240,8 +230,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@@ -294,8 +282,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@@ -342,8 +328,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@@ -392,8 +376,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@@ -445,8 +427,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
@@ -516,8 +496,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
@@ -617,8 +595,6 @@
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
- } catch (UnsupportedFormatException e) {
- fail("UnsupportedFormatException while writing an initial dictionary : " + e);
}
final ArrayList<String> words = new ArrayList<String>();
@@ -630,7 +606,7 @@
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- while(!binaryDictionary.needsToRunGC()) {
+ while(!binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final String word = CodePointUtils.generateWord(random, codePointSet);
words.add(word);
final int unigramProbability = random.nextInt(0xFF);
@@ -650,4 +626,57 @@
dictFile.delete();
}
+
+ public void testUnigramAndBigramCount() {
+ final int flashWithGCIterationCount = 10;
+ final int codePointSetSize = 50;
+ final int unigramCountPerIteration = 1000;
+ final int bigramCountPerIteration = 2000;
+ final int seed = 1123581321;
+
+ final Random random = new Random(seed);
+
+ File dictFile = null;
+ try {
+ dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
+ } catch (IOException e) {
+ fail("IOException while writing an initial dictionary : " + e);
+ }
+
+ final ArrayList<String> words = new ArrayList<String>();
+ final HashSet<Pair<String, String>> bigrams = new HashSet<Pair<String, String>>();
+ final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
+
+ BinaryDictionary binaryDictionary;
+ for (int i = 0; i < flashWithGCIterationCount; i++) {
+ binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ for (int j = 0; j < unigramCountPerIteration; j++) {
+ final String word = CodePointUtils.generateWord(random, codePointSet);
+ words.add(word);
+ final int unigramProbability = random.nextInt(0xFF);
+ binaryDictionary.addUnigramWord(word, unigramProbability);
+ }
+ for (int j = 0; j < bigramCountPerIteration; j++) {
+ final String word0 = words.get(random.nextInt(words.size()));
+ final String word1 = words.get(random.nextInt(words.size()));
+ bigrams.add(new Pair<String, String>(word0, word1));
+ final int bigramProbability = random.nextInt(0xF);
+ binaryDictionary.addBigramWords(word0, word1, bigramProbability);
+ }
+ assertEquals(new HashSet<String>(words).size(), Integer.parseInt(
+ binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
+ assertEquals(new HashSet<Pair<String, String>>(bigrams).size(), Integer.parseInt(
+ binaryDictionary.getPropertyForTests(BinaryDictionary.BIGRAM_COUNT_QUERY)));
+ binaryDictionary.flushWithGC();
+ assertEquals(new HashSet<String>(words).size(), Integer.parseInt(
+ binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
+ assertEquals(new HashSet<Pair<String, String>>(bigrams).size(), Integer.parseInt(
+ binaryDictionary.getPropertyForTests(BinaryDictionary.BIGRAM_COUNT_QUERY)));
+ binaryDictionary.close();
+ }
+
+ dictFile.delete();
+ }
}
diff --git a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java
new file mode 100644
index 0000000..132483d
--- /dev/null
+++ b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.makedict;
+
+import android.test.AndroidTestCase;
+import android.test.suitebuilder.annotation.LargeTest;
+import android.util.Log;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Random;
+
+/**
+ * Unit tests for SparseTable.
+ */
+@LargeTest
+public class SparseTableTests extends AndroidTestCase {
+ private static final String TAG = SparseTableTests.class.getSimpleName();
+
+ private static final int[] SMALL_INDEX = { SparseTable.NOT_EXIST, 0 };
+ private static final int[] BIG_INDEX = { SparseTable.NOT_EXIST, 1, 2, 3, 4, 5, 6, 7};
+
+ private final Random mRandom;
+ private final ArrayList<Integer> mRandomIndex;
+
+ private static final int DEFAULT_SIZE = 10000;
+ private static final int BLOCK_SIZE = 8;
+
+ public SparseTableTests() {
+ this(System.currentTimeMillis(), DEFAULT_SIZE);
+ }
+
+ public SparseTableTests(final long seed, final int tableSize) {
+ super();
+ Log.d(TAG, "Seed for test is " + seed + ", size is " + tableSize);
+ mRandom = new Random(seed);
+ mRandomIndex = new ArrayList<Integer>(tableSize);
+ for (int i = 0; i < tableSize; ++i) {
+ mRandomIndex.add(SparseTable.NOT_EXIST);
+ }
+ }
+
+ public void testInitializeWithArray() {
+ final SparseTable table = new SparseTable(SMALL_INDEX, BIG_INDEX, BLOCK_SIZE);
+ for (int i = 0; i < 8; ++i) {
+ assertEquals(SparseTable.NOT_EXIST, table.get(i));
+ }
+ assertEquals(SparseTable.NOT_EXIST, table.get(8));
+ for (int i = 9; i < 16; ++i) {
+ assertEquals(i - 8, table.get(i));
+ }
+ }
+
+ public void testSet() {
+ final SparseTable table = new SparseTable(16, BLOCK_SIZE);
+ table.set(3, 6);
+ table.set(8, 16);
+ for (int i = 0; i < 16; ++i) {
+ if (i == 3 || i == 8) {
+ assertEquals(i * 2, table.get(i));
+ } else {
+ assertEquals(SparseTable.NOT_EXIST, table.get(i));
+ }
+ }
+ }
+
+ private void generateRandomIndex(final int size, final int prop) {
+ for (int i = 0; i < DEFAULT_SIZE; ++i) {
+ if (mRandom.nextInt(100) < prop) {
+ mRandomIndex.set(i, mRandom.nextInt());
+ } else {
+ mRandomIndex.set(i, SparseTable.NOT_EXIST);
+ }
+ }
+ }
+
+ private void runTestRandomSet() {
+ final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE);
+ int elementCount = 0;
+ for (int i = 0; i < DEFAULT_SIZE; ++i) {
+ if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) {
+ table.set(i, mRandomIndex.get(i));
+ elementCount++;
+ }
+ }
+
+ Log.d(TAG, "table size = " + table.getLookupTableSize() + " + "
+ + table.getContentTableSize());
+ Log.d(TAG, "the table has " + elementCount + " elements");
+ for (int i = 0; i < DEFAULT_SIZE; ++i) {
+ assertEquals(table.get(i), (int)mRandomIndex.get(i));
+ }
+
+ // flush and reload
+ OutputStream lookupOutStream = null;
+ OutputStream contentOutStream = null;
+ InputStream lookupInStream = null;
+ InputStream contentInStream = null;
+ try {
+ final File lookupIndexFile = File.createTempFile("testRandomSet", ".small");
+ final File contentFile = File.createTempFile("testRandomSet", ".big");
+ lookupOutStream = new FileOutputStream(lookupIndexFile);
+ contentOutStream = new FileOutputStream(contentFile);
+ table.write(lookupOutStream, contentOutStream);
+ lookupInStream = new FileInputStream(lookupIndexFile);
+ contentInStream = new FileInputStream(contentFile);
+ final byte[] lookupArray = new byte[(int) lookupIndexFile.length()];
+ final byte[] contentArray = new byte[(int) contentFile.length()];
+ lookupInStream.read(lookupArray);
+ contentInStream.read(contentArray);
+ final SparseTable newTable = new SparseTable(lookupArray, contentArray, BLOCK_SIZE);
+ for (int i = 0; i < DEFAULT_SIZE; ++i) {
+ assertEquals(table.get(i), newTable.get(i));
+ }
+ } catch (IOException e) {
+ Log.d(TAG, "IOException while flushing and realoding", e);
+ } finally {
+ if (lookupOutStream != null) {
+ try {
+ lookupOutStream.close();
+ } catch (IOException e) {
+ Log.d(TAG, "IOException while closing the stream", e);
+ }
+ }
+ if (contentOutStream != null) {
+ try {
+ contentOutStream.close();
+ } catch (IOException e) {
+ Log.d(TAG, "IOException while closing contentStream.", e);
+ }
+ }
+ }
+ }
+
+ public void testRandomSet() {
+ for (int i = 0; i <= 100; i += 10) {
+ generateRandomIndex(DEFAULT_SIZE, i);
+ runTestRandomSet();
+ }
+ }
+}
diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
index 06c4271..ddc9546 100644
--- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
@@ -75,10 +75,10 @@
return new ArrayList<String>(wordSet);
}
- private void addToDict(final UserHistoryPredictionDictionary dict, final List<String> words) {
+ private void addToDict(final UserHistoryDictionary dict, final List<String> words) {
String prevWord = null;
for (String word : words) {
- dict.addToPersonalizationPredictionDictionary(prevWord, word, true);
+ dict.addToDictionary(prevWord, word, true);
prevWord = word;
}
}
@@ -90,8 +90,8 @@
private void addAndWriteRandomWords(final String testFilenameSuffix, final int numberOfWords,
final Random random, final boolean checkContents) {
final List<String> words = generateWords(numberOfWords, random);
- final UserHistoryPredictionDictionary dict =
- PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+ final UserHistoryDictionary dict =
+ PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffix /* locale */, mPrefs);
// Add random words to the user history dictionary.
addToDict(dict, words);
@@ -122,8 +122,8 @@
true /* checksContents */);
} finally {
try {
- final UserHistoryPredictionDictionary dict =
- PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+ final UserHistoryDictionary dict =
+ PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffix, mPrefs);
Log.d(TAG, "waiting for writing ...");
dict.shutdownExecutorForTests();
@@ -134,7 +134,7 @@
Log.d(TAG, "InterruptedException: " + e);
}
- final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
+ final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFile = new File(getContext().getFilesDir(), fileName);
@@ -159,7 +159,7 @@
// Create filename suffixes for this test.
for (int i = 0; i < numberOfLanguages; i++) {
testFilenameSuffixes[i] = "testSwitchingLanguages" + i;
- final String fileName = UserHistoryPredictionDictionary.NAME + "." +
+ final String fileName = UserHistoryDictionary.NAME + "." +
testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFiles[i] = new File(getContext().getFilesDir(), fileName);
}
@@ -181,8 +181,8 @@
try {
Log.d(TAG, "waiting for writing ...");
for (int i = 0; i < numberOfLanguages; i++) {
- final UserHistoryPredictionDictionary dict =
- PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+ final UserHistoryDictionary dict =
+ PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffixes[i], mPrefs);
dict.shutdownExecutorForTests();
while (!dict.isTerminatedForTests()) {
@@ -210,8 +210,8 @@
10000 : 1000;
final Random random = new Random(123456);
- UserHistoryPredictionDictionary dict =
- PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+ UserHistoryDictionary dict =
+ PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffix, mPrefs);
try {
addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random,
@@ -227,7 +227,7 @@
} catch (InterruptedException e) {
Log.d(TAG, "InterruptedException: ", e);
}
- final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
+ final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFile = new File(getContext().getFilesDir(), fileName);
if (dictFile != null) {
diff --git a/tests/src/com/android/inputmethod/latin/utils/SpannableStringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/SpannableStringUtilsTests.java
new file mode 100644
index 0000000..fa6ad16
--- /dev/null
+++ b/tests/src/com/android/inputmethod/latin/utils/SpannableStringUtilsTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.utils;
+
+import android.test.AndroidTestCase;
+import android.test.suitebuilder.annotation.SmallTest;
+import android.text.style.SuggestionSpan;
+import android.text.style.URLSpan;
+import android.text.SpannableStringBuilder;
+import android.text.Spannable;
+import android.text.Spanned;
+
+@SmallTest
+public class SpannableStringUtilsTests extends AndroidTestCase {
+ public void testConcatWithSuggestionSpansOnly() {
+ SpannableStringBuilder s = new SpannableStringBuilder("test string\ntest string\n"
+ + "test string\ntest string\ntest string\ntest string\ntest string\ntest string\n"
+ + "test string\ntest string\n");
+ final int N = 10;
+ for (int i = 0; i < N; ++i) {
+ // Put a PARAGRAPH-flagged span that should not be found in the result.
+ s.setSpan(new SuggestionSpan(getContext(),
+ new String[] {"" + i}, Spannable.SPAN_PARAGRAPH),
+ i * 12, i * 12 + 12, Spannable.SPAN_PARAGRAPH);
+ // Put a normal suggestion span that should be found in the result.
+ s.setSpan(new SuggestionSpan(getContext(), new String[] {"" + i}, 0), i, i * 2, 0);
+ // Put a URL span than should not be found in the result.
+ s.setSpan(new URLSpan("http://a"), i, i * 2, 0);
+ }
+
+ final CharSequence a = s.subSequence(0, 15);
+ final CharSequence b = s.subSequence(15, s.length());
+ final Spanned result =
+ (Spanned)SpannableStringUtils.concatWithNonParagraphSuggestionSpansOnly(a, b);
+
+ Object[] spans = result.getSpans(0, result.length(), SuggestionSpan.class);
+ for (int i = 0; i < spans.length; i++) {
+ final int flags = result.getSpanFlags(spans[i]);
+ assertEquals("Should not find a span with PARAGRAPH flag",
+ flags & Spannable.SPAN_PARAGRAPH, 0);
+ assertTrue("Should be a SuggestionSpan", spans[i] instanceof SuggestionSpan);
+ }
+ }
+}
diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
index eb9fb98..4e396a1 100644
--- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
+++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java
@@ -20,11 +20,6 @@
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.SmallTest;
-import android.text.style.SuggestionSpan;
-import android.text.style.URLSpan;
-import android.text.SpannableStringBuilder;
-import android.text.Spannable;
-import android.text.Spanned;
import java.util.Arrays;
import java.util.List;
@@ -285,34 +280,4 @@
assertEquals(objs[i], newObjArray.get(i));
}
}
-
- public void testConcatWithSuggestionSpansOnly() {
- SpannableStringBuilder s = new SpannableStringBuilder("test string\ntest string\n"
- + "test string\ntest string\ntest string\ntest string\ntest string\ntest string\n"
- + "test string\ntest string\n");
- final int N = 10;
- for (int i = 0; i < N; ++i) {
- // Put a PARAGRAPH-flagged span that should not be found in the result.
- s.setSpan(new SuggestionSpan(getContext(),
- new String[] {"" + i}, Spannable.SPAN_PARAGRAPH),
- i * 12, i * 12 + 12, Spannable.SPAN_PARAGRAPH);
- // Put a normal suggestion span that should be found in the result.
- s.setSpan(new SuggestionSpan(getContext(), new String[] {"" + i}, 0), i, i * 2, 0);
- // Put a URL span than should not be found in the result.
- s.setSpan(new URLSpan("http://a"), i, i * 2, 0);
- }
-
- final CharSequence a = s.subSequence(0, 15);
- final CharSequence b = s.subSequence(15, s.length());
- final Spanned result =
- (Spanned)StringUtils.concatWithNonParagraphSuggestionSpansOnly(a, b);
-
- Object[] spans = result.getSpans(0, result.length(), SuggestionSpan.class);
- for (int i = 0; i < spans.length; i++) {
- final int flags = result.getSpanFlags(spans[i]);
- assertEquals("Should not find a span with PARAGRAPH flag",
- flags & Spannable.SPAN_PARAGRAPH, 0);
- assertTrue("Should be a SuggestionSpan", spans[i] instanceof SuggestionSpan);
- }
- }
}
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
index fa80385..bd06e9f 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
@@ -80,17 +80,17 @@
}
/**
- * Returns a decrypted/uncompressed binary dictionary.
+ * Returns a decrypted/uncompressed dictionary.
*
- * This will decrypt/uncompress any number of times as necessary until it finds the binary
+ * This will decrypt/uncompress any number of times as necessary until it finds the
* dictionary signature, and copy the decoded file to a temporary place.
- * If this is not a binary dictionary, the method returns null.
+ * If this is not a dictionary, the method returns null.
*/
- public static DecoderChainSpec getRawBinaryDictionaryOrNull(final File src) {
- return getRawBinaryDictionaryOrNullInternal(new DecoderChainSpec(), src, 0);
+ public static DecoderChainSpec getRawDictionaryOrNull(final File src) {
+ return getRawDictionaryOrNullInternal(new DecoderChainSpec(), src, 0);
}
- private static DecoderChainSpec getRawBinaryDictionaryOrNullInternal(
+ private static DecoderChainSpec getRawDictionaryOrNullInternal(
final DecoderChainSpec spec, final File src, final int depth) {
// Unfortunately the decoding scheme we use can consider any data to be encrypted
// and will product some output, meaning it's not possible to reliably detect encrypted
@@ -98,7 +98,8 @@
// over and over, ending in a stack overflow. Hence we limit the depth at which we try
// decoding the file.
if (depth > MAX_DECODE_DEPTH) return null;
- if (BinaryDictDecoderUtils.isBinaryDictionary(src)) {
+ if (BinaryDictDecoderUtils.isBinaryDictionary(src)
+ || CombinedInputOutput.isCombinedDictionary(src.getAbsolutePath())) {
spec.mFile = src;
return spec;
}
@@ -106,7 +107,7 @@
final File uncompressedFile = tryGetUncompressedFile(src);
if (null != uncompressedFile) {
final DecoderChainSpec newSpec =
- getRawBinaryDictionaryOrNullInternal(spec, uncompressedFile, depth + 1);
+ getRawDictionaryOrNullInternal(spec, uncompressedFile, depth + 1);
if (null == newSpec) return null;
return newSpec.addStep(COMPRESSION);
}
@@ -114,7 +115,7 @@
final File decryptedFile = tryGetDecryptedFile(src);
if (null != decryptedFile) {
final DecoderChainSpec newSpec =
- getRawBinaryDictionaryOrNullInternal(spec, decryptedFile, depth + 1);
+ getRawDictionaryOrNullInternal(spec, decryptedFile, depth + 1);
if (null == newSpec) return null;
return newSpec.addStep(ENCRYPTION);
}
@@ -175,17 +176,18 @@
return XmlDictInputOutput.readDictionaryXml(
new BufferedInputStream(new FileInputStream(file)),
null /* shortcuts */, null /* bigrams */);
- } else if (CombinedInputOutput.isCombinedDictionary(filename)) {
- if (report) System.out.println("Format : Combined format");
- return CombinedInputOutput.readDictionaryCombined(
- new BufferedInputStream(new FileInputStream(file)));
} else {
- final DecoderChainSpec decodedSpec = getRawBinaryDictionaryOrNull(file);
+ final DecoderChainSpec decodedSpec = getRawDictionaryOrNull(file);
if (null == decodedSpec) {
crash(filename, new RuntimeException(
filename + " does not seem to be a dictionary file"));
+ } else if (CombinedInputOutput.isCombinedDictionary(
+ decodedSpec.mFile.getAbsolutePath())){
+ if (report) System.out.println("Format : Combined format");
+ return CombinedInputOutput.readDictionaryCombined(
+ new BufferedInputStream(new FileInputStream(decodedSpec.mFile)));
} else {
- final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file,
+ final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile,
DictDecoder.USE_BYTEARRAY);
if (report) {
System.out.println("Format : Binary dictionary format");
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java
index 9274dcd..dff3387 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java
@@ -79,7 +79,7 @@
throw new RuntimeException("Too many/too few arguments for command " + COMMAND);
}
final BinaryDictOffdeviceUtils.DecoderChainSpec decodedSpec =
- BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(new File(mArgs[0]));
+ BinaryDictOffdeviceUtils.getRawDictionaryOrNull(new File(mArgs[0]));
if (null == decodedSpec) {
System.out.println(mArgs[0] + " does not seem to be a dictionary");
return;
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
index 1eff497..1baeb7a 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
@@ -64,7 +64,7 @@
// Test for an actually compressed dictionary and its contents
final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec =
- BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(dst);
+ BinaryDictOffdeviceUtils.getRawDictionaryOrNull(dst);
for (final String step : decodeSpec.mDecoderSpec) {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
}
@@ -90,7 +90,7 @@
// Test that a random data file actually fails
assertNull("Wrongly identified data file",
- BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(dst));
+ BinaryDictOffdeviceUtils.getRawDictionaryOrNull(dst));
final File gzDst = File.createTempFile("testGetRawDict", ".tmp");
gzDst.deleteOnExit();
@@ -103,6 +103,6 @@
// Test that a compressed random data file actually fails
assertNull("Wrongly identified data file",
- BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(gzDst));
+ BinaryDictOffdeviceUtils.getRawDictionaryOrNull(gzDst));
}
}
