Read shortcuts as strings in the dictionary.
This has no impact on performance.
Before:
(0) 9.61 (0.01%)
(1) 57514.58 (56.70%)
(2) 10.55 (0.01%)
(3) 10.79 (0.01%)
(4) 133.20 (0.13%)
(5) 43553.87 (42.94%)
(6) 10.03 (0.01%)
(20) 47.20 (0.05%)
Total 101431.47 (sum of others 101289.84)
After:
(0) 10.52 (0.01%)
(1) 56311.16 (56.66%)
(2) 13.40 (0.01%)
(3) 10.98 (0.01%)
(4) 136.72 (0.14%)
(5) 42707.92 (42.97%)
(6) 9.79 (0.01%)
(20) 51.35 (0.05%)
Total 99390.76 (sum of others 99251.84)
The difference is not significant with regard to measure imprecision
Change-Id: I2e4f1ef7a5e99082e67dd27f56cf4fc432bb48fa
diff --git a/native/jni/src/unigram_dictionary.cpp b/native/jni/src/unigram_dictionary.cpp
index ed4c066..50805ad 100644
--- a/native/jni/src/unigram_dictionary.cpp
+++ b/native/jni/src/unigram_dictionary.cpp
@@ -366,10 +366,9 @@
WordsPriorityQueue *masterQueue = queuePool->getMasterQueue();
const int finalFreq = correction->getFinalFreq(freq, &wordPointer, &wordLength);
if (finalFreq != NOT_A_FREQUENCY) {
- if (!terminalAttributes.isShortcutOnly()) {
- addWord(wordPointer, wordLength, finalFreq, masterQueue);
- }
+ addWord(wordPointer, wordLength, finalFreq, masterQueue);
+ const int shortcutFreq = finalFreq > 0 ? finalFreq - 1 : 0;
// Please note that the shortcut candidates will be added to the master queue only.
TerminalAttributes::ShortcutIterator iterator =
terminalAttributes.getShortcutIterator();
@@ -379,11 +378,12 @@
// We need to either modulate the frequency of each shortcut according
// to its own shortcut frequency or to make the queue
// so that the insert order is protected inside the queue for words
- // with the same score.
+ // with the same score. For the moment we use -1 to make sure the shortcut will
+ // never be in front of the word.
uint16_t shortcutTarget[MAX_WORD_LENGTH_INTERNAL];
const int shortcutTargetStringLength = iterator.getNextShortcutTarget(
MAX_WORD_LENGTH_INTERNAL, shortcutTarget);
- addWord(shortcutTarget, shortcutTargetStringLength, finalFreq, masterQueue);
+ addWord(shortcutTarget, shortcutTargetStringLength, shortcutFreq, masterQueue);
}
}
}