Demote skipped characters matched words with respect to length.
Words that matched user input with skipped characters used to be demoted
in BinaryDictionary by a constant factor and not at all in those dictionaries
implemented in java code. To represent the fact that the impact of a skipped
character gets larger as the word is shorter, this change will implement a
demotion that gets larger as the typed word is shorter. The demotion rate
is (n - 2) / (n - 1) where n is the length of the typed word for n >= 2.
It implements it for both BinaryDictionary and java dictionaries.
Bug: 3340731
Change-Id: I3a18be80a9708981d56a950dc25fe08f018b5b89
diff --git a/native/src/defines.h b/native/src/defines.h
index ddd65c9..0d1f037 100644
--- a/native/src/defines.h
+++ b/native/src/defines.h
@@ -135,7 +135,7 @@
#define SUGGEST_WORDS_WITH_TRANSPOSED_CHARACTERS true
// The following "rate"s are used as a multiplier before dividing by 100, so they are in percent.
-#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE 75
+#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE 100
#define WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE 80
#define WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE 75
#define WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE 75
diff --git a/native/src/unigram_dictionary.cpp b/native/src/unigram_dictionary.cpp
index 9aa36b0..17a87a7 100644
--- a/native/src/unigram_dictionary.cpp
+++ b/native/src/unigram_dictionary.cpp
@@ -457,10 +457,17 @@
}
inline int UnigramDictionary::calculateFinalFreq(const int inputIndex, const int depth,
const int matchWeight, const int skipPos, const int excessivePos, const int transposedPos,
- const int freq, const bool sameLength) {
+ const int freq, const bool sameLength) const {
// TODO: Demote by edit distance
int finalFreq = freq * matchWeight;
- if (skipPos >= 0) multiplyRate(WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE, &finalFreq);
+ if (skipPos >= 0) {
+ if (mInputLength >= 3) {
+ multiplyRate(WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE *
+ (mInputLength - 2) / (mInputLength - 1), &finalFreq);
+ } else {
+ finalFreq = 0;
+ }
+ }
if (transposedPos >= 0) multiplyRate(
WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE, &finalFreq);
if (excessivePos >= 0) {
@@ -514,7 +521,7 @@
}
inline bool UnigramDictionary::existsAdjacentProximityChars(const int inputIndex,
- const int inputLength) {
+ const int inputLength) const {
if (inputIndex < 0 || inputIndex >= inputLength) return false;
const int currentChar = *getInputCharsAt(inputIndex);
const int leftIndex = inputIndex - 1;
diff --git a/native/src/unigram_dictionary.h b/native/src/unigram_dictionary.h
index a959845..2912cb0 100644
--- a/native/src/unigram_dictionary.h
+++ b/native/src/unigram_dictionary.h
@@ -71,7 +71,8 @@
const int nextLettersSize);
void registerNextLetter(unsigned short c, int *nextLetters, int nextLettersSize);
int calculateFinalFreq(const int inputIndex, const int depth, const int snr, const int skipPos,
- const int excessivePos, const int transposedPos, const int freq, const bool sameLength);
+ const int excessivePos, const int transposedPos, const int freq,
+ const bool sameLength) const;
void onTerminalWhenUserTypedLengthIsGreaterThanInputLength(unsigned short *word,
const int inputIndex, const int depth, const int snr, int *nextLetters,
const int nextLettersSize, const int skipPos, const int excessivePos,
@@ -95,8 +96,8 @@
bool processCurrentNodeForExactMatch(const int firstChildPos,
const int startInputIndex, const int depth, unsigned short *word,
int *newChildPosition, int *newCount, bool *newTerminal, int *newFreq, int *siblingPos);
- bool existsAdjacentProximityChars(const int inputIndex, const int inputLength);
- inline const int* getInputCharsAt(const int index) {
+ bool existsAdjacentProximityChars(const int inputIndex, const int inputLength) const;
+ inline const int* getInputCharsAt(const int index) const {
return mInputCodes + (index * MAX_PROXIMITY_CHARS);
}
const unsigned char *DICT;