Use IntArrayView for dictionary structure policy.
Change-Id: I0dc94908259d70d5085ff22abf422d90affb1452
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 22ad2d0..81e2ff5 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -32,6 +32,7 @@
#include "suggest/core/suggest_options.h"
#include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
#include "utils/char_utils.h"
+#include "utils/int_array_view.h"
#include "utils/jni_data_utils.h"
#include "utils/log_utils.h"
#include "utils/time_keeper.h"
@@ -581,8 +582,9 @@
return false;
}
}
- if (!dictionaryStructureWithBufferPolicy->addUnigramEntry(wordCodePoints,
- wordCodePointCount, wordProperty.getUnigramProperty())) {
+ if (!dictionaryStructureWithBufferPolicy->addUnigramEntry(
+ CodePointArrayView(wordCodePoints, wordCodePointCount),
+ wordProperty.getUnigramProperty())) {
LogUtils::logToJava(env, "Cannot add unigram to the new dict.");
return false;
}
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp
index d625739..c025bfc 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp
@@ -28,6 +28,7 @@
#include "suggest/core/suggest_options.h"
#include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h"
#include "suggest/policyimpl/typing/typing_suggest_policy_factory.h"
+#include "utils/int_array_view.h"
#include "utils/log_utils.h"
#include "utils/time_keeper.h"
@@ -112,8 +113,8 @@
int Dictionary::getNgramProbability(const PrevWordsInfo *const prevWordsInfo, const int *word,
int length) const {
TimeKeeper::setCurrentTime();
- int nextWordPos = mDictionaryStructureWithBufferPolicy->getTerminalPtNodePositionOfWord(word,
- length, false /* forceLowerCaseSearch */);
+ int nextWordPos = mDictionaryStructureWithBufferPolicy->getTerminalPtNodePositionOfWord(
+ CodePointArrayView(word, length), false /* forceLowerCaseSearch */);
if (NOT_A_DICT_POS == nextWordPos) return NOT_A_PROBABILITY;
if (!prevWordsInfo) {
return getDictionaryStructurePolicy()->getProbabilityOfPtNode(
@@ -135,12 +136,14 @@
return false;
}
TimeKeeper::setCurrentTime();
- return mDictionaryStructureWithBufferPolicy->addUnigramEntry(word, length, unigramProperty);
+ return mDictionaryStructureWithBufferPolicy->addUnigramEntry(CodePointArrayView(word, length),
+ unigramProperty);
}
bool Dictionary::removeUnigramEntry(const int *const codePoints, const int codePointCount) {
TimeKeeper::setCurrentTime();
- return mDictionaryStructureWithBufferPolicy->removeUnigramEntry(codePoints, codePointCount);
+ return mDictionaryStructureWithBufferPolicy->removeUnigramEntry(
+ CodePointArrayView(codePoints, codePointCount));
}
bool Dictionary::addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
@@ -152,7 +155,8 @@
bool Dictionary::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const int *const word, const int length) {
TimeKeeper::setCurrentTime();
- return mDictionaryStructureWithBufferPolicy->removeNgramEntry(prevWordsInfo, word, length);
+ return mDictionaryStructureWithBufferPolicy->removeNgramEntry(prevWordsInfo,
+ CodePointArrayView(word, length));
}
bool Dictionary::flush(const char *const filePath) {
@@ -181,7 +185,7 @@
const int codePointCount) {
TimeKeeper::setCurrentTime();
return mDictionaryStructureWithBufferPolicy->getWordProperty(
- codePoints, codePointCount);
+ CodePointArrayView(codePoints, codePointCount));
}
int Dictionary::getNextWordAndNextToken(const int token, int *const outCodePoints,
diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
index 5052f46..0faf000 100644
--- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
+++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
@@ -21,6 +21,7 @@
#include "defines.h"
#include "suggest/core/dictionary/property/word_property.h"
+#include "utils/int_array_view.h"
namespace latinime {
@@ -49,33 +50,32 @@
DicNodeVector *const childDicNodes) const = 0;
virtual int getCodePointsAndProbabilityAndReturnCodePointCount(
- const int nodePos, const int maxCodePointCount, int *const outCodePoints,
+ const int ptNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const = 0;
- virtual int getTerminalPtNodePositionOfWord(const int *const inWord,
- const int length, const bool forceLowerCaseSearch) const = 0;
+ virtual int getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
+ const bool forceLowerCaseSearch) const = 0;
- virtual int getProbability(const int unigramProbability,
- const int bigramProbability) const = 0;
+ virtual int getProbability(const int unigramProbability, const int bigramProbability) const = 0;
virtual int getProbabilityOfPtNode(const int *const prevWordsPtNodePos,
- const int nodePos) const = 0;
+ const int ptNodePos) const = 0;
virtual void iterateNgramEntries(const int *const prevWordsPtNodePos,
NgramListener *const listener) const = 0;
- virtual int getShortcutPositionOfPtNode(const int nodePos) const = 0;
+ virtual int getShortcutPositionOfPtNode(const int ptNodePos) const = 0;
virtual const DictionaryHeaderStructurePolicy *getHeaderStructurePolicy() const = 0;
virtual const DictionaryShortcutsStructurePolicy *getShortcutsStructurePolicy() const = 0;
// Returns whether the update was success or not.
- virtual bool addUnigramEntry(const int *const word, const int length,
+ virtual bool addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty) = 0;
// Returns whether the update was success or not.
- virtual bool removeUnigramEntry(const int *const word, const int length) = 0;
+ virtual bool removeUnigramEntry(const CodePointArrayView wordCodePoints) = 0;
// Returns whether the update was success or not.
virtual bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
@@ -83,7 +83,7 @@
// Returns whether the update was success or not.
virtual bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
- const int *const word, const int length) = 0;
+ const CodePointArrayView wordCodePoints) = 0;
// Returns whether the flush was success or not.
virtual bool flush(const char *const filePath) = 0;
@@ -99,8 +99,7 @@
const int maxResultLength) = 0;
// Used for testing.
- virtual const WordProperty getWordProperty(const int *const codePonts,
- const int codePointCount) const = 0;
+ virtual const WordProperty getWordProperty(const CodePointArrayView wordCodePoints) const = 0;
// Method to iterate all words in the dictionary.
// The returned token has to be used to get the next word. If token is 0, this method newly
diff --git a/native/jni/src/suggest/core/session/prev_words_info.h b/native/jni/src/suggest/core/session/prev_words_info.h
index e44e876..9b3a7d4 100644
--- a/native/jni/src/suggest/core/session/prev_words_info.h
+++ b/native/jni/src/suggest/core/session/prev_words_info.h
@@ -21,6 +21,7 @@
#include "suggest/core/dictionary/binary_dictionary_bigrams_iterator.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "utils/char_utils.h"
+#include "utils/int_array_view.h"
namespace latinime {
@@ -91,19 +92,11 @@
}
// n is 1-indexed.
- const int *getNthPrevWordCodePoints(const int n) const {
+ const CodePointArrayView getNthPrevWordCodePoints(const int n) const {
if (n <= 0 || n > MAX_PREV_WORD_COUNT_FOR_N_GRAM) {
- return nullptr;
+ return CodePointArrayView();
}
- return mPrevWordCodePoints[n - 1];
- }
-
- // n is 1-indexed.
- int getNthPrevWordCodePointCount(const int n) const {
- if (n <= 0 || n > MAX_PREV_WORD_COUNT_FOR_N_GRAM) {
- return 0;
- }
- return mPrevWordCodePointCount[n - 1];
+ return CodePointArrayView(mPrevWordCodePoints[n - 1], mPrevWordCodePointCount[n - 1]);
}
// n is 1-indexed.
@@ -134,8 +127,9 @@
return NOT_A_DICT_POS;
}
}
+ const CodePointArrayView codePointArrayView(codePoints, codePointCount);
const int wordPtNodePos = dictStructurePolicy->getTerminalPtNodePositionOfWord(
- codePoints, codePointCount, false /* forceLowerCaseSearch */);
+ codePointArrayView, false /* forceLowerCaseSearch */);
if (wordPtNodePos != NOT_A_DICT_POS || !tryLowerCaseSearch) {
// Return the position when when the word was found or doesn't try lower case
// search.
@@ -144,7 +138,7 @@
// Check bigrams for lower-cased previous word if original was not found. Useful for
// auto-capitalized words like "The [current_word]".
return dictStructurePolicy->getTerminalPtNodePositionOfWord(
- codePoints, codePointCount, true /* forceLowerCaseSearch */);
+ codePointArrayView, true /* forceLowerCaseSearch */);
}
void clear() {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp
index c86ae93..9f6ae11 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp
@@ -104,12 +104,12 @@
return codePointCount;
}
-int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord,
- const int length, const bool forceLowerCaseSearch) const {
+int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
+ const bool forceLowerCaseSearch) const {
DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition());
- const int ptNodePos =
- readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch);
+ const int ptNodePos = readingHelper.getTerminalPtNodePositionOfWord(wordCodePoints.data(),
+ wordCodePoints.size(), forceLowerCaseSearch);
if (readingHelper.isError()) {
mIsCorrupted = true;
AKLOGE("Dictionary reading error in createAndGetAllChildDicNodes().");
@@ -194,7 +194,7 @@
ptNodeParams.getTerminalId());
}
-bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int length,
+bool Ver4PatriciaTriePolicy::addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty) {
if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: addUnigramEntry() is called for non-updatable dictionary.");
@@ -205,8 +205,9 @@
mDictBuffer->getTailPosition());
return false;
}
- if (length > MAX_WORD_LENGTH) {
- AKLOGE("The word is too long to insert to the dictionary, length: %d", length);
+ if (wordCodePoints.size() > MAX_WORD_LENGTH) {
+ AKLOGE("The word is too long to insert to the dictionary, length: %zd",
+ wordCodePoints.size());
return false;
}
for (const auto &shortcut : unigramProperty->getShortcuts()) {
@@ -220,8 +221,8 @@
readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false;
int codePointsToAdd[MAX_WORD_LENGTH];
- int codePointCountToAdd = length;
- memmove(codePointsToAdd, word, sizeof(int) * length);
+ int codePointCountToAdd = wordCodePoints.size();
+ memmove(codePointsToAdd, wordCodePoints.data(), sizeof(int) * codePointCountToAdd);
if (unigramProperty->representsBeginningOfSentence()) {
codePointCountToAdd = CharUtils::attachBeginningOfSentenceMarker(codePointsToAdd,
codePointCountToAdd, MAX_WORD_LENGTH);
@@ -229,14 +230,15 @@
if (codePointCountToAdd <= 0) {
return false;
}
- if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointsToAdd, codePointCountToAdd,
- unigramProperty, &addedNewUnigram)) {
+ const CodePointArrayView codePointArrayView(codePointsToAdd, codePointCountToAdd);
+ if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointArrayView.data(),
+ codePointArrayView.size(), unigramProperty, &addedNewUnigram)) {
if (addedNewUnigram && !unigramProperty->representsBeginningOfSentence()) {
mUnigramCount++;
}
if (unigramProperty->getShortcuts().size() > 0) {
// Add shortcut target.
- const int wordPos = getTerminalPtNodePositionOfWord(word, length,
+ const int wordPos = getTerminalPtNodePositionOfWord(codePointArrayView,
false /* forceLowerCaseSearch */);
if (wordPos == NOT_A_DICT_POS) {
AKLOGE("Cannot find terminal PtNode position to add shortcut target.");
@@ -259,12 +261,12 @@
}
}
-bool Ver4PatriciaTriePolicy::removeUnigramEntry(const int *const word, const int length) {
+bool Ver4PatriciaTriePolicy::removeUnigramEntry(const CodePointArrayView wordCodePoints) {
if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary.");
return false;
}
- const int ptNodePos = getTerminalPtNodePositionOfWord(word, length,
+ const int ptNodePos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */);
if (ptNodePos == NOT_A_DICT_POS) {
return false;
@@ -305,7 +307,6 @@
false /* isBlacklisted */, MAX_PROBABILITY /* probability */,
NOT_A_TIMESTAMP /* timestamp */, 0 /* level */, 0 /* count */, &shortcuts);
if (!addUnigramEntry(prevWordsInfo->getNthPrevWordCodePoints(1 /* n */),
- prevWordsInfo->getNthPrevWordCodePointCount(1 /* n */),
&beginningOfSentenceUnigramProperty)) {
AKLOGE("Cannot add unigram entry for the beginning-of-sentence.");
return false;
@@ -318,8 +319,8 @@
}
}
const int word1Pos = getTerminalPtNodePositionOfWord(
- bigramProperty->getTargetCodePoints()->data(),
- bigramProperty->getTargetCodePoints()->size(), false /* forceLowerCaseSearch */);
+ CodePointArrayView(*bigramProperty->getTargetCodePoints()),
+ false /* forceLowerCaseSearch */);
if (word1Pos == NOT_A_DICT_POS) {
return false;
}
@@ -336,7 +337,7 @@
}
bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
- const int *const word, const int length) {
+ const CodePointArrayView wordCodePoints) {
if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary.");
return false;
@@ -350,8 +351,9 @@
AKLOGE("prev words info is not valid for removing n-gram entry form the dictionary.");
return false;
}
- if (length > MAX_WORD_LENGTH) {
- AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %d", length);
+ if (wordCodePoints.size() > MAX_WORD_LENGTH) {
+ AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %zd",
+ wordCodePoints.size());
}
int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
@@ -360,7 +362,7 @@
if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
return false;
}
- const int wordPos = getTerminalPtNodePositionOfWord(word, length,
+ const int wordPos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */);
if (wordPos == NOT_A_DICT_POS) {
return false;
@@ -445,9 +447,9 @@
}
}
-const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const codePoints,
- const int codePointCount) const {
- const int ptNodePos = getTerminalPtNodePositionOfWord(codePoints, codePointCount,
+const WordProperty Ver4PatriciaTriePolicy::getWordProperty(
+ const CodePointArrayView wordCodePoints) const {
+ const int ptNodePos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */);
if (ptNodePos == NOT_A_DICT_POS) {
AKLOGE("getWordProperty is called for invalid word.");
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h
index fac3828..df119e3 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h
@@ -39,6 +39,7 @@
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+#include "utils/int_array_view.h"
namespace latinime {
namespace backward {
@@ -75,7 +76,7 @@
mBigramCount(mHeaderPolicy->getBigramCount()),
mTerminalPtNodePositionsForIteratingWords(), mIsCorrupted(false) {};
- AK_FORCE_INLINE int getRootPosition() const {
+ virtual int getRootPosition() const {
return 0;
}
@@ -86,8 +87,8 @@
const int terminalPtNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const;
- int getTerminalPtNodePositionOfWord(const int *const inWord,
- const int length, const bool forceLowerCaseSearch) const;
+ int getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
+ const bool forceLowerCaseSearch) const;
int getProbability(const int unigramProbability, const int bigramProbability) const;
@@ -106,16 +107,16 @@
return &mShortcutPolicy;
}
- bool addUnigramEntry(const int *const word, const int length,
+ bool addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty);
- bool removeUnigramEntry(const int *const word, const int length);
+ bool removeUnigramEntry(const CodePointArrayView wordCodePoints);
bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const BigramProperty *const bigramProperty);
- bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word1,
- const int length1);
+ bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
+ const CodePointArrayView wordCodePoints);
bool flush(const char *const filePath);
@@ -126,8 +127,7 @@
void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength);
- const WordProperty getWordProperty(const int *const codePoints,
- const int codePointCount) const;
+ const WordProperty getWordProperty(const CodePointArrayView wordCodePoints) const;
int getNextWordAndNextToken(const int token, int *const outCodePoints,
int *const outCodePointCount);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp
index aca64b3..4ac366e 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp
@@ -268,12 +268,12 @@
// This function gets the position of the terminal PtNode of the exact matching word in the
// dictionary. If no match is found, it returns NOT_A_DICT_POS.
-int PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord,
- const int length, const bool forceLowerCaseSearch) const {
+int PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
+ const bool forceLowerCaseSearch) const {
DynamicPtReadingHelper readingHelper(&mPtNodeReader, &mPtNodeArrayReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition());
- const int ptNodePos =
- readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch);
+ const int ptNodePos = readingHelper.getTerminalPtNodePositionOfWord(wordCodePoints.data(),
+ wordCodePoints.size(), forceLowerCaseSearch);
if (readingHelper.isError()) {
mIsCorrupted = true;
AKLOGE("Dictionary reading error in createAndGetAllChildDicNodes().");
@@ -377,9 +377,9 @@
return siblingPos;
}
-const WordProperty PatriciaTriePolicy::getWordProperty(const int *const codePoints,
- const int codePointCount) const {
- const int ptNodePos = getTerminalPtNodePositionOfWord(codePoints, codePointCount,
+const WordProperty PatriciaTriePolicy::getWordProperty(
+ const CodePointArrayView wordCodePoints) const {
+ const int ptNodePos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */);
if (ptNodePos == NOT_A_DICT_POS) {
AKLOGE("getWordProperty was called for invalid word.");
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
index 4257b0b..4d9af28 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
@@ -30,6 +30,7 @@
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "utils/byte_array_view.h"
+#include "utils/int_array_view.h"
namespace latinime {
@@ -63,8 +64,8 @@
const int terminalNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const;
- int getTerminalPtNodePositionOfWord(const int *const inWord,
- const int length, const bool forceLowerCaseSearch) const;
+ int getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
+ const bool forceLowerCaseSearch) const;
int getProbability(const int unigramProbability, const int bigramProbability) const;
@@ -83,14 +84,14 @@
return &mShortcutListPolicy;
}
- bool addUnigramEntry(const int *const word, const int length,
+ bool addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty) {
// This method should not be called for non-updatable dictionary.
AKLOGI("Warning: addUnigramEntry() is called for non-updatable dictionary.");
return false;
}
- bool removeUnigramEntry(const int *const word, const int length) {
+ bool removeUnigramEntry(const CodePointArrayView wordCodePoints) {
// This method should not be called for non-updatable dictionary.
AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary.");
return false;
@@ -103,8 +104,8 @@
return false;
}
- bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word,
- const int length) {
+ bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
+ const CodePointArrayView wordCodePoints) {
// This method should not be called for non-updatable dictionary.
AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary.");
return false;
@@ -136,8 +137,7 @@
}
}
- const WordProperty getWordProperty(const int *const codePoints,
- const int codePointCount) const;
+ const WordProperty getWordProperty(const CodePointArrayView wordCodePoints) const;
int getNextWordAndNextToken(const int token, int *const outCodePoints,
int *const outCodePointCount);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
index ae3208c..619cdb5 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
@@ -94,12 +94,12 @@
return codePointCount;
}
-int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord,
- const int length, const bool forceLowerCaseSearch) const {
+int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
+ const bool forceLowerCaseSearch) const {
DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition());
- const int ptNodePos =
- readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch);
+ const int ptNodePos = readingHelper.getTerminalPtNodePositionOfWord(wordCodePoints.data(),
+ wordCodePoints.size(), forceLowerCaseSearch);
if (readingHelper.isError()) {
mIsCorrupted = true;
AKLOGE("Dictionary reading error in createAndGetAllChildDicNodes().");
@@ -189,7 +189,7 @@
ptNodeParams.getTerminalId());
}
-bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int length,
+bool Ver4PatriciaTriePolicy::addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty) {
if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: addUnigramEntry() is called for non-updatable dictionary.");
@@ -200,8 +200,9 @@
mDictBuffer->getTailPosition());
return false;
}
- if (length > MAX_WORD_LENGTH) {
- AKLOGE("The word is too long to insert to the dictionary, length: %d", length);
+ if (wordCodePoints.size() > MAX_WORD_LENGTH) {
+ AKLOGE("The word is too long to insert to the dictionary, length: %zd",
+ wordCodePoints.size());
return false;
}
for (const auto &shortcut : unigramProperty->getShortcuts()) {
@@ -215,8 +216,8 @@
readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false;
int codePointsToAdd[MAX_WORD_LENGTH];
- int codePointCountToAdd = length;
- memmove(codePointsToAdd, word, sizeof(int) * length);
+ int codePointCountToAdd = wordCodePoints.size();
+ memmove(codePointsToAdd, wordCodePoints.data(), sizeof(int) * codePointCountToAdd);
if (unigramProperty->representsBeginningOfSentence()) {
codePointCountToAdd = CharUtils::attachBeginningOfSentenceMarker(codePointsToAdd,
codePointCountToAdd, MAX_WORD_LENGTH);
@@ -224,14 +225,15 @@
if (codePointCountToAdd <= 0) {
return false;
}
- if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointsToAdd, codePointCountToAdd,
- unigramProperty, &addedNewUnigram)) {
+ const CodePointArrayView codePointArrayView(codePointsToAdd, codePointCountToAdd);
+ if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointArrayView.data(),
+ codePointArrayView.size(), unigramProperty, &addedNewUnigram)) {
if (addedNewUnigram && !unigramProperty->representsBeginningOfSentence()) {
mUnigramCount++;
}
if (unigramProperty->getShortcuts().size() > 0) {
// Add shortcut target.
- const int wordPos = getTerminalPtNodePositionOfWord(word, length,
+ const int wordPos = getTerminalPtNodePositionOfWord(codePointArrayView,
false /* forceLowerCaseSearch */);
if (wordPos == NOT_A_DICT_POS) {
AKLOGE("Cannot find terminal PtNode position to add shortcut target.");
@@ -254,12 +256,12 @@
}
}
-bool Ver4PatriciaTriePolicy::removeUnigramEntry(const int *const word, const int length) {
+bool Ver4PatriciaTriePolicy::removeUnigramEntry(const CodePointArrayView wordCodePoints) {
if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary.");
return false;
}
- const int ptNodePos = getTerminalPtNodePositionOfWord(word, length,
+ const int ptNodePos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */);
if (ptNodePos == NOT_A_DICT_POS) {
return false;
@@ -313,7 +315,6 @@
false /* isBlacklisted */, MAX_PROBABILITY /* probability */,
NOT_A_TIMESTAMP /* timestamp */, 0 /* level */, 0 /* count */, &shortcuts);
if (!addUnigramEntry(prevWordsInfo->getNthPrevWordCodePoints(1 /* n */),
- prevWordsInfo->getNthPrevWordCodePointCount(1 /* n */),
&beginningOfSentenceUnigramProperty)) {
AKLOGE("Cannot add unigram entry for the beginning-of-sentence.");
return false;
@@ -326,8 +327,8 @@
}
}
const int word1Pos = getTerminalPtNodePositionOfWord(
- bigramProperty->getTargetCodePoints()->data(),
- bigramProperty->getTargetCodePoints()->size(), false /* forceLowerCaseSearch */);
+ CodePointArrayView(*bigramProperty->getTargetCodePoints()),
+ false /* forceLowerCaseSearch */);
if (word1Pos == NOT_A_DICT_POS) {
return false;
}
@@ -344,7 +345,7 @@
}
bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
- const int *const word, const int length) {
+ const CodePointArrayView wordCodePoints) {
if (!mBuffers->isUpdatable()) {
AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary.");
return false;
@@ -358,8 +359,9 @@
AKLOGE("prev words info is not valid for removing n-gram entry form the dictionary.");
return false;
}
- if (length > MAX_WORD_LENGTH) {
- AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %d", length);
+ if (wordCodePoints.size() > MAX_WORD_LENGTH) {
+ AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %zd",
+ wordCodePoints.size());
}
int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
@@ -369,7 +371,7 @@
if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
return false;
}
- const int wordPos = getTerminalPtNodePositionOfWord(word, length,
+ const int wordPos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */);
if (wordPos == NOT_A_DICT_POS) {
return false;
@@ -453,9 +455,9 @@
}
}
-const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const codePoints,
- const int codePointCount) const {
- const int ptNodePos = getTerminalPtNodePositionOfWord(codePoints, codePointCount,
+const WordProperty Ver4PatriciaTriePolicy::getWordProperty(
+ const CodePointArrayView wordCodePoints) const {
+ const int ptNodePos = getTerminalPtNodePositionOfWord(wordCodePoints,
false /* forceLowerCaseSearch */);
if (ptNodePos == NOT_A_DICT_POS) {
AKLOGE("getWordProperty is called for invalid word.");
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
index 90e06c7..24f92a4 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
@@ -30,6 +30,7 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+#include "utils/int_array_view.h"
namespace latinime {
@@ -65,8 +66,8 @@
const int terminalPtNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const;
- int getTerminalPtNodePositionOfWord(const int *const inWord,
- const int length, const bool forceLowerCaseSearch) const;
+ int getTerminalPtNodePositionOfWord(const CodePointArrayView wordCodePoints,
+ const bool forceLowerCaseSearch) const;
int getProbability(const int unigramProbability, const int bigramProbability) const;
@@ -85,16 +86,16 @@
return &mShortcutPolicy;
}
- bool addUnigramEntry(const int *const word, const int length,
+ bool addUnigramEntry(const CodePointArrayView wordCodePoints,
const UnigramProperty *const unigramProperty);
- bool removeUnigramEntry(const int *const word, const int length);
+ bool removeUnigramEntry(const CodePointArrayView wordCodePoints);
bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
const BigramProperty *const bigramProperty);
- bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word1,
- const int length1);
+ bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
+ const CodePointArrayView wordCodePoints);
bool flush(const char *const filePath);
@@ -105,8 +106,7 @@
void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength);
- const WordProperty getWordProperty(const int *const codePoints,
- const int codePointCount) const;
+ const WordProperty getWordProperty(const CodePointArrayView wordCodePoints) const;
int getNextWordAndNextToken(const int token, int *const outCodePoints,
int *const outCodePointCount);
diff --git a/native/jni/src/utils/int_array_view.h b/native/jni/src/utils/int_array_view.h
index 53f2d29..8797b59 100644
--- a/native/jni/src/utils/int_array_view.h
+++ b/native/jni/src/utils/int_array_view.h
@@ -105,6 +105,7 @@
using WordIdArrayView = IntArrayView;
using PtNodePosArrayView = IntArrayView;
+using CodePointArrayView = IntArrayView;
} // namespace latinime
#endif // LATINIME_MEMORY_VIEW_H