Add method to remove entry from language model dict content.
Bug: 14425059
Change-Id: Id21af0110e770caa3e95cb5d7ba8b3d1af8e0b12
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp
index f3bc4a0..f6721c0 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp
@@ -53,6 +53,16 @@
return mTrieMap.put(terminalId, probabilityEntry->encode(mHasHistoricalInfo), bitmapEntryIndex);
}
+bool LanguageModelDictContent::removeNgramProbabilityEntry(const WordIdArrayView prevWordIds,
+ const int wordId) {
+ const int bitmapEntryIndex = getBitmapEntryIndex(prevWordIds);
+ if (bitmapEntryIndex == TrieMap::INVALID_INDEX) {
+ // Cannot find bitmap entry for the probability entry. The entry doesn't exist.
+ return false;
+ }
+ return mTrieMap.remove(wordId, bitmapEntryIndex);
+}
+
bool LanguageModelDictContent::runGCInner(
const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
const TrieMap::TrieMapRange trieMapRange,
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h
index 104ee25..bd07f2f 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h
@@ -61,12 +61,18 @@
return setNgramProbabilityEntry(WordIdArrayView(), wordId, probabilityEntry);
}
+ bool removeProbabilityEntry(const int wordId) {
+ return removeNgramProbabilityEntry(WordIdArrayView(), wordId);
+ }
+
ProbabilityEntry getNgramProbabilityEntry(const WordIdArrayView prevWordIds,
const int wordId) const;
bool setNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId,
const ProbabilityEntry *const probabilityEntry);
+ bool removeNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId);
+
private:
DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp
index e630aba..944a59c 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp
@@ -420,6 +420,10 @@
bool TrieMap::removeInner(const Entry &bitmapEntry) {
const int tableSize = popCount(bitmapEntry.getBitmap());
+ if (tableSize <= 0) {
+ // The table is empty. No need to remove any entries.
+ return true;
+ }
for (int i = 0; i < tableSize; ++i) {
const int entryIndex = bitmapEntry.getTableIndex() + i;
const Entry entry = readEntry(entryIndex);
@@ -444,7 +448,7 @@
}
}
}
- return freeTable(bitmapEntry.getTableIndex(), tableSize);
+ return true;
}
} // namespace latinime
diff --git a/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp
index 6eef204..a1a3b1a 100644
--- a/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp
+++ b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp
@@ -35,6 +35,13 @@
LanguageModelDictContent.getProbabilityEntry(wordId);
EXPECT_EQ(flag, entry.getFlags());
EXPECT_EQ(probability, entry.getProbability());
+
+ // Remove
+ EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId));
+ EXPECT_FALSE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid());
+ EXPECT_FALSE(LanguageModelDictContent.removeProbabilityEntry(wordId));
+ EXPECT_TRUE(LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry));
+ EXPECT_TRUE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid());
}
TEST(LanguageModelDictContentTest, TestUnigramProbabilityWithHistoricalInfo) {
@@ -53,6 +60,13 @@
EXPECT_EQ(timestamp, entry.getHistoricalInfo()->getTimeStamp());
EXPECT_EQ(level, entry.getHistoricalInfo()->getLevel());
EXPECT_EQ(count, entry.getHistoricalInfo()->getCount());
+
+ // Remove
+ EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId));
+ EXPECT_FALSE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid());
+ EXPECT_FALSE(LanguageModelDictContent.removeProbabilityEntry(wordId));
+ EXPECT_TRUE(LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry));
+ EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId));
}
} // namespace
diff --git a/native/jni/tests/suggest/policyimpl/dictionary/utils/trie_map_test.cpp b/native/jni/tests/suggest/policyimpl/dictionary/utils/trie_map_test.cpp
index 8c8e883..9904a3a 100644
--- a/native/jni/tests/suggest/policyimpl/dictionary/utils/trie_map_test.cpp
+++ b/native/jni/tests/suggest/policyimpl/dictionary/utils/trie_map_test.cpp
@@ -70,6 +70,8 @@
EXPECT_FALSE(result.mIsValid);
EXPECT_EQ(TrieMap::INVALID_INDEX, result.mNextLevelBitmapEntryIndex);
EXPECT_EQ(11ull, trieMap.getRoot(12).mValue);
+ EXPECT_TRUE(trieMap.putRoot(S_INT_MAX, 0xFFFFFFFFFull));
+ EXPECT_TRUE(trieMap.remove(S_INT_MAX, trieMap.getRootBitmapEntryIndex()));
}
TEST(TrieMapTest, TestSetAndGetLarge) {