Merge "Fix a flaky test."
diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java b/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java
index 5c60a73..f841f20 100644
--- a/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java
+++ b/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java
@@ -48,7 +48,6 @@
import com.android.inputmethod.latin.utils.ApplicationUtils;
import com.android.inputmethod.latin.utils.FeedbackUtils;
import com.android.inputmethod.latin.utils.SubtypeLocaleUtils;
-import com.android.inputmethod.research.ResearchLogger;
import com.android.inputmethodcommon.InputMethodSettingsFragment;
import java.util.TreeSet;
@@ -143,12 +142,7 @@
feedbackSettings.setOnPreferenceClickListener(new OnPreferenceClickListener() {
@Override
public boolean onPreferenceClick(final Preference pref) {
- if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) {
- // Use development-only feedback mechanism
- ResearchLogger.getInstance().presentFeedbackDialogFromSettings();
- } else {
- FeedbackUtils.showFeedbackForm(getActivity());
- }
+ FeedbackUtils.showFeedbackForm(getActivity());
return true;
}
});
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index c4383d7..db0c189 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -343,7 +343,7 @@
static const int GET_PROPERTY_RESULT_LENGTH = 100;
char resultChars[GET_PROPERTY_RESULT_LENGTH];
resultChars[0] = '\0';
- dictionary->getProperty(queryChars, resultChars, GET_PROPERTY_RESULT_LENGTH);
+ dictionary->getProperty(queryChars, queryUtf8Length, resultChars, GET_PROPERTY_RESULT_LENGTH);
return env->NewStringUTF(resultChars);
}
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp
index 7b83f27..a990418 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.cpp
+++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp
@@ -120,9 +120,10 @@
return mDictionaryStructureWithBufferPolicy.get()->needsToRunGC(mindsBlockByGC);
}
-void Dictionary::getProperty(const char *const query, char *const outResult,
+void Dictionary::getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength) {
- return mDictionaryStructureWithBufferPolicy.get()->getProperty(query, outResult, maxResultLength);
+ return mDictionaryStructureWithBufferPolicy.get()->getProperty(query, queryLength, outResult,
+ maxResultLength);
}
void Dictionary::logDictionaryInfo(JNIEnv *const env) const {
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h
index e52a40f..977ad35 100644
--- a/native/jni/src/suggest/core/dictionary/dictionary.h
+++ b/native/jni/src/suggest/core/dictionary/dictionary.h
@@ -85,7 +85,7 @@
bool needsToRunGC(const bool mindsBlockByGC);
- void getProperty(const char *const query, char *const outResult,
+ void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength);
const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
index e649844..d7de48a 100644
--- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
+++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
@@ -85,7 +85,7 @@
// Currently, this method is used only for testing. You may want to consider creating new
// dedicated method instead of this if you want to use this in the production.
- virtual void getProperty(const char *const query, char *const outResult,
+ virtual void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength) = 0;
protected:
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp
index 8bbbacd..628d3ab 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp
@@ -52,6 +52,9 @@
false /* hasNext */, newTargetTerminalId, &writingPos)) {
return false;
}
+ if (outAddedNewEntry) {
+ *outAddedNewEntry = true;
+ }
return true;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
index 5d99632..f8644c0 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
@@ -112,7 +112,7 @@
return false;
}
- void getProperty(const char *const query, char *const outResult,
+ void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength) {
// getProperty is not supported for this class.
if (maxResultLength > 0) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.cpp
index b25589c..d4fb937 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.cpp
@@ -34,7 +34,7 @@
namespace latinime {
-// Note that these are corresponding definitions in Java side in BinaryDictionaryTests and
+// Note that there are corresponding definitions in Java side in BinaryDictionaryTests and
// BinaryDictionaryDecayingTests.
const char *const DynamicPatriciaTriePolicy::UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
const char *const DynamicPatriciaTriePolicy::BIGRAM_COUNT_QUERY = "BIGRAM_COUNT";
@@ -277,21 +277,22 @@
return false;
}
-void DynamicPatriciaTriePolicy::getProperty(const char *const query, char *const outResult,
- const int maxResultLength) {
- if (strncmp(query, UNIGRAM_COUNT_QUERY, maxResultLength) == 0) {
+void DynamicPatriciaTriePolicy::getProperty(const char *const query, const int queryLength,
+ char *const outResult, const int maxResultLength) {
+ const int compareLength = queryLength + 1 /* terminator */;
+ if (strncmp(query, UNIGRAM_COUNT_QUERY, compareLength) == 0) {
snprintf(outResult, maxResultLength, "%d", mUnigramCount);
- } else if (strncmp(query, BIGRAM_COUNT_QUERY, maxResultLength) == 0) {
+ } else if (strncmp(query, BIGRAM_COUNT_QUERY, compareLength) == 0) {
snprintf(outResult, maxResultLength, "%d", mBigramCount);
- } else if (strncmp(query, MAX_UNIGRAM_COUNT_QUERY, maxResultLength) == 0) {
+ } else if (strncmp(query, MAX_UNIGRAM_COUNT_QUERY, compareLength) == 0) {
snprintf(outResult, maxResultLength, "%d",
mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_UNIGRAM_COUNT :
static_cast<int>(DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE));
- } else if (strncmp(query, MAX_BIGRAM_COUNT_QUERY, maxResultLength) == 0) {
+ } else if (strncmp(query, MAX_BIGRAM_COUNT_QUERY, compareLength) == 0) {
snprintf(outResult, maxResultLength, "%d",
mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT :
static_cast<int>(DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE));
- } else if (strncmp(query, SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY, maxResultLength) == 0) {
+ } else if (strncmp(query, SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY, compareLength) == 0) {
mNeedsToDecayForTesting = true;
}
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h
index 4546984..636c9bf 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h
@@ -103,7 +103,7 @@
bool needsToRunGC(const bool mindsBlockByGC) const;
- void getProperty(const char *const query, char *const outResult,
+ void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength);
private:
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
index c22b9b4..9066876 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
@@ -72,4 +72,66 @@
return true;
}
+bool BigramDictContent::runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ const BigramDictContent *const originalBigramDictContent,
+ int *const outBigramEntryCount) {
+ for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
+ it != terminalIdMap->end(); ++it) {
+ const int originalBigramListPos =
+ originalBigramDictContent->getBigramListHeadPos(it->first);
+ if (originalBigramListPos == NOT_A_DICT_POS) {
+ // This terminal does not have a bigram list.
+ continue;
+ }
+ const int bigramListPos = getContentBuffer()->getTailPosition();
+ int bigramEntryCount = 0;
+ // Copy bigram list with GC from original content.
+ if (!runGCBigramList(originalBigramListPos, originalBigramDictContent, bigramListPos,
+ terminalIdMap, &bigramEntryCount)) {
+ return false;
+ }
+ if (bigramEntryCount == 0) {
+ // All bigram entries are useless. This terminal does not have a bigram list.
+ continue;
+ }
+ *outBigramEntryCount += bigramEntryCount;
+ // Set bigram list position to the lookup table.
+ if (!getUpdatableAddressLookupTable()->set(it->second, bigramListPos)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool BigramDictContent::runGCBigramList(const int bigramListPos,
+ const BigramDictContent *const sourceBigramDictContent, const int toPos,
+ const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ int *const outEntrycount) {
+ bool hasNext = true;
+ int readingPos = bigramListPos;
+ int writingPos = toPos;
+ while (hasNext) {
+ int probability = NOT_A_PROBABILITY;
+ int targetTerminalId = Ver4DictConstants::NOT_A_TERMINAL_ID;
+ sourceBigramDictContent->getBigramEntryAndAdvancePosition(&probability, &hasNext,
+ &targetTerminalId, &readingPos);
+ if (targetTerminalId == Ver4DictConstants::NOT_A_TERMINAL_ID) {
+ continue;
+ }
+ TerminalPositionLookupTable::TerminalIdMap::const_iterator it =
+ terminalIdMap->find(targetTerminalId);
+ if (it == terminalIdMap->end()) {
+ AKLOGE("terminal Id %d is not in the terminal position map. map size: %zd",
+ targetTerminalId, terminalIdMap->size());
+ return false;
+ }
+ if (!writeBigramEntryAndAdvancePosition(probability, hasNext, it->second,
+ &writingPos)) {
+ return false;
+ }
+ *outEntrycount += 1;
+ }
+ return true;
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
index c5410b8..ec0d756 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
@@ -19,6 +19,7 @@
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
namespace latinime {
@@ -65,6 +66,10 @@
Ver4DictConstants::BIGRAM_FILE_EXTENSION);
}
+ bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ const BigramDictContent *const originalBigramDictContent,
+ int *const outBigramEntryCount);
+
private:
DISALLOW_COPY_AND_ASSIGN(BigramDictContent);
@@ -72,6 +77,11 @@
return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK)
| (hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0);
}
+
+ bool runGCBigramList(const int bigramListPos,
+ const BigramDictContent *const sourceBigramDictContent, const int toPos,
+ const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ int *const outEntryCount);
};
} // namespace latinime
#endif /* LATINIME_BIGRAM_DICT_CONTENT_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
index 73a472a..0958dd1 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
@@ -19,6 +19,7 @@
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
@@ -28,13 +29,14 @@
class ProbabilityDictContent : public SingleDictContent {
public:
ProbabilityDictContent(const char *const dictDirPath, const bool isUpdatable)
- : SingleDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION,
- isUpdatable) {}
+ : SingleDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable),
+ mSize(getBuffer()->getTailPosition() / (Ver4DictConstants::PROBABILITY_SIZE
+ + Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE)) {}
- ProbabilityDictContent() {}
+ ProbabilityDictContent() : mSize(0) {}
int getProbability(const int terminalId) const {
- if (terminalId < 0 || terminalId >= getSize()) {
+ if (terminalId < 0 || terminalId >= mSize) {
return NOT_A_PROBABILITY;
}
return Ver4PatriciaTrieReadingUtils::getProbability(getBuffer(), terminalId);
@@ -44,7 +46,7 @@
if (terminalId < 0) {
return false;
}
- if (terminalId >= getSize()) {
+ if (terminalId >= mSize) {
// Write new entry.
int writingPos = getBuffer()->getTailPosition();
while (writingPos <= getEntryPos(terminalId)) {
@@ -58,6 +60,7 @@
Ver4DictConstants::PROBABILITY_SIZE, &writingPos)) {
return false;
}
+ mSize++;
}
}
const int probabilityWritingPos = getEntryPos(terminalId)
@@ -67,7 +70,32 @@
}
bool flushToFile(const char *const dictDirPath) const {
- return flush(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION);
+ if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
+ ProbabilityDictContent probabilityDictContentToWrite;
+ for (int i = 0; i < mSize; ++i) {
+ if (!probabilityDictContentToWrite.setProbability(i, getProbability(i))) {
+ return false;
+ }
+ }
+ return probabilityDictContentToWrite.flush(dictDirPath,
+ Ver4DictConstants::FREQ_FILE_EXTENSION);
+ } else {
+ return flush(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION);
+ }
+ }
+
+ bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ const ProbabilityDictContent *const originalProbabilityDictContent) {
+ mSize = 0;
+ for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
+ it != terminalIdMap->end(); ++it) {
+ if (!setProbability(it->second,
+ originalProbabilityDictContent->getProbability(it->first))) {
+ return false;
+ }
+ mSize++;
+ }
+ return true;
}
private:
@@ -78,10 +106,7 @@
+ Ver4DictConstants::PROBABILITY_SIZE);
}
- int getSize() const {
- return getBuffer()->getTailPosition() / (Ver4DictConstants::PROBABILITY_SIZE
- + Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE);
- }
+ int mSize;
};
} // namespace latinime
#endif /* LATINIME_PROBABILITY_DICT_CONTENT_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
index f09bd44..14eff67 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
@@ -48,6 +48,30 @@
Ver4DictConstants::SHORTCUT_FILE_EXTENSION);
}
+bool ShortcutDictContent::runGC(
+ const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ const ShortcutDictContent *const originalShortcutDictContent) {
+ for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
+ it != terminalIdMap->end(); ++it) {
+ const int originalShortcutListPos =
+ originalShortcutDictContent->getShortcutListHeadPos(it->first);
+ if (originalShortcutListPos == NOT_A_DICT_POS) {
+ continue;
+ }
+ const int shortcutListPos = getContentBuffer()->getTailPosition();
+ // Copy shortcut list with GC from original content.
+ if (!copyShortcutList(originalShortcutListPos, originalShortcutDictContent,
+ shortcutListPos)) {
+ return false;
+ }
+ // Set shortcut list position to the lookup table.
+ if (!getUpdatableAddressLookupTable()->set(it->second, shortcutListPos)) {
+ return false;
+ }
+ }
+ return true;
+}
+
bool ShortcutDictContent::copyShortcutList(const int shortcutListPos,
const ShortcutDictContent *const sourceShortcutDictContent, const int toPos) {
bool hasNext = true;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
index 71a8f6b..902016a 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
@@ -47,6 +47,9 @@
bool flushToFile(const char *const dictDirPath) const;
+ bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
+ const ShortcutDictContent *const originalShortcutDictContent);
+
private:
DISALLOW_COPY_AND_ASSIGN(ShortcutDictContent);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
index 1c5caa1..c654206 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
@@ -18,22 +18,6 @@
namespace latinime {
-bool SparseTableDictContent::copyContent(
- const SparseTableDictContent *const sparseTableDictContent) {
- if (!mExpandableLookupTableBuffer.copy(
- &sparseTableDictContent->mExpandableLookupTableBuffer)) {
- return false;
- }
- if (!mExpandableAddressTableBuffer.copy(
- &sparseTableDictContent->mExpandableAddressTableBuffer)) {
- return false;
- }
- if (!mExpandableContentBuffer.copy(&sparseTableDictContent->mExpandableContentBuffer)) {
- return false;
- }
- return true;
-}
-
bool SparseTableDictContent::flush(const char *const dictDirPath,
const char *const lookupTableFileName, const char *const addressTableFileName,
const char *const contentFileName) const {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
index 1515687..9a4f1e1 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
@@ -75,8 +75,6 @@
|| mExpandableContentBuffer.isNearSizeLimit();
}
- bool copyContent(const SparseTableDictContent *const sparseTableDictContent);
-
protected:
SparseTable *getUpdatableAddressLookupTable() {
return &mAddressLookupTable;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
index 0717925..e8a3142 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
@@ -126,6 +126,12 @@
newChildrenPosition, &childrenPosFieldPos);
}
+bool Ver4PatriciaTrieNodeWriter::updateTerminalId(const PtNodeParams *const toBeUpdatedPtNodeParams,
+ const int newTerminalId) {
+ return mTrieBuffer->writeUint(newTerminalId, Ver4DictConstants::TERMINAL_ID_FIELD_SIZE,
+ toBeUpdatedPtNodeParams->getTerminalIdFieldPos());
+}
+
bool Ver4PatriciaTrieNodeWriter::writePtNodeAndAdvancePosition(
const PtNodeParams *const ptNodeParams, int *const ptNodeWritingPos) {
const int nodePos = *ptNodeWritingPos;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
index 23e54cc..1245152 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
@@ -57,6 +57,9 @@
virtual bool updateChildrenPosition(const PtNodeParams *const toBeUpdatedPtNodeParams,
const int newChildrenPosition);
+ bool updateTerminalId(const PtNodeParams *const toBeUpdatedPtNodeParams,
+ const int newTerminalId);
+
virtual bool writePtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams,
int *const ptNodeWritingPos);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
index eeab312..7c4025a 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
@@ -25,6 +25,14 @@
namespace latinime {
+// Note that there are corresponding definitions in Java side in BinaryDictionaryTests and
+// BinaryDictionaryDecayingTests.
+const char *const Ver4PatriciaTriePolicy::UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT";
+const char *const Ver4PatriciaTriePolicy::BIGRAM_COUNT_QUERY = "BIGRAM_COUNT";
+const char *const Ver4PatriciaTriePolicy::MAX_UNIGRAM_COUNT_QUERY = "MAX_UNIGRAM_COUNT";
+const char *const Ver4PatriciaTriePolicy::MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT";
+const char *const Ver4PatriciaTriePolicy::SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY =
+ "SET_NEEDS_TO_DECAY_FOR_TESTING";
const int Ver4PatriciaTriePolicy::MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS = 1024;
const int Ver4PatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS =
Ver4DictConstants::MAX_DICTIONARY_SIZE - MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS;
@@ -257,9 +265,24 @@
return false;
}
-void Ver4PatriciaTriePolicy::getProperty(const char *const query, char *const outResult,
- const int maxResultLength) {
- // TODO: Implement.
+void Ver4PatriciaTriePolicy::getProperty(const char *const query, const int queryLength,
+ char *const outResult, const int maxResultLength) {
+ const int compareLength = queryLength + 1 /* terminator */;
+ if (strncmp(query, UNIGRAM_COUNT_QUERY, compareLength) == 0) {
+ snprintf(outResult, maxResultLength, "%d", mUnigramCount);
+ } else if (strncmp(query, BIGRAM_COUNT_QUERY, compareLength) == 0) {
+ snprintf(outResult, maxResultLength, "%d", mBigramCount);
+ } else if (strncmp(query, MAX_UNIGRAM_COUNT_QUERY, compareLength) == 0) {
+ snprintf(outResult, maxResultLength, "%d",
+ mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_UNIGRAM_COUNT :
+ static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE));
+ } else if (strncmp(query, MAX_BIGRAM_COUNT_QUERY, compareLength) == 0) {
+ snprintf(outResult, maxResultLength, "%d",
+ mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT :
+ static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE));
+ } else if (strncmp(query, SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY, compareLength) == 0) {
+ mNeedsToDecayForTesting = true;
+ }
}
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
index 4d35caf..6fe978d 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
@@ -103,12 +103,17 @@
bool needsToRunGC(const bool mindsBlockByGC) const;
- void getProperty(const char *const query, char *const outResult,
+ void getProperty(const char *const query, const int queryLength, char *const outResult,
const int maxResultLength);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTriePolicy);
+ static const char *const UNIGRAM_COUNT_QUERY;
+ static const char *const BIGRAM_COUNT_QUERY;
+ static const char *const MAX_UNIGRAM_COUNT_QUERY;
+ static const char *const MAX_BIGRAM_COUNT_QUERY;
+ static const char *const SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY;
// When the dictionary size is near the maximum size, we have to refuse dynamic operations to
// prevent the dictionary from overflowing.
static const int MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
index 11adf2c..f141d52 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
@@ -20,7 +20,6 @@
#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
-#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h"
#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
@@ -141,15 +140,6 @@
Ver4PatriciaTrieNodeWriter newPtNodeWriter(buffersToWrite->getWritableTrieBuffer(),
buffersToWrite, &newPtNodeReader, &newBigramPolicy, &newShortcutPolicy);
- if(!buffersToWrite->getUpdatableBigramDictContent()->copyContent(
- mBuffers->getBigramDictContent())) {
- return false;
- }
- if(!buffersToWrite->getUpdatableShortcutDictContent()->copyContent(
- mBuffers->getShortcutDictContent())) {
- return false;
- }
-
DynamicPatriciaTrieReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(),
&newPtNodeReader);
newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
@@ -160,11 +150,50 @@
return false;
}
- // TODO: GC for dict contents.
-
+ // Re-assign terminal IDs for valid terminal PtNodes.
+ TerminalPositionLookupTable::TerminalIdMap terminalIdMap;
+ if(!buffersToWrite->getUpdatableTerminalPositionLookupTable()->runGCTerminalIds(
+ &terminalIdMap)) {
+ return false;
+ }
+ TraversePolicyToUpdateAllTerminalIds traversePolicyToUpdateAllTerminalIds(&newPtNodeWriter,
+ &terminalIdMap);
+ if (!newDictReadingHelper.traverseAllPtNodesInPostorderDepthFirstManner(
+ &traversePolicyToUpdateAllTerminalIds)) {
+ return false;
+ }
+ // Run GC for probability dict content.
+ if (!buffersToWrite->getUpdatableProbabilityDictContent()->runGC(&terminalIdMap,
+ mBuffers->getProbabilityDictContent())) {
+ return false;
+ }
+ // Run GC for bigram dict content.
+ if(!buffersToWrite->getUpdatableBigramDictContent()->runGC(&terminalIdMap,
+ mBuffers->getBigramDictContent(), outBigramCount)) {
+ return false;
+ }
+ // Run GC for shortcut dict content.
+ if(!buffersToWrite->getUpdatableShortcutDictContent()->runGC(&terminalIdMap,
+ mBuffers->getShortcutDictContent())) {
+ return false;
+ }
*outUnigramCount = traversePolicyToUpdateAllPositionFields.getUnigramCount();
- *outBigramCount = traversePolicyToUpdateAllPositionFields.getBigramCount();
return true;
}
+bool Ver4PatriciaTrieWritingHelper::TraversePolicyToUpdateAllTerminalIds::onVisitingPtNode(
+ const PtNodeParams *const ptNodeParams) {
+ if (!ptNodeParams->isTerminal()) {
+ return true;
+ }
+ TerminalPositionLookupTable::TerminalIdMap::const_iterator it =
+ mTerminalIdMap->find(ptNodeParams->getTerminalId());
+ if (it == mTerminalIdMap->end()) {
+ AKLOGE("terminal Id %d is not in the terminal position map. map size: %zd",
+ ptNodeParams->getTerminalId(), mTerminalIdMap->size());
+ return false;
+ }
+ return mPtNodeWriter->updateTerminalId(ptNodeParams, it->second);
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
index 8072aa3..ea2fd53 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
@@ -18,11 +18,14 @@
#define LATINIME_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h"
+#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
namespace latinime {
class HeaderPolicy;
class Ver4DictBuffers;
+class Ver4PatriciaTrieNodeWriter;
class Ver4PatriciaTrieWritingHelper {
public:
@@ -39,6 +42,28 @@
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper);
+ class TraversePolicyToUpdateAllTerminalIds
+ : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
+ public:
+ TraversePolicyToUpdateAllTerminalIds(Ver4PatriciaTrieNodeWriter *const ptNodeWriter,
+ const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap)
+ : mPtNodeWriter(ptNodeWriter), mTerminalIdMap(terminalIdMap) {};
+
+ bool onAscend() { return true; }
+
+ bool onDescend(const int ptNodeArrayPos) { return true; }
+
+ bool onReadingPtNodeArrayTail() { return true; }
+
+ bool onVisitingPtNode(const PtNodeParams *const ptNodeParams);
+
+ private:
+ DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllTerminalIds);
+
+ Ver4PatriciaTrieNodeWriter *const mPtNodeWriter;
+ const TerminalPositionLookupTable::TerminalIdMap *const mTerminalIdMap;
+ };
+
bool runGC(const int rootPtNodeArrayPos, const HeaderPolicy *const headerPolicy,
Ver4DictBuffers *const buffersToWrite, int *const outUnigramCount,
int *const outBigramCount, const bool needsToDecay);