Merge "Add a new option for version 4 to dicttool."
diff --git a/java/res/values/donottranslate.xml b/java/res/values/donottranslate.xml
index 82c5ce4..42e692d 100644
--- a/java/res/values/donottranslate.xml
+++ b/java/res/values/donottranslate.xml
@@ -158,47 +158,47 @@
<!-- Compatibility map from subtypeLocale:subtypeExtraValue to keyboardLayoutSet -->
<string-array name="locale_and_extra_value_to_keyboard_layout_set_map">
- <item>en_US:TrySuppressingImeSwitcher,AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>en_US:TrySuppressingImeSwitcher,AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwerty</item>
- <item>en_GB:TrySuppressingImeSwitcher,AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>en_GB:TrySuppressingImeSwitcher,AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwerty</item>
- <item>ar:SupportTouchPositionCorrection</item>
+ <item>ar:SupportTouchPositionCorrection,EmojiCapable</item>
<item>arabic</item>
- <item>cs:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>cs:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwertz</item>
- <item>da:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>da:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>nordic</item>
- <item>de:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>de:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwertz</item>
- <item>es:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>es:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>spanish</item>
- <item>fi:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>fi:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>nordic</item>
- <item>fr:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>fr:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>azerty</item>
- <item>fr_CA:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>fr_CA:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwerty</item>
- <item>hr:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>hr:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwertz</item>
- <item>hu:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>hu:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwertz</item>
- <item>it:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>it:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwerty</item>
- <item>iw:SupportTouchPositionCorrection</item>
+ <item>iw:SupportTouchPositionCorrection,EmojiCapable</item>
<item>hebrew</item>
- <item>nb:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>nb:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>nordic</item>
- <item>nl:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>nl:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwerty</item>
- <item>pl:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>pl:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwerty</item>
- <item>ru:SupportTouchPositionCorrection</item>
+ <item>ru:SupportTouchPositionCorrection,EmojiCapable</item>
<item>east_slavic</item>
- <item>sr:SupportTouchPositionCorrection</item>
+ <item>sr:SupportTouchPositionCorrection,EmojiCapable</item>
<item>south_slavic</item>
- <item>sv:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>sv:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>nordic</item>
- <item>tr:AsciiCapable,SupportTouchPositionCorrection</item>
+ <item>tr:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item>
<item>qwerty</item>
</string-array>
diff --git a/java/res/xml-sw600dp/row_pcqwerty5.xml b/java/res/xml-sw600dp/row_pcqwerty5.xml
index a79d2a8..b854f10 100644
--- a/java/res/xml-sw600dp/row_pcqwerty5.xml
+++ b/java/res/xml-sw600dp/row_pcqwerty5.xml
@@ -53,7 +53,7 @@
latin:keyXPos="-9.0%p"
latin:keyWidth="9.0%p"
latin:backgroundType="functional"
- latin:keyboardLayout="@xml/key_symbols_period" />
+ latin:keyboardLayout="@xml/key_f2" />
</default>
</switch>
</Row>
diff --git a/java/res/xml/method.xml b/java/res/xml/method.xml
index d7424c0..d92dc29 100644
--- a/java/res/xml/method.xml
+++ b/java/res/xml/method.xml
@@ -62,8 +62,8 @@
mn: Mongolian/mongolian
ms: Malay/qwerty
nb: Norwegian Bokmål/nordic
- ne: Nepali Romanized/nepali_romanized
- ne: Nepali Traditional/nepali_traditional
+ ne: Nepali Romanized/nepali_romanized # disabled temporarily
+ ne: Nepali Traditional/nepali_traditional # disabled temporarily
nl: Dutch/qwerty
nl_BE: Dutch Belgium/azerty
pl: Polish/qwerty
@@ -380,6 +380,7 @@
android:imeSubtypeMode="keyboard"
android:imeSubtypeExtraValue="AsciiCapable,SupportTouchPositionCorrection,EmojiCapable"
/>
+ <!--
<subtype android:icon="@drawable/ic_subtype_keyboard"
android:label="@string/subtype_generic"
android:subtypeId="0xd80a4cee"
@@ -394,6 +395,7 @@
android:imeSubtypeMode="keyboard"
android:imeSubtypeExtraValue="KeyboardLayoutSet=nepali_traditional,EmojiCapable"
/>
+ -->
<subtype android:icon="@drawable/ic_subtype_keyboard"
android:label="@string/subtype_generic"
android:subtypeId="0x3f9fd91e"
diff --git a/java/res/xml/row_pcqwerty5.xml b/java/res/xml/row_pcqwerty5.xml
index 0e61805..4ec908b 100644
--- a/java/res/xml/row_pcqwerty5.xml
+++ b/java/res/xml/row_pcqwerty5.xml
@@ -51,13 +51,13 @@
latin:keyWidth="11.538%p" />
<Key
latin:keyStyle="spaceKeyStyle"
- latin:keyWidth="42.310%p" />
+ latin:keyWidth="38.464%p" />
</case>
<!-- languageSwitchKeyEnabled="false" -->
<default>
<Key
latin:keyStyle="spaceKeyStyle"
- latin:keyWidth="53.848%p" />
+ latin:keyWidth="50.002%p" />
</default>
</switch>
<Key
@@ -71,9 +71,9 @@
</case>
<!-- keyboardLayoutSetElement="alphabetManualShifted|alphabetShiftLocked|alphabetShiftLockShifted" -->
<default>
- <include
- latin:keyboardLayout="@xml/key_symbols_period"
- latin:backgroundType="functional" />
+ <Key
+ latin:keyStyle="emojiKeyStyle"
+ latin:keyWidth="fillRight" />
</default>
</switch>
</Row>
diff --git a/java/src/com/android/inputmethod/keyboard/ProximityInfo.java b/java/src/com/android/inputmethod/keyboard/ProximityInfo.java
index cd127c7..a031669 100644
--- a/java/src/com/android/inputmethod/keyboard/ProximityInfo.java
+++ b/java/src/com/android/inputmethod/keyboard/ProximityInfo.java
@@ -245,8 +245,8 @@
final int threshold = (int) (defaultWidth * SEARCH_DISTANCE);
final int thresholdSquared = threshold * threshold;
// Round-up so we don't have any pixels outside the grid
- final int fullGridWidth = mGridWidth * mCellWidth;
- final int fullGridHeight = mGridHeight * mCellHeight;
+ final int lastPixelXCoordinate = mGridWidth * mCellWidth - 1;
+ final int lastPixelYCoordinate = mGridHeight * mCellHeight - 1;
// For large layouts, 'neighborsFlatBuffer' is about 80k of memory: gridSize is usually 512,
// keycount is about 40 and a pointer to a Key is 4 bytes. This contains, for each cell,
@@ -329,22 +329,20 @@
final int yMiddleOfTopCell = topPixelWithinThreshold - yDeltaToGrid + halfCellHeight;
final int yStart = Math.max(halfCellHeight,
yMiddleOfTopCell + (yDeltaToGrid <= halfCellHeight ? 0 : mCellHeight));
- final int yEnd = Math.min(fullGridHeight, keyY + key.getHeight() + threshold);
+ final int yEnd = Math.min(lastPixelYCoordinate, keyY + key.getHeight() + threshold);
final int leftPixelWithinThreshold = keyX - threshold;
final int xDeltaToGrid = leftPixelWithinThreshold % mCellWidth;
final int xMiddleOfLeftCell = leftPixelWithinThreshold - xDeltaToGrid + halfCellWidth;
final int xStart = Math.max(halfCellWidth,
xMiddleOfLeftCell + (xDeltaToGrid <= halfCellWidth ? 0 : mCellWidth));
- final int xEnd = Math.min(fullGridWidth, keyX + key.getWidth() + threshold);
+ final int xEnd = Math.min(lastPixelXCoordinate, keyX + key.getWidth() + threshold);
int baseIndexOfCurrentRow = (yStart / mCellHeight) * mGridWidth + (xStart / mCellWidth);
for (int centerY = yStart; centerY <= yEnd; centerY += mCellHeight) {
int index = baseIndexOfCurrentRow;
for (int centerX = xStart; centerX <= xEnd; centerX += mCellWidth) {
- // TODO: Remove "index < neighborCountPerCell.length" below.
- if (index < neighborCountPerCell.length
- && key.squaredDistanceToEdge(centerX, centerY) < thresholdSquared) {
+ if (key.squaredDistanceToEdge(centerX, centerY) < thresholdSquared) {
neighborsFlatBuffer[index * keyCount + neighborCountPerCell[index]] = key;
++neighborCountPerCell[index];
}
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index 2a90764..fcd7ede 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -311,9 +311,10 @@
}
@Override
- public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
+ public ArrayList<SuggestedWordInfo> getSuggestionsWithSessionId(final WordComposer composer,
final String prevWord, final ProximityInfo proximityInfo,
- final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) {
+ final boolean blockOffensiveWords, final int[] additionalFeaturesOptions,
+ final int sessionId) {
reloadDictionaryIfRequired();
final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList();
final AsyncResultHolder<ArrayList<SuggestedWordInfo>> holder =
@@ -321,14 +322,16 @@
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
public void run() {
- final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
- mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo,
- blockOffensiveWords, additionalFeaturesOptions);
+ final ArrayList<SuggestedWordInfo> inMemDictSuggestion = composer.isBatchMode() ?
+ null : mDictionaryWriter.getSuggestionsWithSessionId(composer, prevWord,
+ proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
+ sessionId);
// TODO: Remove checking mIsUpdatable and use native suggestion.
if (mBinaryDictionary != null && !mIsUpdatable) {
final ArrayList<SuggestedWordInfo> binarySuggestion =
- mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo,
- blockOffensiveWords, additionalFeaturesOptions);
+ mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
+ proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
+ sessionId);
if (inMemDictSuggestion == null) {
holder.set(binarySuggestion);
} else if (binarySuggestion == null) {
@@ -342,11 +345,18 @@
}
}
});
-
return holder.get(null, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);
}
@Override
+ public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
+ final String prevWord, final ProximityInfo proximityInfo,
+ final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) {
+ return getSuggestionsWithSessionId(composer, prevWord, proximityInfo, blockOffensiveWords,
+ additionalFeaturesOptions, 0 /* sessionId */);
+ }
+
+ @Override
public boolean isValidWord(final String word) {
reloadDictionaryIfRequired();
return isValidWordInner(word);
diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java
index 1684d47..6c18c94 100644
--- a/java/src/com/android/inputmethod/latin/Suggest.java
+++ b/java/src/com/android/inputmethod/latin/Suggest.java
@@ -361,12 +361,6 @@
// At second character typed, search the unigrams (scores being affected by bigrams)
for (final String key : mDictionaries.keySet()) {
- // Skip User history dictionary for lookup
- // TODO: The user history dictionary should just override getSuggestionsWithSessionId
- // to make sure it doesn't return anything and we should remove this test
- if (key.equals(Dictionary.TYPE_USER_HISTORY)) {
- continue;
- }
final Dictionary dictionary = mDictionaries.get(key);
suggestionsSet.addAll(dictionary.getSuggestionsWithSessionId(wordComposer,
prevWordForBigram, proximityInfo, blockOffensiveWords,
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h
index 884bcd7..9092a80 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h
@@ -98,6 +98,13 @@
flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
*outOffsetFieldSize = 1;
}
+
+ // Currently, all newly written bigram position fields are 3 bytes to simplify dictionary
+ // writing.
+ // TODO: Remove following 2 lines and optimize memory space.
+ flags = (flags & (~MASK_ATTRIBUTE_ADDRESS_TYPE)) | FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES;
+ *outOffsetFieldSize = 3;
+
*outBigramFlags = flags;
*outOffset = absOffest;
return true;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
index 4c44d22..bd58b99 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp
@@ -54,8 +54,8 @@
}
}
-bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos,
- int *outBigramsCount) {
+bool DynamicBigramListPolicy::copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite,
+ int *const fromPos, int *const toPos, int *const outBigramsCount) const {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos);
if (usesAdditionalBuffer) {
*fromPos -= mBuffer->getOriginalBufferSize();
@@ -86,10 +86,10 @@
continue;
}
// Write bigram entry. Target buffer is always the additional buffer.
- if (!mBuffer->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */,toPos)) {
+ if (!bufferToWrite->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */,toPos)) {
return false;
}
- if (!mBuffer->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize,
+ if (!bufferToWrite->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize,
toPos)) {
return false;
}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
index dafb62d..5d02d32 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h
@@ -44,10 +44,11 @@
void skipAllBigrams(int *const pos) const;
- // Copy bigrams from the bigram list that starts at fromPos to toPos and advance these
- // positions after bigram lists. This method skips invalid bigram entries and write the valid
- // bigram entry count to outBigramsCount.
- bool copyAllBigrams(int *const fromPos, int *const toPos, int *outBigramsCount);
+ // Copy bigrams from the bigram list that starts at fromPos in mBuffer to toPos in
+ // bufferToWrite and advance these positions after bigram lists. This method skips invalid
+ // bigram entries and write the valid bigram entry count to outBigramsCount.
+ bool copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite, int *const fromPos,
+ int *const toPos, int *const outBigramsCount) const;
bool addNewBigramEntryToBigramList(const int bigramPos, const int probability, int *const pos);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
index 31178fb..f4c98d8 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp
@@ -97,8 +97,8 @@
return false;
}
int writingPos = newNodePos;
- // Write a new PtNode using original PtNode's info to the tail of the dictionary.
- if (!writePtNodeToBufferByCopyingPtNodeInfo(&nodeReader, nodeReader.getParentPos(),
+ // Write a new PtNode using original PtNode's info to the tail of the dictionary in mBuffer.
+ if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, &nodeReader, nodeReader.getParentPos(),
mMergedNodeCodePoints, nodeReader.getCodePointCount(), nodeReader.getProbability(),
&writingPos)) {
return false;
@@ -143,38 +143,20 @@
if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) {
return;
}
- const int tmpFileNameBufSize = strlen(fileName)
- + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1;
- char tmpFileName[tmpFileNameBufSize];
- snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName,
- TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
- FILE *const file = fopen(tmpFileName, "wb");
- if (!file) {
+ flushAllToFile(fileName, &headerBuffer, mBuffer);
+}
+
+void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
+ const char *const fileName, const HeaderPolicy *const headerPolicy) {
+ BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
+ if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */)) {
return;
}
- // Write header.
- if (fwrite(headerBuffer.getBuffer(true /* usesAdditionalBuffer */),
- headerBuffer.getTailPosition(), 1, file) < 1) {
- fclose(file);
- remove(tmpFileName);
+ BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */);
+ if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) {
return;
}
- // Write data in original buffer.
- if (fwrite(mBuffer->getBuffer(false /* usesAdditionalBuffer */),
- mBuffer->getOriginalBufferSize(), 1, file) < 1) {
- fclose(file);
- remove(tmpFileName);
- return;
- }
- // Write data in additional buffer.
- if (fwrite(mBuffer->getBuffer(true /* usesAdditionalBuffer */),
- mBuffer->getTailPosition() - mBuffer->getOriginalBufferSize(), 1, file) < 1) {
- fclose(file);
- remove(tmpFileName);
- return;
- }
- fclose(file);
- rename(tmpFileName, fileName);
+ flushAllToFile(fileName, &headerBuffer, &newDictBuffer);
}
bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition(
@@ -232,7 +214,8 @@
}
// Write new PtNode at writingPos.
-bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const bool isBlacklisted,
+bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(
+ BufferWithExtendableBuffer *const bufferToWrite, const bool isBlacklisted,
const bool isNotAWord, const int parentPos, const int *const codePoints,
const int codePointCount, const int probability, const int childrenPos,
const int originalBigramListPos, const int originalShortcutListPos,
@@ -240,38 +223,39 @@
const int nodePos = *writingPos;
// Write dummy flags. The Node flags are updated with appropriate flags at the last step of the
// PtNode writing.
- if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, 0 /* nodeFlags */,
- writingPos)) {
+ if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(bufferToWrite,
+ 0 /* nodeFlags */, writingPos)) {
return false;
}
// Calculate a parent offset and write the offset.
const int parentOffset = (parentPos != NOT_A_DICT_POS) ? parentPos - nodePos : NOT_A_DICT_POS;
- if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(mBuffer,
+ if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(bufferToWrite,
parentOffset, writingPos)) {
return false;
}
// Write code points
- if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mBuffer, codePoints,
- codePointCount, writingPos)) {
+ if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(bufferToWrite,
+ codePoints, codePointCount, writingPos)) {
return false;
}
// Write probability when the probability is a valid probability, which means this node is
// terminal.
if (probability != NOT_A_PROBABILITY) {
- if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(mBuffer,
+ if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(bufferToWrite,
probability, writingPos)) {
return false;
}
}
// Write children position
- if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer,
+ if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(bufferToWrite,
childrenPos, writingPos)) {
return false;
}
// Copy shortcut list when the originalShortcutListPos is valid dictionary position.
if (originalShortcutListPos != NOT_A_DICT_POS) {
int fromPos = originalShortcutListPos;
- if (!mShortcutPolicy->copyAllShortcutsAndReturnIfSucceededOrNot(&fromPos, writingPos)) {
+ if (!mShortcutPolicy->copyAllShortcutsAndReturnIfSucceededOrNot(bufferToWrite, &fromPos,
+ writingPos)) {
return false;
}
}
@@ -279,7 +263,7 @@
int bigramCount = 0;
if (originalBigramListPos != NOT_A_DICT_POS) {
int fromPos = originalBigramListPos;
- if (!mBigramPolicy->copyAllBigrams(&fromPos, writingPos, &bigramCount)) {
+ if (!mBigramPolicy->copyAllBigrams(bufferToWrite, &fromPos, writingPos, &bigramCount)) {
return false;
}
}
@@ -291,27 +275,29 @@
bigramCount > 0 /* hasBigrams */, codePointCount > 1 /* hasMultipleChars */,
CHILDREN_POSITION_FIELD_SIZE);
int flagsFieldPos = nodePos;
- if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, nodeFlags,
+ if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(bufferToWrite, nodeFlags,
&flagsFieldPos)) {
return false;
}
return true;
}
-bool DynamicPatriciaTrieWritingHelper::writePtNodeToBuffer(const int parentPos,
+bool DynamicPatriciaTrieWritingHelper::writePtNodeToBuffer(
+ BufferWithExtendableBuffer *const bufferToWrite, const int parentPos,
const int *const codePoints, const int codePointCount, const int probability,
int *const writingPos) {
- return writePtNodeWithFullInfoToBuffer(false /* isBlacklisted */, false /* isNotAWord */,
- parentPos, codePoints, codePointCount, probability,
+ return writePtNodeWithFullInfoToBuffer(bufferToWrite, false /* isBlacklisted */,
+ false /* isNotAWord */, parentPos, codePoints, codePointCount, probability,
NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */,
NOT_A_DICT_POS /* originalShortcutPos */, writingPos);
}
bool DynamicPatriciaTrieWritingHelper::writePtNodeToBufferByCopyingPtNodeInfo(
+ BufferWithExtendableBuffer *const bufferToWrite,
const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos,
const int *const codePoints, const int codePointCount, const int probability,
int *const writingPos) {
- return writePtNodeWithFullInfoToBuffer(originalNode->isBlacklisted(),
+ return writePtNodeWithFullInfoToBuffer(bufferToWrite, originalNode->isBlacklisted(),
originalNode->isNotAWord(), parentPos, codePoints, codePointCount, probability,
originalNode->getChildrenPos(), originalNode->getBigramsPos(),
originalNode->getShortcutPos(), writingPos);
@@ -345,8 +331,9 @@
if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) {
return false;
}
- if (!writePtNodeToBufferByCopyingPtNodeInfo(originalPtNode, originalPtNode->getParentPos(),
- codePoints, originalPtNode->getCodePointCount(), probability, &movedPos)) {
+ if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, originalPtNode,
+ originalPtNode->getParentPos(), codePoints, originalPtNode->getCodePointCount(),
+ probability, &movedPos)) {
return false;
}
}
@@ -374,8 +361,8 @@
1 /* arraySize */, &writingPos)) {
return false;
}
- if (!writePtNodeToBuffer(parentPtNodePos, nodeCodePoints, nodeCodePointCount, probability,
- &writingPos)) {
+ if (!writePtNodeToBuffer(mBuffer, parentPtNodePos, nodeCodePoints, nodeCodePointCount,
+ probability, &writingPos)) {
return false;
}
if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
@@ -404,8 +391,9 @@
// Write the 1st part of the reallocating node. The children position will be updated later
// with actual children position.
const int newProbability = addsExtraChild ? NOT_A_PROBABILITY : probabilityOfNewPtNode;
- if (!writePtNodeToBuffer(reallocatingPtNode->getParentPos(), reallocatingPtNodeCodePoints,
- overlappingCodePointCount, newProbability, &writingPos)) {
+ if (!writePtNodeToBuffer(mBuffer, reallocatingPtNode->getParentPos(),
+ reallocatingPtNodeCodePoints, overlappingCodePointCount, newProbability,
+ &writingPos)) {
return false;
}
const int actualChildrenPos = writingPos;
@@ -417,14 +405,15 @@
}
// Write the 2nd part of the reallocating node.
const int secondPartOfReallocatedPtNodePos = writingPos;
- if (!writePtNodeToBufferByCopyingPtNodeInfo(reallocatingPtNode, firstPartOfReallocatedPtNodePos,
+ if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, reallocatingPtNode,
+ firstPartOfReallocatedPtNodePos,
reallocatingPtNodeCodePoints + overlappingCodePointCount,
reallocatingPtNode->getCodePointCount() - overlappingCodePointCount,
reallocatingPtNode->getProbability(), &writingPos)) {
return false;
}
if (addsExtraChild) {
- if (!writePtNodeToBuffer(firstPartOfReallocatedPtNodePos,
+ if (!writePtNodeToBuffer(mBuffer, firstPartOfReallocatedPtNodePos,
newNodeCodePoints + overlappingCodePointCount,
newNodeCodePointCount - overlappingCodePointCount, probabilityOfNewPtNode,
&writingPos)) {
@@ -452,4 +441,64 @@
return true;
}
+// TODO: Create a struct which contains header, body and etc... and use here as an argument.
+void DynamicPatriciaTrieWritingHelper::flushAllToFile(const char *const fileName,
+ BufferWithExtendableBuffer *const dictHeader,
+ BufferWithExtendableBuffer *const dictBody) const {
+ const int tmpFileNameBufSize = strlen(fileName)
+ + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */;
+ // Name of a temporary file used for writing that is a connected string of original name and
+ // TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE.
+ char tmpFileName[tmpFileNameBufSize];
+ snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName,
+ TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
+ FILE *const file = fopen(tmpFileName, "wb");
+ if (!file) {
+ AKLOGI("Dictionary file %s cannnot be opened.", tmpFileName);
+ ASSERT(false);
+ return;
+ }
+ // Write the dictionary header.
+ if (!writeBufferToFilePointer(file, dictHeader)) {
+ remove(tmpFileName);
+ AKLOGI("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition());
+ ASSERT(false);
+ return;
+ }
+ // Write the dictionary body.
+ if (!writeBufferToFilePointer(file, dictBody)) {
+ remove(tmpFileName);
+ AKLOGI("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition());
+ ASSERT(false);
+ return;
+ }
+ fclose(file);
+ rename(tmpFileName, fileName);
+}
+
+// This closes file pointer when an error is caused and returns whether the writing was succeeded
+// or not.
+bool DynamicPatriciaTrieWritingHelper::writeBufferToFilePointer(FILE *const file,
+ const BufferWithExtendableBuffer *const buffer) const {
+ const int originalBufSize = buffer->getOriginalBufferSize();
+ if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */),
+ originalBufSize, 1, file) < 1) {
+ fclose(file);
+ return false;
+ }
+ const int additionalBufSize = buffer->getTailPosition() - buffer->getOriginalBufferSize();
+ if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */),
+ additionalBufSize, 1, file) < 1) {
+ fclose(file);
+ return false;
+ }
+ return true;
+}
+
+bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
+ BufferWithExtendableBuffer *const bufferToWrite) {
+ // TODO: Implement.
+ return false;
+}
+
} // namespace latinime
diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
index 219ea98..8f78d84 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h
@@ -17,6 +17,7 @@
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H
+#include <cstdio>
#include <stdint.h>
#include "defines.h"
@@ -51,7 +52,8 @@
void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy);
- void writeToDictFileWithGC(const char *const fileName, const HeaderPolicy *const headerPolicy);
+ void writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const fileName,
+ const HeaderPolicy *const headerPolicy);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper);
@@ -66,15 +68,17 @@
bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate,
const int movedPos, const int bigramLinkedNodePos);
- bool writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, const bool isNotAWord,
+ bool writePtNodeWithFullInfoToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
+ const bool isBlacklisted, const bool isNotAWord,
const int parentPos, const int *const codePoints, const int codePointCount,
const int probability, const int childrenPos, const int originalBigramListPos,
const int originalShortcutListPos, int *const writingPos);
- bool writePtNodeToBuffer(const int parentPos, const int *const codePoints,
- const int codePointCount, const int probability, int *const writingPos);
+ bool writePtNodeToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
+ const int parentPos, const int *const codePoints, const int codePointCount,
+ const int probability, int *const writingPos);
- bool writePtNodeToBufferByCopyingPtNodeInfo(
+ bool writePtNodeToBufferByCopyingPtNodeInfo(BufferWithExtendableBuffer *const bufferToWrite,
const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos,
const int *const codePoints, const int codePointCount, const int probability,
int *const writingPos);
@@ -97,6 +101,15 @@
const int *const reallocatingPtNodeCodePoints, const int overlappingCodePointCount,
const int probabilityOfNewPtNode, const int *const newNodeCodePoints,
const int newNodeCodePointCount);
+
+ void flushAllToFile(const char *const fileName,
+ BufferWithExtendableBuffer *const dictHeader,
+ BufferWithExtendableBuffer *const dictBody) const;
+
+ bool writeBufferToFilePointer(FILE *const file,
+ const BufferWithExtendableBuffer *const buffer) const;
+
+ bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite);
};
} // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */
diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h
index 1803c09..bd3211f 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h
@@ -31,7 +31,7 @@
*/
class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
public:
- explicit DynamicShortcutListPolicy(BufferWithExtendableBuffer *const buffer)
+ explicit DynamicShortcutListPolicy(const BufferWithExtendableBuffer *const buffer)
: mBuffer(buffer) {}
~DynamicShortcutListPolicy() {}
@@ -82,18 +82,20 @@
}
}
- // Copy shortcuts from the shortcut list that starts at fromPos to toPos and advance these
- // positions after the shortcut lists. This returns whether the copy was succeeded or not.
- bool copyAllShortcutsAndReturnIfSucceededOrNot(int *const fromPos, int *const toPos) {
+ // Copy shortcuts from the shortcut list that starts at fromPos in mBuffer to toPos in
+ // bufferToWrite and advance these positions after the shortcut lists. This returns whether
+ // the copy was succeeded or not.
+ bool copyAllShortcutsAndReturnIfSucceededOrNot(BufferWithExtendableBuffer *const bufferToWrite,
+ int *const fromPos, int *const toPos) const {
const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos);
- const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);
if (usesAdditionalBuffer) {
*fromPos -= mBuffer->getOriginalBufferSize();
}
const int shortcutListSize = ShortcutListReadingUtils
- ::getShortcutListSizeAndForwardPointer(buffer, fromPos);
+ ::getShortcutListSizeAndForwardPointer(mBuffer->getBuffer(usesAdditionalBuffer),
+ fromPos);
// Copy shortcut list size.
- if (!mBuffer->writeUintAndAdvancePosition(
+ if (!bufferToWrite->writeUintAndAdvancePosition(
shortcutListSize + ShortcutListReadingUtils::getShortcutListSizeFieldSize(),
ShortcutListReadingUtils::getShortcutListSizeFieldSize(), toPos)) {
return false;
@@ -102,7 +104,7 @@
for (int i = 0; i < shortcutListSize; ++i) {
const uint8_t data = ByteArrayUtils::readUint8AndAdvancePosition(
mBuffer->getBuffer(usesAdditionalBuffer), fromPos);
- if (!mBuffer->writeUintAndAdvancePosition(data, 1 /* size */, toPos)) {
+ if (!bufferToWrite->writeUintAndAdvancePosition(data, 1 /* size */, toPos)) {
return false;
}
}
@@ -115,7 +117,7 @@
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicShortcutListPolicy);
- BufferWithExtendableBuffer *const mBuffer;
+ const BufferWithExtendableBuffer *const mBuffer;
};
} // namespace latinime
#endif // LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H