Merge "Fix wrong padding of some background drawables"
diff --git a/java/src/com/android/inputmethod/latin/AssetFileAddress.java b/java/src/com/android/inputmethod/latin/AssetFileAddress.java
index 8751925..855a8d8 100644
--- a/java/src/com/android/inputmethod/latin/AssetFileAddress.java
+++ b/java/src/com/android/inputmethod/latin/AssetFileAddress.java
@@ -52,4 +52,12 @@
if (!f.isFile()) return null;
return new AssetFileAddress(filename, offset, length);
}
+
+ public boolean pointsToPhysicalFile() {
+ return 0 == mOffset;
+ }
+
+ public void deleteUnderlyingFile() {
+ new File(mFilename).delete();
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index 0e03b52..aa530ff 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -57,7 +57,7 @@
@UsedForTesting
public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT";
- public static final int NOT_A_VALID_TIME_STAMP = -1;
+ public static final int NOT_A_VALID_TIMESTAMP = -1;
private long mNativeDict;
private final Locale mLocale;
@@ -121,6 +121,7 @@
String[] attributeKeyStringArray, String[] attributeValueStringArray);
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
boolean isUpdatable);
+ private static native boolean hasValidContentsNative(long dict);
private static native void flushNative(long dict, String filePath);
private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC);
private static native void flushWithGCNative(long dict, String filePath);
@@ -242,6 +243,10 @@
return mNativeDict != 0;
}
+ public boolean hasValidContents() {
+ return hasValidContentsNative(mNativeDict);
+ }
+
public int getFormatVersion() {
return getFormatVersionNative(mNativeDict);
}
@@ -285,18 +290,6 @@
return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
}
- // Add a unigram entry to binary dictionary in native code.
- public void addUnigramWord(final String word, final int probability) {
- if (TextUtils.isEmpty(word)) {
- return;
- }
- final int[] codePoints = StringUtils.toCodePointArray(word);
- final int[] shortcutTargetCodePoints = new int[0];
- addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints,
- NOT_A_PROBABILITY, false /* isNotAWord */, false /* isBlacklisted */,
- NOT_A_VALID_TIME_STAMP);
- }
-
// Add a unigram entry to binary dictionary with unigram attributes in native code.
public void addUnigramWord(final String word, final int probability,
final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord,
@@ -311,17 +304,6 @@
shortcutProbability, isNotAWord, isBlacklisted, timestamp);
}
- // Add a bigram entry to binary dictionary in native code.
- public void addBigramWords(final String word0, final String word1, final int probability) {
- if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
- return;
- }
- final int[] codePoints0 = StringUtils.toCodePointArray(word0);
- final int[] codePoints1 = StringUtils.toCodePointArray(word1);
- addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability,
- NOT_A_VALID_TIME_STAMP);
- }
-
// Add a bigram entry to binary dictionary with timestamp in native code.
public void addBigramWords(final String word0, final String word1, final int probability,
final int timestamp) {
@@ -403,7 +385,7 @@
private void reopen() {
close();
final File dictFile = new File(mDictFilePath);
- // WARNING: Because we pass 0 as the offstet and file.length() as the length, this can
+ // WARNING: Because we pass 0 as the offset and file.length() as the length, this can
// only be called for actual files. Right now it's only called by the flush() family of
// functions, which require an updatable dictionary, so it's okay. But beware.
loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */,
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java
index ad94a04..b4382bc 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java
@@ -98,7 +98,7 @@
* This creates a URI builder able to build a URI pointing to the dictionary
* pack content provider for a specific dictionary id.
*/
- private static Uri.Builder getProviderUriBuilder(final String path) {
+ public static Uri.Builder getProviderUriBuilder(final String path) {
return new Uri.Builder().scheme(ContentResolver.SCHEME_CONTENT)
.authority(DictionaryPackConstants.AUTHORITY).appendPath(path);
}
@@ -339,15 +339,25 @@
Log.e(TAG, "Could not copy a word list. Will not be able to use it.");
// If we can't copy it we should warn the dictionary provider so that it can mark it
// as invalid.
- wordListUriBuilder.appendQueryParameter(QUERY_PARAMETER_DELETE_RESULT,
- QUERY_PARAMETER_FAILURE);
+ reportBrokenFileToDictionaryProvider(providerClient, clientId, wordlistId);
+ }
+
+ public static boolean reportBrokenFileToDictionaryProvider(
+ final ContentProviderClient providerClient, final String clientId,
+ final String wordlistId) {
try {
+ final Uri.Builder wordListUriBuilder = getContentUriBuilderForType(clientId,
+ providerClient, QUERY_PATH_DATAFILE, wordlistId /* extraPath */);
+ wordListUriBuilder.appendQueryParameter(QUERY_PARAMETER_DELETE_RESULT,
+ QUERY_PARAMETER_FAILURE);
if (0 >= providerClient.delete(wordListUriBuilder.build(), null, null)) {
- Log.e(TAG, "In addition, we were unable to delete it.");
+ Log.e(TAG, "Unable to delete a word list.");
}
} catch (RemoteException e) {
- Log.e(TAG, "In addition, communication with the dictionary provider was cut", e);
+ Log.e(TAG, "Communication with the dictionary provider was cut", e);
+ return false;
}
+ return true;
}
// Ideally the two following methods should be merged, but AssetFileDescriptor does not
diff --git a/java/src/com/android/inputmethod/latin/DictionaryFactory.java b/java/src/com/android/inputmethod/latin/DictionaryFactory.java
index 828e54f..bcb38da 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryFactory.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryFactory.java
@@ -16,6 +16,7 @@
package com.android.inputmethod.latin;
+import android.content.ContentProviderClient;
import android.content.Context;
import android.content.res.AssetFileDescriptor;
import android.content.res.Resources;
@@ -62,8 +63,12 @@
final ReadOnlyBinaryDictionary readOnlyBinaryDictionary =
new ReadOnlyBinaryDictionary(f.mFilename, f.mOffset, f.mLength,
useFullEditDistance, locale, Dictionary.TYPE_MAIN);
- if (readOnlyBinaryDictionary.isValidDictionary()) {
+ if (readOnlyBinaryDictionary.hasValidContents()) {
dictList.add(readOnlyBinaryDictionary);
+ } else {
+ readOnlyBinaryDictionary.close();
+ // Prevent this dictionary to do any further harm.
+ killDictionary(context, f);
}
}
}
@@ -75,6 +80,51 @@
}
/**
+ * Kills a dictionary so that it is never used again, if possible.
+ * @param context The context to contact the dictionary provider, if possible.
+ * @param f A file address to the dictionary to kill.
+ */
+ private static void killDictionary(final Context context, final AssetFileAddress f) {
+ if (f.pointsToPhysicalFile()) {
+ f.deleteUnderlyingFile();
+ // Warn the dictionary provider if the dictionary came from there.
+ final ContentProviderClient providerClient;
+ try {
+ providerClient = context.getContentResolver().acquireContentProviderClient(
+ BinaryDictionaryFileDumper.getProviderUriBuilder("").build());
+ } catch (final SecurityException e) {
+ Log.e(TAG, "No permission to communicate with the dictionary provider", e);
+ return;
+ }
+ if (null == providerClient) {
+ Log.e(TAG, "Can't establish communication with the dictionary provider");
+ return;
+ }
+ final String wordlistId =
+ DictionaryInfoUtils.getWordListIdFromFileName(new File(f.mFilename).getName());
+ if (null != wordlistId) {
+ // TODO: this is a reasonable last resort, but it is suboptimal.
+ // The following will remove the entry for this dictionary with the dictionary
+ // provider. When the metadata is downloaded again, we will try downloading it
+ // again.
+ // However, in the practice that will mean the user will find themselves without
+ // the new dictionary. That's fine for languages where it's included in the APK,
+ // but for other languages it will leave the user without a dictionary at all until
+ // the next update, which may be a few days away.
+ // Ideally, we would trigger a new download right away, and use increasing retry
+ // delays for this particular id/version combination.
+ // Then again, this is expected to only ever happen in case of human mistake. If
+ // the wrong file is on the server, the following is still doing the right thing.
+ // If it's a file left over from the last version however, it's not great.
+ BinaryDictionaryFileDumper.reportBrokenFileToDictionaryProvider(
+ providerClient,
+ context.getString(R.string.dictionary_pack_client_id),
+ wordlistId);
+ }
+ }
+ }
+
+ /**
* Initializes a main dictionary collection from a dictionary pack, with default flags.
*
* This searches for a content provider providing a dictionary pack for the specified
diff --git a/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java
index 68505ce..c8e4014 100644
--- a/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java
@@ -44,6 +44,15 @@
locale, dictType, false /* isUpdatable */);
}
+ public boolean hasValidContents() {
+ mLock.readLock().lock();
+ try {
+ return mBinaryDictionary.hasValidContents();
+ } finally {
+ mLock.readLock().unlock();
+ }
+ }
+
public boolean isValidDictionary() {
return mBinaryDictionary.isValidDictionary();
}
diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
index 555c71b..93ccc62 100644
--- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
+++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
@@ -368,6 +368,7 @@
// and latinime::HeaderReadWriteUtils.
public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE";
public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE";
+ public static final String HAS_HISTORICAL_INFO_ATTRIBUTE = "HAS_HISTORICAL_INFO";
public static final String ATTRIBUTE_VALUE_TRUE = "1";
public static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
index f4b7d6e..529ddc2 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
@@ -94,6 +94,8 @@
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+ attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE,
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName);
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString());
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE,
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 57c170f..71567e2 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -135,6 +135,14 @@
delete dictionary;
}
+static bool latinime_BinaryDictionary_hasValidContents(JNIEnv *env, jclass clazz,
+ jlong dict) {
+ Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+ if (!dictionary) return false;
+ // TODO: check format version
+ return true;
+}
+
static int latinime_BinaryDictionary_getFormatVersion(JNIEnv *env, jclass clazz, jlong dict) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return 0;
@@ -438,6 +446,11 @@
reinterpret_cast<void *>(latinime_BinaryDictionary_close)
},
{
+ const_cast<char *>("hasValidContentsNative"),
+ const_cast<char *>("(J)Z"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_hasValidContents)
+ },
+ {
const_cast<char *>("getFormatVersionNative"),
const_cast<char *>("(J)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_getFormatVersion)
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
index 9726fb8..4c4e43b 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
@@ -68,6 +68,13 @@
mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
&mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {}
+ // Temporary dummy header.
+ HeaderPolicy()
+ : mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0),
+ mAttributeMap(), mMultiWordCostMultiplier(0.0f), mIsDecayingDict(false),
+ mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
+ mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
+
~HeaderPolicy() {}
virtual int getFormatVersionNumber() const {
@@ -139,7 +146,7 @@
const int unigramCount, const int bigramCount, const int extendedRegionSize) const;
private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy);
+ DISALLOW_COPY_AND_ASSIGN(HeaderPolicy);
static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
static const char *const IS_DECAYING_DICT_KEY;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
index 07b8f18..40f6469 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
@@ -18,7 +18,7 @@
#define LATINIME_VER4_DICT_BUFFER_H
#include "defines.h"
-#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h"
+#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h"
@@ -39,8 +39,9 @@
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable));
}
- static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers() {
- return Ver4DictBuffersPtr(new Ver4DictBuffers());
+ static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers(
+ const HeaderPolicy *const headerPolicy) {
+ return Ver4DictBuffersPtr(new Ver4DictBuffers(headerPolicy));
}
AK_FORCE_INLINE bool isValid() const {
@@ -57,6 +58,10 @@
|| mShortcutDictContent.isNearSizeLimit();
}
+ AK_FORCE_INLINE const HeaderPolicy *getHeaderPolicy() const {
+ return &mHeaderPolicy;
+ }
+
AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableHeaderBuffer() {
return &mExpandableHeaderBuffer;
}
@@ -118,30 +123,31 @@
AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable)
: mDictBuffer(dictBuffer),
- mHeaderSize(HeaderReadWriteUtils::getHeaderSize(mDictBuffer.get()->getBuffer())),
- mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderSize,
+ mHeaderPolicy(mDictBuffer.get()->getBuffer(), FormatUtils::VERSION_4),
+ mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
- mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderSize,
- dictBuffer.get()->getBufferSize() - mHeaderSize,
+ mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderPolicy.getSize(),
+ dictBuffer.get()->getBufferSize() - mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
// TODO: Quit using header size.
- mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderSize),
- mProbabilityDictContent(dictDirPath, false /* hasHistoricalInfo */, isUpdatable),
- mBigramDictContent(dictDirPath, false /* hasHistoricalInfo */, isUpdatable),
+ mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderPolicy.getSize()),
+ mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistricalInfoOfWords(),
+ isUpdatable),
+ mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistricalInfoOfWords(), isUpdatable),
mShortcutDictContent(dictDirPath, isUpdatable),
mIsUpdatable(isUpdatable) {}
- AK_FORCE_INLINE Ver4DictBuffers()
- : mDictBuffer(0), mHeaderSize(0),
+ AK_FORCE_INLINE Ver4DictBuffers(const HeaderPolicy *const headerPolicy)
+ : mDictBuffer(0), mHeaderPolicy(),
mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mTerminalPositionLookupTable(),
- mProbabilityDictContent(false /* hasHistoricalInfo */),
- mBigramDictContent(false /* hasHistoricalInfo */), mShortcutDictContent(),
+ mProbabilityDictContent(headerPolicy->hasHistricalInfoOfWords()),
+ mBigramDictContent(headerPolicy->hasHistricalInfoOfWords()), mShortcutDictContent(),
mIsUpdatable(true) {}
const MmappedBuffer::MmappedBufferPtr mDictBuffer;
- const int mHeaderSize;
+ const HeaderPolicy mHeaderPolicy;
BufferWithExtendableBuffer mExpandableHeaderBuffer;
BufferWithExtendableBuffer mExpandableTrieBuffer;
TerminalPositionLookupTable mTerminalPositionLookupTable;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
index 719f358..f9baeee 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
@@ -50,7 +50,7 @@
break;
}
bool isTerminal = ptNodeParams.isTerminal() && !ptNodeParams.isDeleted();
- if (isTerminal && mHeaderPolicy.isDecayingDict()) {
+ if (isTerminal && mHeaderPolicy->isDecayingDict()) {
// A DecayingDict may have a terminal PtNode that has a terminal DicNode whose
// probability is NOT_A_PROBABILITY. In such case, we don't want to treat it as a
// valid terminal DicNode.
@@ -85,7 +85,7 @@
int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,
const int bigramProbability) const {
- if (mHeaderPolicy.isDecayingDict()) {
+ if (mHeaderPolicy->isDecayingDict()) {
// Both probabilities are encoded. Decode them and get probability.
return ForgettingCurveUtils::getProbability(unigramProbability, bigramProbability);
} else {
@@ -229,7 +229,7 @@
AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath);
return;
}
- mWritingHelper.writeToDictFile(filePath, &mHeaderPolicy, mUnigramCount, mBigramCount);
+ mWritingHelper.writeToDictFile(filePath, mUnigramCount, mBigramCount);
}
void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
@@ -237,10 +237,10 @@
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
return;
}
- const bool needsToDecay = mHeaderPolicy.isDecayingDict()
+ const bool needsToDecay = mHeaderPolicy->isDecayingDict()
&& (mNeedsToDecayForTesting || ForgettingCurveUtils::needsToDecay(
- false /* mindsBlockByDecay */, mUnigramCount, mBigramCount, &mHeaderPolicy));
- mWritingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy, needsToDecay);
+ false /* mindsBlockByDecay */, mUnigramCount, mBigramCount, mHeaderPolicy));
+ mWritingHelper.writeToDictFileWithGC(getRootPosition(), filePath, needsToDecay);
mNeedsToDecayForTesting = false;
}
@@ -252,7 +252,7 @@
if (mBuffers.get()->isNearSizeLimit()) {
// Additional buffer size is near the limit.
return true;
- } else if (mHeaderPolicy.getExtendedRegionSize() + mDictBuffer->getUsedAdditionalBufferSize()
+ } else if (mHeaderPolicy->getExtendedRegionSize() + mDictBuffer->getUsedAdditionalBufferSize()
> Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE) {
// Total extended region size of the trie exceeds the limit.
return true;
@@ -260,9 +260,9 @@
&& mDictBuffer->getUsedAdditionalBufferSize() > 0) {
// Needs to reduce dictionary size.
return true;
- } else if (mHeaderPolicy.isDecayingDict()) {
+ } else if (mHeaderPolicy->isDecayingDict()) {
return mNeedsToDecayForTesting || ForgettingCurveUtils::needsToDecay(
- mindsBlockByGC, mUnigramCount, mBigramCount, &mHeaderPolicy);
+ mindsBlockByGC, mUnigramCount, mBigramCount, mHeaderPolicy);
}
return false;
}
@@ -276,11 +276,11 @@
snprintf(outResult, maxResultLength, "%d", mBigramCount);
} else if (strncmp(query, MAX_UNIGRAM_COUNT_QUERY, compareLength) == 0) {
snprintf(outResult, maxResultLength, "%d",
- mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_UNIGRAM_COUNT :
+ mHeaderPolicy->isDecayingDict() ? ForgettingCurveUtils::MAX_UNIGRAM_COUNT :
static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE));
} else if (strncmp(query, MAX_BIGRAM_COUNT_QUERY, compareLength) == 0) {
snprintf(outResult, maxResultLength, "%d",
- mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT :
+ mHeaderPolicy->isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT :
static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE));
} else if (strncmp(query, SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY, compareLength) == 0) {
mNeedsToDecayForTesting = true;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
index 360b6b8..e385107 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
@@ -38,22 +38,20 @@
class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
Ver4PatriciaTriePolicy(const Ver4DictBuffers::Ver4DictBuffersPtr &buffers)
- : mBuffers(buffers),
- mHeaderPolicy(mBuffers.get()->getWritableHeaderBuffer()->getBuffer(
- false /* usesAdditionalBuffer*/), FormatUtils::VERSION_4),
+ : mBuffers(buffers), mHeaderPolicy(mBuffers.get()->getHeaderPolicy()),
mDictBuffer(mBuffers.get()->getWritableTrieBuffer()),
mBigramPolicy(mBuffers.get()->getUpdatableBigramDictContent(),
- mBuffers.get()->getTerminalPositionLookupTable(), &mHeaderPolicy,
- mHeaderPolicy.isDecayingDict()),
+ mBuffers.get()->getTerminalPositionLookupTable(), mHeaderPolicy,
+ mHeaderPolicy->isDecayingDict()),
mShortcutPolicy(mBuffers.get()->getShortcutDictContent(),
mBuffers.get()->getTerminalPositionLookupTable()),
mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent()),
mNodeWriter(mDictBuffer, mBuffers.get(), &mNodeReader, &mBigramPolicy,
- &mShortcutPolicy, mHeaderPolicy.isDecayingDict()),
+ &mShortcutPolicy, mHeaderPolicy->isDecayingDict()),
mUpdatingHelper(mDictBuffer, &mNodeReader, &mNodeWriter),
mWritingHelper(mBuffers.get()),
- mUnigramCount(mHeaderPolicy.getUnigramCount()),
- mBigramCount(mHeaderPolicy.getBigramCount()), mNeedsToDecayForTesting(false) {};
+ mUnigramCount(mHeaderPolicy->getUnigramCount()),
+ mBigramCount(mHeaderPolicy->getBigramCount()), mNeedsToDecayForTesting(false) {};
AK_FORCE_INLINE int getRootPosition() const {
return 0;
@@ -78,7 +76,7 @@
int getBigramsPositionOfPtNode(const int ptNodePos) const;
const DictionaryHeaderStructurePolicy *getHeaderStructurePolicy() const {
- return &mHeaderPolicy;
+ return mHeaderPolicy;
}
const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const {
@@ -121,7 +119,7 @@
static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS;
Ver4DictBuffers::Ver4DictBuffersPtr mBuffers;
- const HeaderPolicy mHeaderPolicy;
+ const HeaderPolicy *const mHeaderPolicy;
BufferWithExtendableBuffer *const mDictBuffer;
Ver4BigramListPolicy mBigramPolicy;
Ver4ShortcutListPolicy mShortcutPolicy;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
index e8368af..dd62b9c 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
@@ -32,8 +32,8 @@
namespace latinime {
void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePath,
- const HeaderPolicy *const headerPolicy, const int unigramCount,
- const int bigramCount) const {
+ const int unigramCount, const int bigramCount) const {
+ const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy();
const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */;
char dirPath[dirPathBufSize];
FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath);
@@ -53,9 +53,10 @@
}
void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
- const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
- const bool needsToDecay) {
- Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(Ver4DictBuffers::createVer4DictBuffers());
+ const char *const trieFilePath, const bool needsToDecay) {
+ const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy();
+ Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
+ Ver4DictBuffers::createVer4DictBuffers(headerPolicy));
int unigramCount = 0;
int bigramCount = 0;
if (needsToDecay) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
index ea2fd53..9344bde 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
@@ -32,12 +32,11 @@
Ver4PatriciaTrieWritingHelper(Ver4DictBuffers *const buffers)
: mBuffers(buffers) {}
- void writeToDictFile(const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
- const int unigramCount, const int bigramCount) const;
+ void writeToDictFile(const char *const trieFilePath, const int unigramCount,
+ const int bigramCount) const;
void writeToDictFileWithGC(const int rootPtNodeArrayPos,
- const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
- const bool needsToDecay);
+ const char *const trieFilePath, const bool needsToDecay);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
index ff762d7..d959b26 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
@@ -43,8 +43,9 @@
/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
- Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = Ver4DictBuffers::createVer4DictBuffers();
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap);
+ Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
+ Ver4DictBuffers::createVer4DictBuffers(&headerPolicy);
headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(),
true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */,
0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
index 5b888ab..95504ef 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
@@ -53,6 +53,20 @@
super.tearDown();
}
+ private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
+ final int probability) {
+ binaryDictionary.addUnigramWord(word, probability, "" /* shortcutTarget */,
+ BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
+ false /* isNotAWord */, false /* isBlacklisted */,
+ BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */);
+ }
+
+ private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0,
+ final String word1, final int probability) {
+ binaryDictionary.addBigramWords(word0, word1, probability,
+ BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */);
+ }
+
private void forcePassingShortTime(final BinaryDictionary binaryDictionary) {
// Entries having low probability would be suppressed once in 3 GCs.
final int count = 3;
@@ -115,35 +129,35 @@
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
assertFalse(binaryDictionary.isValidWord("a"));
- binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
assertFalse(binaryDictionary.isValidWord("a"));
- binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
assertFalse(binaryDictionary.isValidWord("a"));
- binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
assertTrue(binaryDictionary.isValidWord("a"));
- binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
assertTrue(binaryDictionary.isValidWord("b"));
- binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY);
+ addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
assertFalse(binaryDictionary.isValidBigram("a", "b"));
- binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY);
+ addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
assertFalse(binaryDictionary.isValidBigram("a", "b"));
- binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY);
+ addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
assertFalse(binaryDictionary.isValidBigram("a", "b"));
- binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY);
+ addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
assertTrue(binaryDictionary.isValidBigram("a", "b"));
- binaryDictionary.addUnigramWord("c", DUMMY_PROBABILITY);
- binaryDictionary.addBigramWords("a", "c", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY);
+ addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY);
assertTrue(binaryDictionary.isValidBigram("a", "c"));
// Add bigrams of not valid unigrams.
- binaryDictionary.addBigramWords("x", "y", Dictionary.NOT_A_PROBABILITY);
+ addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY);
assertFalse(binaryDictionary.isValidBigram("x", "y"));
- binaryDictionary.addBigramWords("x", "y", DUMMY_PROBABILITY);
+ addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY);
assertFalse(binaryDictionary.isValidBigram("x", "y"));
binaryDictionary.close();
@@ -165,39 +179,39 @@
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
assertTrue(binaryDictionary.isValidWord("a"));
forcePassingShortTime(binaryDictionary);
assertFalse(binaryDictionary.isValidWord("a"));
- binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
forcePassingShortTime(binaryDictionary);
assertTrue(binaryDictionary.isValidWord("a"));
forcePassingLongTime(binaryDictionary);
assertFalse(binaryDictionary.isValidWord("a"));
- binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
- binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
+ addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
assertTrue(binaryDictionary.isValidBigram("a", "b"));
forcePassingShortTime(binaryDictionary);
assertFalse(binaryDictionary.isValidBigram("a", "b"));
- binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
- binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
- binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
- binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY);
- binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
+ addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
+ addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
+ addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
+ addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
assertTrue(binaryDictionary.isValidBigram("a", "b"));
forcePassingShortTime(binaryDictionary);
assertTrue(binaryDictionary.isValidBigram("a", "b"));
@@ -241,7 +255,7 @@
binaryDictionary.getPropertyForTests(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY));
for (int i = 0; i < unigramTypedCount; i++) {
final String word = words.get(random.nextInt(words.size()));
- binaryDictionary.addUnigramWord(word, DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int unigramCountBeforeGC =
@@ -310,9 +324,9 @@
binaryDictionary.getPropertyForTests(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY));
for (int i = 0; i < bigramTypedCount; ++i) {
final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size()));
- binaryDictionary.addUnigramWord(bigram.first, DUMMY_PROBABILITY);
- binaryDictionary.addUnigramWord(bigram.second, DUMMY_PROBABILITY);
- binaryDictionary.addBigramWords(bigram.first, bigram.second, DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY);
+ addUnigramWord(binaryDictionary, bigram.second, DUMMY_PROBABILITY);
+ addBigramWords(binaryDictionary, bigram.first, bigram.second, DUMMY_PROBABILITY);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int bigramCountBeforeGC =
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
index 8565db9..9cccded 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
@@ -105,6 +105,20 @@
binaryDictionary.close();
}
+ private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
+ final int probability) {
+ binaryDictionary.addUnigramWord(word, probability, "" /* shortcutTarget */,
+ BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
+ false /* isNotAWord */, false /* isBlacklisted */,
+ BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */);
+ }
+
+ private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0,
+ final String word1, final int probability) {
+ binaryDictionary.addBigramWords(word0, word1, probability,
+ BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */);
+ }
+
public void testAddUnigramWord() {
testAddUnigramWord(4 /* formatVersion */);
}
@@ -121,21 +135,21 @@
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int probability = 100;
- binaryDictionary.addUnigramWord("aaa", probability);
+ addUnigramWord(binaryDictionary, "aaa", probability);
// Reallocate and create.
- binaryDictionary.addUnigramWord("aab", probability);
+ addUnigramWord(binaryDictionary, "aab", probability);
// Insert into children.
- binaryDictionary.addUnigramWord("aac", probability);
+ addUnigramWord(binaryDictionary, "aac", probability);
// Make terminal.
- binaryDictionary.addUnigramWord("aa", probability);
+ addUnigramWord(binaryDictionary, "aa", probability);
// Create children.
- binaryDictionary.addUnigramWord("aaaa", probability);
+ addUnigramWord(binaryDictionary, "aaaa", probability);
// Reallocate and make termianl.
- binaryDictionary.addUnigramWord("a", probability);
+ addUnigramWord(binaryDictionary, "a", probability);
final int updatedProbability = 200;
// Update.
- binaryDictionary.addUnigramWord("aaa", updatedProbability);
+ addUnigramWord(binaryDictionary, "aaa", updatedProbability);
assertEquals(probability, binaryDictionary.getFrequency("aab"));
assertEquals(probability, binaryDictionary.getFrequency("aac"));
@@ -175,7 +189,7 @@
probabilityMap.put(word, random.nextInt(0xFF));
}
for (String word : probabilityMap.keySet()) {
- binaryDictionary.addUnigramWord(word, probabilityMap.get(word));
+ addUnigramWord(binaryDictionary, word, probabilityMap.get(word));
}
for (String word : probabilityMap.keySet()) {
assertEquals(word, (int)probabilityMap.get(word), binaryDictionary.getFrequency(word));
@@ -201,13 +215,13 @@
final int unigramProbability = 100;
final int bigramProbability = 10;
final int updatedBigramProbability = 15;
- binaryDictionary.addUnigramWord("aaa", unigramProbability);
- binaryDictionary.addUnigramWord("abb", unigramProbability);
- binaryDictionary.addUnigramWord("bcc", unigramProbability);
- binaryDictionary.addBigramWords("aaa", "abb", bigramProbability);
- binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability);
- binaryDictionary.addBigramWords("abb", "aaa", bigramProbability);
- binaryDictionary.addBigramWords("abb", "bcc", bigramProbability);
+ addUnigramWord(binaryDictionary, "aaa", unigramProbability);
+ addUnigramWord(binaryDictionary, "abb", unigramProbability);
+ addUnigramWord(binaryDictionary, "bcc", unigramProbability);
+ addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability);
+ addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability);
+ addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
+ addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
final int probability = binaryDictionary.calculateProbability(unigramProbability,
bigramProbability);
@@ -220,7 +234,7 @@
assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa"));
assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc"));
- binaryDictionary.addBigramWords("aaa", "abb", updatedBigramProbability);
+ addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability);
final int updatedProbability = binaryDictionary.calculateProbability(unigramProbability,
updatedBigramProbability);
assertEquals(updatedProbability, binaryDictionary.getBigramProbability("aaa", "abb"));
@@ -236,16 +250,16 @@
binaryDictionary.getBigramProbability("aaa", "aaa"));
// Testing bigram link.
- binaryDictionary.addUnigramWord("abcde", unigramProbability);
- binaryDictionary.addUnigramWord("fghij", unigramProbability);
- binaryDictionary.addBigramWords("abcde", "fghij", bigramProbability);
- binaryDictionary.addUnigramWord("fgh", unigramProbability);
- binaryDictionary.addUnigramWord("abc", unigramProbability);
- binaryDictionary.addUnigramWord("f", unigramProbability);
+ addUnigramWord(binaryDictionary, "abcde", unigramProbability);
+ addUnigramWord(binaryDictionary, "fghij", unigramProbability);
+ addBigramWords(binaryDictionary, "abcde", "fghij", bigramProbability);
+ addUnigramWord(binaryDictionary, "fgh", unigramProbability);
+ addUnigramWord(binaryDictionary, "abc", unigramProbability);
+ addUnigramWord(binaryDictionary, "f", unigramProbability);
assertEquals(probability, binaryDictionary.getBigramProbability("abcde", "fghij"));
assertEquals(Dictionary.NOT_A_PROBABILITY,
binaryDictionary.getBigramProbability("abcde", "fgh"));
- binaryDictionary.addBigramWords("abcde", "fghij", updatedBigramProbability);
+ addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability);
assertEquals(updatedProbability, binaryDictionary.getBigramProbability("abcde", "fghij"));
dictFile.delete();
@@ -284,7 +298,7 @@
words.add(word);
final int unigramProbability = random.nextInt(0xFF);
unigramProbabilities.put(word, unigramProbability);
- binaryDictionary.addUnigramWord(word, unigramProbability);
+ addUnigramWord(binaryDictionary, word, unigramProbability);
}
for (int i = 0; i < bigramCount; i++) {
@@ -297,7 +311,7 @@
bigramWords.add(bigram);
final int bigramProbability = random.nextInt(0xF);
bigramProbabilities.put(bigram, bigramProbability);
- binaryDictionary.addBigramWords(word0, word1, bigramProbability);
+ addBigramWords(binaryDictionary, word0, word1, bigramProbability);
}
for (final Pair<String, String> bigram : bigramWords) {
@@ -328,13 +342,13 @@
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int unigramProbability = 100;
final int bigramProbability = 10;
- binaryDictionary.addUnigramWord("aaa", unigramProbability);
- binaryDictionary.addUnigramWord("abb", unigramProbability);
- binaryDictionary.addUnigramWord("bcc", unigramProbability);
- binaryDictionary.addBigramWords("aaa", "abb", bigramProbability);
- binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability);
- binaryDictionary.addBigramWords("abb", "aaa", bigramProbability);
- binaryDictionary.addBigramWords("abb", "bcc", bigramProbability);
+ addUnigramWord(binaryDictionary, "aaa", unigramProbability);
+ addUnigramWord(binaryDictionary, "abb", unigramProbability);
+ addUnigramWord(binaryDictionary, "bcc", unigramProbability);
+ addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability);
+ addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability);
+ addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
+ addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc"));
@@ -343,7 +357,7 @@
binaryDictionary.removeBigramWords("aaa", "abb");
assertEquals(false, binaryDictionary.isValidBigram("aaa", "abb"));
- binaryDictionary.addBigramWords("aaa", "abb", bigramProbability);
+ addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability);
assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
@@ -378,8 +392,8 @@
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int probability = 100;
- binaryDictionary.addUnigramWord("aaa", probability);
- binaryDictionary.addUnigramWord("abcd", probability);
+ addUnigramWord(binaryDictionary, "aaa", probability);
+ addUnigramWord(binaryDictionary, "abcd", probability);
// Close without flushing.
binaryDictionary.close();
@@ -390,8 +404,8 @@
assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("aaa"));
assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("abcd"));
- binaryDictionary.addUnigramWord("aaa", probability);
- binaryDictionary.addUnigramWord("abcd", probability);
+ addUnigramWord(binaryDictionary, "aaa", probability);
+ addUnigramWord(binaryDictionary, "abcd", probability);
binaryDictionary.flush();
binaryDictionary.close();
@@ -401,7 +415,7 @@
assertEquals(probability, binaryDictionary.getFrequency("aaa"));
assertEquals(probability, binaryDictionary.getFrequency("abcd"));
- binaryDictionary.addUnigramWord("bcde", probability);
+ addUnigramWord(binaryDictionary, "bcde", probability);
binaryDictionary.flush();
binaryDictionary.close();
@@ -431,13 +445,13 @@
final int unigramProbability = 100;
final int bigramProbability = 10;
- binaryDictionary.addUnigramWord("aaa", unigramProbability);
- binaryDictionary.addUnigramWord("abb", unigramProbability);
- binaryDictionary.addUnigramWord("bcc", unigramProbability);
- binaryDictionary.addBigramWords("aaa", "abb", bigramProbability);
- binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability);
- binaryDictionary.addBigramWords("abb", "aaa", bigramProbability);
- binaryDictionary.addBigramWords("abb", "bcc", bigramProbability);
+ addUnigramWord(binaryDictionary, "aaa", unigramProbability);
+ addUnigramWord(binaryDictionary, "abb", unigramProbability);
+ addUnigramWord(binaryDictionary, "bcc", unigramProbability);
+ addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability);
+ addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability);
+ addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
+ addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
binaryDictionary.flushWithGC();
binaryDictionary.close();
@@ -497,7 +511,7 @@
words.add(word);
final int unigramProbability = random.nextInt(0xFF);
unigramProbabilities.put(word, unigramProbability);
- binaryDictionary.addUnigramWord(word, unigramProbability);
+ addUnigramWord(binaryDictionary, word, unigramProbability);
}
for (int i = 0; i < bigramCount; i++) {
@@ -510,7 +524,7 @@
bigramWords.add(bigram);
final int bigramProbability = random.nextInt(0xF);
bigramProbabilities.put(bigram, bigramProbability);
- binaryDictionary.addBigramWords(word0, word1, bigramProbability);
+ addBigramWords(binaryDictionary, word0, word1, bigramProbability);
}
binaryDictionary.flushWithGC();
@@ -568,7 +582,7 @@
words.add(word);
final int unigramProbability = random.nextInt(0xFF);
unigramProbabilities.put(word, unigramProbability);
- binaryDictionary.addUnigramWord(word, unigramProbability);
+ addUnigramWord(binaryDictionary, word, unigramProbability);
}
binaryDictionary.flushWithGC();
binaryDictionary.close();
@@ -584,7 +598,7 @@
words.add(word);
final int unigramProbability = random.nextInt(0xFF);
unigramProbabilities.put(word, unigramProbability);
- binaryDictionary.addUnigramWord(word, unigramProbability);
+ addUnigramWord(binaryDictionary, word, unigramProbability);
}
// Add bigram.
if (random.nextFloat() < addBigramProb && words.size() > 2) {
@@ -602,7 +616,7 @@
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
bigramWords.add(bigram);
bigramProbabilities.put(bigram, bigramProbability);
- binaryDictionary.addBigramWords(word0, word1, bigramProbability);
+ addBigramWords(binaryDictionary, word0, word1, bigramProbability);
}
// Remove bigram.
if (random.nextFloat() < removeBigramProb && !bigramWords.isEmpty()) {
@@ -674,7 +688,7 @@
words.add(word);
final int unigramProbability = random.nextInt(0xFF);
unigramProbabilities.put(word, unigramProbability);
- binaryDictionary.addUnigramWord(word, unigramProbability);
+ addUnigramWord(binaryDictionary, word, unigramProbability);
}
for (int j = 0; j < words.size(); j++) {
@@ -722,7 +736,7 @@
final String word = CodePointUtils.generateWord(random, codePointSet);
words.add(word);
final int unigramProbability = random.nextInt(0xFF);
- binaryDictionary.addUnigramWord(word, unigramProbability);
+ addUnigramWord(binaryDictionary, word, unigramProbability);
}
for (int j = 0; j < bigramCountPerIteration; j++) {
final String word0 = words.get(random.nextInt(words.size()));
@@ -732,7 +746,7 @@
}
bigrams.add(new Pair<String, String>(word0, word1));
final int bigramProbability = random.nextInt(0xF);
- binaryDictionary.addBigramWords(word0, word1, bigramProbability);
+ addBigramWords(binaryDictionary, word0, word1, bigramProbability);
}
assertEquals(new HashSet<String>(words).size(), Integer.parseInt(
binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
@@ -781,10 +795,10 @@
unigramProbabilities.put(word, probability);
if (prevWord == null) {
languageModelParams[i] = new LanguageModelParam(word, probability,
- BinaryDictionary.NOT_A_VALID_TIME_STAMP);
+ BinaryDictionary.NOT_A_VALID_TIMESTAMP);
} else {
languageModelParams[i] = new LanguageModelParam(prevWord, word, probability,
- bigramProbability, BinaryDictionary.NOT_A_VALID_TIME_STAMP);
+ bigramProbability, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
bigramProbabilities.put(new Pair<String, String>(prevWord, word),
bigramProbability);
}