Get locale using detected language for personalization.
Bug: 16547557
Change-Id: If3d88a548e5a2255ff81c819b056f77bfbe237ae
diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java
index 47aaead..e1acf3d 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java
@@ -24,6 +24,7 @@
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
+import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback;
import com.android.inputmethod.latin.PrevWordsInfo.WordInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.personalization.ContextualDictionary;
@@ -36,7 +37,6 @@
import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesAndSuggestions;
import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary;
import com.android.inputmethod.latin.utils.ExecutorUtils;
-import com.android.inputmethod.latin.utils.LanguageModelParam;
import com.android.inputmethod.latin.utils.SuggestionResults;
import java.io.File;
@@ -67,6 +67,7 @@
// To synchronize assigning mDictionaryGroup to ensure closing dictionaries.
private final Object mLock = new Object();
private final DistracterFilter mDistracterFilter;
+ private final PersonalizationDictionaryFacilitator mPersonalizationDictionaryFacilitator;
private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS =
new String[] {
@@ -174,14 +175,18 @@
public DictionaryFacilitator() {
mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER;
+ mPersonalizationDictionaryFacilitator = null;
}
public DictionaryFacilitator(final Context context) {
mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context);
+ mPersonalizationDictionaryFacilitator =
+ new PersonalizationDictionaryFacilitator(context, mDistracterFilter);
}
public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
mDistracterFilter.updateEnabledSubtypes(enabledSubtypes);
+ mPersonalizationDictionaryFacilitator.updateEnabledSubtypes(enabledSubtypes);
}
public Locale getLocale() {
@@ -353,6 +358,9 @@
dictionaryGroup.closeDict(dictType);
}
mDistracterFilter.close();
+ if (mPersonalizationDictionaryFacilitator != null) {
+ mPersonalizationDictionaryFacilitator.close();
+ }
}
@UsedForTesting
@@ -372,11 +380,11 @@
}
public void flushPersonalizationDictionary() {
- final ExpandableBinaryDictionary personalizationDict =
+ final ExpandableBinaryDictionary personalizationDictUsedForSuggestion =
mDictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION);
- if (personalizationDict != null) {
- personalizationDict.asyncFlushBinaryDictionary();
- }
+ mPersonalizationDictionaryFacilitator.flushPersonalizationDictionariesToUpdate(
+ personalizationDictUsedForSuggestion);
+ mDistracterFilter.close();
}
public void waitForLoadingMainDictionary(final long timeout, final TimeUnit unit)
@@ -580,6 +588,7 @@
// personalization dictionary.
public void clearPersonalizationDictionary() {
clearSubDictionary(Dictionary.TYPE_PERSONALIZATION);
+ mPersonalizationDictionaryFacilitator.clearDictionariesToUpdate();
}
public void clearContextualDictionary() {
@@ -589,30 +598,9 @@
public void addEntriesToPersonalizationDictionary(
final PersonalizationDataChunk personalizationDataChunk,
final SpacingAndPunctuations spacingAndPunctuations,
- final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {
- final ExpandableBinaryDictionary personalizationDict =
- mDictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION);
- if (personalizationDict == null) {
- if (callback != null) {
- callback.onFinished();
- }
- return;
- }
- // TODO: Get locale from personalizationDataChunk.mDetectedLanguage.
- final Locale dataChunkLocale = getLocale();
- final ArrayList<LanguageModelParam> languageModelParams =
- LanguageModelParam.createLanguageModelParamsFrom(
- personalizationDataChunk.mTokens,
- personalizationDataChunk.mTimestampInSeconds, spacingAndPunctuations,
- dataChunkLocale, new DistracterFilterCheckingIsInDictionary(
- mDistracterFilter, personalizationDict));
- if (languageModelParams == null || languageModelParams.isEmpty()) {
- if (callback != null) {
- callback.onFinished();
- }
- return;
- }
- personalizationDict.addMultipleDictionaryEntriesDynamically(languageModelParams, callback);
+ final AddMultipleDictionaryEntriesCallback callback) {
+ mPersonalizationDictionaryFacilitator.addEntriesToPersonalizationDictionariesToUpdate(
+ personalizationDataChunk, spacingAndPunctuations, callback);
}
public void addPhraseToContextualDictionary(final String[] phrase, final int probability,
diff --git a/java/src/com/android/inputmethod/latin/PersonalizationDictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/PersonalizationDictionaryFacilitator.java
new file mode 100644
index 0000000..03c108f
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/PersonalizationDictionaryFacilitator.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import android.content.Context;
+import android.view.inputmethod.InputMethodSubtype;
+
+import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback;
+import com.android.inputmethod.latin.personalization.PersonalizationDataChunk;
+import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
+import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
+import com.android.inputmethod.latin.utils.DistracterFilter;
+import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary;
+import com.android.inputmethod.latin.utils.LanguageModelParam;
+import com.android.inputmethod.latin.utils.SubtypeLocaleUtils;
+
+/**
+ * Class for managing and updating personalization dictionaries.
+ */
+public class PersonalizationDictionaryFacilitator {
+ private final Context mContext;
+ private final DistracterFilter mDistracterFilter;
+ private final HashMap<String, HashSet<Locale>> mLangToLocalesMap = new HashMap<>();
+ private final HashMap<Locale, ExpandableBinaryDictionary> mPersonalizationDictsToUpdate =
+ new HashMap<>();
+
+ PersonalizationDictionaryFacilitator(final Context context,
+ final DistracterFilter distracterFilter) {
+ mContext = context;
+ mDistracterFilter = distracterFilter;
+ }
+
+ public void close() {
+ mLangToLocalesMap.clear();
+ for (final ExpandableBinaryDictionary dict : mPersonalizationDictsToUpdate.values()) {
+ dict.close();
+ }
+ mPersonalizationDictsToUpdate.clear();
+ }
+
+ public void clearDictionariesToUpdate() {
+ for (final ExpandableBinaryDictionary dict : mPersonalizationDictsToUpdate.values()) {
+ dict.clear();
+ }
+ mPersonalizationDictsToUpdate.clear();
+ }
+
+ public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
+ for (final InputMethodSubtype subtype : enabledSubtypes) {
+ final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype);
+ final String language = locale.getLanguage();
+ final HashSet<Locale> locales = mLangToLocalesMap.get(language);
+ if (locales != null) {
+ locales.add(locale);
+ } else {
+ final HashSet<Locale> localeSet = new HashSet<>();
+ localeSet.add(locale);
+ mLangToLocalesMap.put(language, localeSet);
+ }
+ }
+ }
+
+ /**
+ * Flush personalization dictionaries to dictionary files. Close dictionaries after writing
+ * files except the dictionary that is used for generating suggestions.
+ *
+ * @param personalizationDictUsedForSuggestion the personalization dictionary used for
+ * generating suggestions that won't be closed.
+ */
+ public void flushPersonalizationDictionariesToUpdate(
+ final ExpandableBinaryDictionary personalizationDictUsedForSuggestion) {
+ for (final ExpandableBinaryDictionary personalizationDict :
+ mPersonalizationDictsToUpdate.values()) {
+ personalizationDict.asyncFlushBinaryDictionary();
+ if (personalizationDict != personalizationDictUsedForSuggestion) {
+ // Close if the dictionary is not being used for suggestion.
+ personalizationDict.close();
+ }
+ }
+ mDistracterFilter.close();
+ mPersonalizationDictsToUpdate.clear();
+ }
+
+ private ExpandableBinaryDictionary getPersonalizationDictToUpdate(final Context context,
+ final Locale locale) {
+ ExpandableBinaryDictionary personalizationDict = mPersonalizationDictsToUpdate.get(locale);
+ if (personalizationDict != null) {
+ return personalizationDict;
+ }
+ personalizationDict = PersonalizationDictionary.getDictionary(context, locale,
+ null /* dictFile */, "" /* dictNamePrefix */);
+ mPersonalizationDictsToUpdate.put(locale, personalizationDict);
+ return personalizationDict;
+ }
+
+ private void addEntriesToPersonalizationDictionariesForLocale(final Locale locale,
+ final PersonalizationDataChunk personalizationDataChunk,
+ final SpacingAndPunctuations spacingAndPunctuations,
+ final AddMultipleDictionaryEntriesCallback callback) {
+ final ExpandableBinaryDictionary personalizationDict =
+ getPersonalizationDictToUpdate(mContext, locale);
+ if (personalizationDict == null) {
+ if (callback != null) {
+ callback.onFinished();
+ }
+ return;
+ }
+ final ArrayList<LanguageModelParam> languageModelParams =
+ LanguageModelParam.createLanguageModelParamsFrom(
+ personalizationDataChunk.mTokens,
+ personalizationDataChunk.mTimestampInSeconds, spacingAndPunctuations,
+ locale, new DistracterFilterCheckingIsInDictionary(
+ mDistracterFilter, personalizationDict));
+ if (languageModelParams == null || languageModelParams.isEmpty()) {
+ if (callback != null) {
+ callback.onFinished();
+ }
+ return;
+ }
+ personalizationDict.addMultipleDictionaryEntriesDynamically(languageModelParams, callback);
+ }
+
+ public void addEntriesToPersonalizationDictionariesToUpdate(
+ final PersonalizationDataChunk personalizationDataChunk,
+ final SpacingAndPunctuations spacingAndPunctuations,
+ final AddMultipleDictionaryEntriesCallback callback) {
+ final HashSet<Locale> locales =
+ mLangToLocalesMap.get(personalizationDataChunk.mDetectedLanguage);
+ if (locales == null || locales.isEmpty()) {
+ if (callback != null) {
+ callback.onFinished();
+ }
+ return;
+ }
+ final AtomicInteger remainingTaskCount = new AtomicInteger(locales.size());
+ final AddMultipleDictionaryEntriesCallback callbackForLocales =
+ new AddMultipleDictionaryEntriesCallback() {
+ @Override
+ public void onFinished() {
+ if (remainingTaskCount.decrementAndGet() == 0) {
+ // Update tasks for all locales have been finished.
+ if (callback != null) {
+ callback.onFinished();
+ }
+ }
+ }
+ };
+ for (final Locale locale : locales) {
+ addEntriesToPersonalizationDictionariesForLocale(locale, personalizationDataChunk,
+ spacingAndPunctuations, callbackForLocales);
+ }
+ }
+}
diff --git a/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java
index e9a97ff..4e7e814 100644
--- a/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java
@@ -29,6 +29,7 @@
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
+import com.android.inputmethod.latin.RichInputMethodManager;
import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback;
import com.android.inputmethod.latin.makedict.CodePointUtils;
import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
@@ -36,6 +37,7 @@
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
+import android.view.inputmethod.InputMethodSubtype;
/**
* Unit tests for personalization dictionary
@@ -55,16 +57,28 @@
final DictionaryFacilitator dictionaryFacilitator = new DictionaryFacilitator(getContext());
dictionaryFacilitator.resetDictionariesForTesting(getContext(), LOCALE_EN_US, dictTypes,
new HashMap<String, File>(), new HashMap<String, Map<String, String>>());
+ // Set subtypes.
+ RichInputMethodManager.init(getContext());
+ final RichInputMethodManager richImm = RichInputMethodManager.getInstance();
+ final ArrayList<InputMethodSubtype> subtypes = new ArrayList<>();
+ subtypes.add(richImm.findSubtypeByLocaleAndKeyboardLayoutSet(
+ LOCALE_EN_US.toString(), "qwerty"));
+ dictionaryFacilitator.updateEnabledSubtypes(subtypes);
return dictionaryFacilitator;
}
public void testAddManyTokens() {
final DictionaryFacilitator dictionaryFacilitator = getDictionaryFacilitator();
dictionaryFacilitator.clearPersonalizationDictionary();
- final int dataChunkCount = 20;
- final int wordCountInOneChunk = 2000;
+ final int dataChunkCount = 100;
+ final int wordCountInOneChunk = 200;
+ final int uniqueWordCount = 100;
final Random random = new Random(System.currentTimeMillis());
final int[] codePointSet = CodePointUtils.LATIN_ALPHABETS_LOWER;
+ final ArrayList<String> words = new ArrayList<>();
+ for (int i = 0; i < uniqueWordCount; i++) {
+ words.add(CodePointUtils.generateWord(random, codePointSet));
+ }
final SpacingAndPunctuations spacingAndPunctuations =
new SpacingAndPunctuations(getContext().getResources());
@@ -75,7 +89,7 @@
for (int i = 0; i < dataChunkCount; i++) {
final ArrayList<String> tokens = new ArrayList<>();
for (int j = 0; j < wordCountInOneChunk; j++) {
- tokens.add(CodePointUtils.generateWord(random, codePointSet));
+ tokens.add(words.get(random.nextInt(words.size())));
}
final PersonalizationDataChunk personalizationDataChunk = new PersonalizationDataChunk(
true /* inputByUser */, tokens, timeStampInSeconds, DUMMY_PACKAGE_NAME,