diff --git a/java/src/com/android/inputmethod/keyboard/EmojiPalettesView.java b/java/src/com/android/inputmethod/keyboard/EmojiPalettesView.java
index ef9448c..4f59611 100644
--- a/java/src/com/android/inputmethod/keyboard/EmojiPalettesView.java
+++ b/java/src/com/android/inputmethod/keyboard/EmojiPalettesView.java
@@ -74,6 +74,7 @@
         ViewPager.OnPageChangeListener, View.OnClickListener,
         ScrollKeyboardView.OnKeyClickListener {
     private static final String TAG = EmojiPalettesView.class.getSimpleName();
+    private static final boolean DEBUG_PAGER = false;
     private final int mKeyBackgroundId;
     private final int mEmojiFunctionalKeyBackgroundId;
     private final KeyboardLayoutSet mLayoutSet;
@@ -566,6 +567,20 @@
         // TODO:
     }
 
+    public void startEmojiPalettes() {
+        if (DEBUG_PAGER) {
+            Log.d(TAG, "allocate emoji palettes memory");
+        }
+        mEmojiPager.setAdapter(mEmojiPalettesAdapter);
+    }
+
+    public void stopEmojiPalettes() {
+        if (DEBUG_PAGER) {
+            Log.d(TAG, "deallocate emoji palettes memory");
+        }
+        mEmojiPager.setAdapter(null);
+    }
+
     public void setKeyboardActionListener(final KeyboardActionListener listener) {
         mKeyboardActionListener = listener;
         mDeleteKeyOnTouchListener.setKeyboardActionListener(mKeyboardActionListener);
@@ -663,6 +678,9 @@
 
         @Override
         public Object instantiateItem(final ViewGroup container, final int position) {
+            if (DEBUG_PAGER) {
+                Log.d(TAG, "instantiate item: " + position);
+            }
             final ScrollKeyboardView oldKeyboardView = mActiveKeyboardViews.get(position);
             if (oldKeyboardView != null) {
                 oldKeyboardView.deallocateMemory();
@@ -694,13 +712,19 @@
         @Override
         public void destroyItem(final ViewGroup container, final int position,
                 final Object object) {
-            ScrollKeyboardView keyboardView = mActiveKeyboardViews.get(position);
+            if (DEBUG_PAGER) {
+                Log.d(TAG, "destroy item: " + position + ", " + object.getClass().getSimpleName());
+            }
+            final ScrollKeyboardView keyboardView = mActiveKeyboardViews.get(position);
             if (keyboardView != null) {
                 keyboardView.deallocateMemory();
                 mActiveKeyboardViews.remove(position);
             }
-            container.removeView(keyboardView);
-            keyboardView = null;
+            if (object instanceof View) {
+                container.removeView((View)object);
+            } else {
+                Log.w(TAG, "Warning!!! Emoji palette may be leaking. " + object);
+            }
         }
     }
 
diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
index 4fc1082..9760983 100644
--- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
+++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java
@@ -260,12 +260,14 @@
     private void setMainKeyboardFrame() {
         mMainKeyboardFrame.setVisibility(View.VISIBLE);
         mEmojiPalettesView.setVisibility(View.GONE);
+        mEmojiPalettesView.stopEmojiPalettes();
     }
 
     // Implements {@link KeyboardState.SwitchActions}.
     @Override
     public void setEmojiKeyboard() {
         mMainKeyboardFrame.setVisibility(View.GONE);
+        mEmojiPalettesView.startEmojiPalettes();
         mEmojiPalettesView.setVisibility(View.VISIBLE);
     }
 
@@ -336,6 +338,16 @@
         return mKeyboardView;
     }
 
+    public void deallocateMemory() {
+        if (mKeyboardView != null) {
+            mKeyboardView.cancelAllOngoingEvents();
+            mKeyboardView.deallocateMemory();
+        }
+        if (mEmojiPalettesView != null) {
+            mEmojiPalettesView.stopEmojiPalettes();
+        }
+    }
+
     public View onCreateInputView(final boolean isHardwareAcceleratedDrawingEnabled) {
         if (mKeyboardView != null) {
             mKeyboardView.closing();
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index 4d7e43e..ccdbd0d 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -926,10 +926,11 @@
                 suggest.setAutoCorrectionThreshold(currentSettingsValues.mAutoCorrectionThreshold);
             }
 
-            if (canReachInputConnection) {
-                // If we can't reach the input connection, we don't want to call loadKeyboard yet.
-                // It will be done in #retryResetCaches.
-                switcher.loadKeyboard(editorInfo, currentSettingsValues);
+            switcher.loadKeyboard(editorInfo, currentSettingsValues);
+            if (!canReachInputConnection) {
+                // If we can't reach the input connection, we will call loadKeyboard again later,
+                // so we need to save its state now. The call will be done in #retryResetCaches.
+                switcher.saveKeyboardState();
             }
         } else if (restarting) {
             // TODO: Come up with a more comprehensive way to reset the keyboard layout when
@@ -1049,11 +1050,7 @@
     private void onFinishInputViewInternal(final boolean finishingInput) {
         super.onFinishInputView(finishingInput);
         mKeyboardSwitcher.onFinishInputView();
-        final MainKeyboardView mainKeyboardView = mKeyboardSwitcher.getMainKeyboardView();
-        if (mainKeyboardView != null) {
-            mainKeyboardView.cancelAllOngoingEvents();
-            mainKeyboardView.deallocateMemory();
-        }
+        mKeyboardSwitcher.deallocateMemory();
         // Remove pending messages related to update suggestions
         mHandler.cancelUpdateSuggestionStrip();
         // Should do the following in onFinishInputInternal but until JB MR2 it's not called :(