diff --git a/res/layout/directory_header.xml b/res/layout/directory_header.xml
index 53207e0..cf36093 100644
--- a/res/layout/directory_header.xml
+++ b/res/layout/directory_header.xml
@@ -18,7 +18,7 @@
 <RelativeLayout
     xmlns:android="http://schemas.android.com/apk/res/android"
     android:layout_width="match_parent"
-    android:layout_height="48dip"
+    android:layout_height="56dip"
     android:background="@drawable/directory_bg"
     >
     <TextView
@@ -49,7 +49,6 @@
         android:layout_alignParentLeft="true"
         android:layout_centerVertical="true"
         android:layout_marginLeft="6dip"
-        android:layout_marginBottom="-4dip"
         android:layout_above="@id/display_name"
         android:textAppearance="?android:attr/textAppearanceMedium"
         android:singleLine="true"
diff --git a/src/com/android/contacts/list/ContactEntryListAdapter.java b/src/com/android/contacts/list/ContactEntryListAdapter.java
index 2717c51..1ebf3c9 100644
--- a/src/com/android/contacts/list/ContactEntryListAdapter.java
+++ b/src/com/android/contacts/list/ContactEntryListAdapter.java
@@ -108,9 +108,9 @@
             if (partition instanceof DirectoryPartition) {
                 DirectoryPartition directoryPartition = (DirectoryPartition)partition;
                 if (!directoryPartition.isLoading()) {
-                    directoryPartition.setLoading(true);
                     notify = true;
                 }
+                directoryPartition.setStatus(DirectoryPartition.STATUS_NOT_LOADED);
             }
         }
         if (notify) {
@@ -269,7 +269,7 @@
 
         Partition partition = getPartition(partitionIndex);
         if (partition instanceof DirectoryPartition) {
-            ((DirectoryPartition)partition).setLoading(false);
+            ((DirectoryPartition)partition).setStatus(DirectoryPartition.STATUS_LOADED);
         }
 
         super.changeCursor(partitionIndex, cursor);
diff --git a/src/com/android/contacts/list/ContactEntryListFragment.java b/src/com/android/contacts/list/ContactEntryListFragment.java
index ca5a9a8..f115676 100644
--- a/src/com/android/contacts/list/ContactEntryListFragment.java
+++ b/src/com/android/contacts/list/ContactEntryListFragment.java
@@ -304,13 +304,18 @@
             Partition partition = mAdapter.getPartition(i);
             if (partition instanceof DirectoryPartition) {
                 DirectoryPartition directoryPartition = (DirectoryPartition)partition;
-                if (mLoadPriorityDirectoriesOnly == directoryPartition.isPriorityDirectory()) {
-                    startLoadingDirectoryPartition(i);
+                if (directoryPartition.getStatus() == DirectoryPartition.STATUS_NOT_LOADED) {
+                    if (directoryPartition.isPriorityDirectory() || !mLoadPriorityDirectoriesOnly) {
+                        startLoadingDirectoryPartition(i);
+                    }
                 }
             } else {
                 getLoaderManager().initLoader(i, null, this);
             }
         }
+
+        // Next time this method is called, we should start loading non-priority directories
+        mLoadPriorityDirectoriesOnly = false;
     }
 
     @Override
@@ -331,6 +336,7 @@
 
     private void startLoadingDirectoryPartition(int partitionIndex) {
         DirectoryPartition partition = (DirectoryPartition)mAdapter.getPartition(partitionIndex);
+        partition.setStatus(DirectoryPartition.STATUS_LOADING);
         long directoryId = partition.getDirectoryId();
         if (mForceLoad) {
             if (directoryId == Directory.DEFAULT) {
@@ -384,20 +390,19 @@
 
         int loaderId = loader.getId();
         if (loaderId == DIRECTORY_LOADER_ID) {
-            removePendingDirectorySearchRequests();
             mAdapter.changeDirectories(data);
+            startLoading();
         } else {
             onPartitionLoaded(loaderId, data);
-        }
-
-        if (isDirectorySearchEnabled()) {
-            if (mLoadDirectoryList) {
-                mLoadDirectoryList = false;
-                getLoaderManager().initLoader(DIRECTORY_LOADER_ID, null, this);
-            } else if (mLoadPriorityDirectoriesOnly) {
-                mLoadPriorityDirectoriesOnly = false;
-                startLoading();
+            if (isDirectorySearchEnabled()) {
+                if (mLoadDirectoryList) {
+                    mLoadDirectoryList = false;
+                    getLoaderManager().initLoader(DIRECTORY_LOADER_ID, null, this);
+                } else {
+                    startLoading();
+                }
             }
+
         }
 
 // TODO fix the empty view
@@ -425,6 +430,7 @@
     }
 
     protected void reloadData() {
+        removePendingDirectorySearchRequests();
         mAdapter.onDataReload();
         mLoadPriorityDirectoriesOnly = true;
         mForceLoad = true;
diff --git a/src/com/android/contacts/list/DirectoryPartition.java b/src/com/android/contacts/list/DirectoryPartition.java
index b55ed31..88cc879 100644
--- a/src/com/android/contacts/list/DirectoryPartition.java
+++ b/src/com/android/contacts/list/DirectoryPartition.java
@@ -23,10 +23,15 @@
  * Model object for a {@link Directory} row.
  */
 public final class DirectoryPartition extends CompositeCursorAdapter.Partition {
+
+    public static final int STATUS_NOT_LOADED = 0;
+    public static final int STATUS_LOADING = 1;
+    public static final int STATUS_LOADED = 2;
+
     private long mDirectoryId;
     private String mDirectoryType;
     private String mDisplayName;
-    private boolean mLoading;
+    private int mStatus;
     private boolean mPriorityDirectory;
 
     public DirectoryPartition(boolean showIfEmpty, boolean hasHeader) {
@@ -67,12 +72,16 @@
         this.mDisplayName = displayName;
     }
 
-    public boolean isLoading() {
-        return mLoading;
+    public int getStatus() {
+        return mStatus;
     }
 
-    public void setLoading(boolean loading) {
-        mLoading = loading;
+    public void setStatus(int status) {
+        mStatus = status;
+    }
+
+    public boolean isLoading() {
+        return mStatus == STATUS_NOT_LOADED || mStatus == STATUS_LOADING;
     }
 
     /**