diff --git a/Ext4Crypt.cpp b/Ext4Crypt.cpp
index 3a31861..11f104a 100644
--- a/Ext4Crypt.cpp
+++ b/Ext4Crypt.cpp
@@ -443,7 +443,10 @@
     bool success = true;
     s_ce_keys.erase(user_id);
     std::string raw_ref;
-    success &= lookup_key_ref(s_ce_key_raw_refs, user_id, &raw_ref) && evict_key(raw_ref);
+    // If we haven't loaded the CE key, no need to evict it.
+    if (lookup_key_ref(s_ce_key_raw_refs, user_id, &raw_ref)) {
+        success &= evict_key(raw_ref);
+    }
     s_ce_key_raw_refs.erase(user_id);
     success &= lookup_key_ref(s_de_key_raw_refs, user_id, &raw_ref) && evict_key(raw_ref);
     s_de_key_raw_refs.erase(user_id);