diff --git a/Android.bp b/Android.bp
index 4852fe4..1c0ed17 100644
--- a/Android.bp
+++ b/Android.bp
@@ -54,6 +54,7 @@
         "libdiskconfig",
         "libext4_utils",
         "libf2fs_sparseblock",
+        "libgsi",
         "libhardware",
         "libhardware_legacy",
         "libincfs",
diff --git a/Checkpoint.cpp b/Checkpoint.cpp
index 61035e5..755f0e3 100644
--- a/Checkpoint.cpp
+++ b/Checkpoint.cpp
@@ -47,7 +47,6 @@
 using android::base::SetProperty;
 using android::binder::Status;
 using android::fs_mgr::Fstab;
-using android::fs_mgr::ReadDefaultFstab;
 using android::fs_mgr::ReadFstabFromFile;
 using android::hardware::hidl_string;
 using android::hardware::boot::V1_0::BoolResult;
diff --git a/EncryptInplace.cpp b/EncryptInplace.cpp
index bdb2da7..057b3ef 100644
--- a/EncryptInplace.cpp
+++ b/EncryptInplace.cpp
@@ -20,614 +20,361 @@
 #include <ext4_utils/ext4_utils.h>
 #include <f2fs_sparseblock.h>
 #include <fcntl.h>
-#include <inttypes.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <sys/stat.h>
-#include <sys/types.h>
 #include <time.h>
 
 #include <algorithm>
+#include <vector>
 
 #include <android-base/logging.h>
 #include <android-base/properties.h>
+#include <android-base/unique_fd.h>
 
-// HORRIBLE HACK, FIXME
-#include "cryptfs.h"
-
-// FIXME horrible cut-and-paste code
-static inline int unix_read(int fd, void* buff, int len) {
-    return TEMP_FAILURE_RETRY(read(fd, buff, len));
-}
-
-static inline int unix_write(int fd, const void* buff, int len) {
-    return TEMP_FAILURE_RETRY(write(fd, buff, len));
-}
-
-#define CRYPT_SECTORS_PER_BUFSIZE (CRYPT_INPLACE_BUFSIZE / CRYPT_SECTOR_SIZE)
-
-/* aligned 32K writes tends to make flash happy.
- * SD card association recommends it.
- */
-#define BLOCKS_AT_A_TIME 8
-
-struct encryptGroupsData {
-    int realfd;
-    int cryptofd;
-    off64_t numblocks;
-    off64_t one_pct, cur_pct, new_pct;
-    off64_t blocks_already_done, tot_numblocks;
-    off64_t used_blocks_already_done, tot_used_blocks;
-    const char* real_blkdev;
-    const char* crypto_blkdev;
-    int count;
-    off64_t offset;
-    char* buffer;
-    off64_t last_written_sector;
-    int completed;
-    time_t time_started;
-    int remaining_time;
-    bool set_progress_properties;
+enum EncryptInPlaceError {
+    kSuccess,
+    kFailed,
+    kFilesystemNotFound,
 };
 
-static void update_progress(struct encryptGroupsData* data, int is_used) {
-    data->blocks_already_done++;
+static uint64_t round_up(uint64_t val, size_t amount) {
+    if (val % amount) val += amount - (val % amount);
+    return val;
+}
 
-    if (is_used) {
-        data->used_blocks_already_done++;
-    }
-    if (data->tot_used_blocks) {
-        data->new_pct = data->used_blocks_already_done / data->one_pct;
+class InPlaceEncrypter {
+  public:
+    bool EncryptInPlace(const std::string& crypto_blkdev, const std::string& real_blkdev,
+                        uint64_t nr_sec, bool set_progress_properties);
+    bool ProcessUsedBlock(uint64_t block_num);
+
+  private:
+    // aligned 32K writes tends to make flash happy.
+    // SD card association recommends it.
+    static const size_t kIOBufferSize = 32768;
+
+    // Avoid spamming the logs.  Print the "Encrypting blocks" log message once
+    // every 10000 blocks (which is usually every 40 MB or so), and once at the end.
+    static const int kLogInterval = 10000;
+
+    std::string DescribeFilesystem();
+    void InitFs(const std::string& fs_type, uint64_t blocks_to_encrypt, uint64_t total_blocks,
+                unsigned int block_size);
+    void UpdateProgress(size_t blocks, bool done);
+    bool EncryptPendingData();
+    bool DoEncryptInPlace();
+
+    // ext4 methods
+    bool ReadExt4BlockBitmap(uint32_t group, uint8_t* buf);
+    uint64_t FirstBlockInGroup(uint32_t group);
+    uint32_t NumBlocksInGroup(uint32_t group);
+    uint32_t NumBaseMetaBlocksInGroup(uint64_t group);
+    EncryptInPlaceError EncryptInPlaceExt4();
+
+    // f2fs methods
+    EncryptInPlaceError EncryptInPlaceF2fs();
+
+    std::string real_blkdev_;
+    std::string crypto_blkdev_;
+    uint64_t nr_sec_;
+    bool set_progress_properties_;
+
+    android::base::unique_fd realfd_;
+    android::base::unique_fd cryptofd_;
+
+    time_t time_started_;
+    int remaining_time_;
+
+    std::string fs_type_;
+    uint64_t blocks_done_;
+    uint64_t blocks_to_encrypt_;
+    unsigned int block_size_;
+    unsigned int cur_pct_;
+
+    std::vector<uint8_t> io_buffer_;
+    uint64_t first_pending_block_;
+    size_t blocks_pending_;
+};
+
+std::string InPlaceEncrypter::DescribeFilesystem() {
+    if (fs_type_.empty())
+        return "full block device " + real_blkdev_;
+    else
+        return fs_type_ + " filesystem on " + real_blkdev_;
+}
+
+// Finishes initializing the encrypter, now that the filesystem details are known.
+void InPlaceEncrypter::InitFs(const std::string& fs_type, uint64_t blocks_to_encrypt,
+                              uint64_t total_blocks, unsigned int block_size) {
+    fs_type_ = fs_type;
+    blocks_done_ = 0;
+    blocks_to_encrypt_ = blocks_to_encrypt;
+    block_size_ = block_size;
+    cur_pct_ = 0;
+
+    // Allocate the I/O buffer.  kIOBufferSize should always be a multiple of
+    // the filesystem block size, but round it up just in case.
+    io_buffer_.resize(round_up(kIOBufferSize, block_size));
+    first_pending_block_ = 0;
+    blocks_pending_ = 0;
+
+    LOG(INFO) << "Encrypting " << DescribeFilesystem() << " in-place via " << crypto_blkdev_;
+    LOG(INFO) << blocks_to_encrypt << " blocks (" << (blocks_to_encrypt * block_size) / 1000000
+              << " MB) of " << total_blocks << " blocks are in-use";
+}
+
+void InPlaceEncrypter::UpdateProgress(size_t blocks, bool done) {
+    // A log message already got printed for blocks_done_ if one was due, so the
+    // next message will be due at the *next* block rounded up to kLogInterval.
+    uint64_t blocks_next_msg = round_up(blocks_done_ + 1, kLogInterval);
+
+    blocks_done_ += blocks;
+
+    // Ensure that a log message gets printed at the end, but not if one was
+    // already printed due to the block count being a multiple of kLogInterval.
+    // E.g. we want to show "50000 of 50327" and then "50327 of "50327", but not
+    // "50000 of 50000" and then redundantly "50000 of 50000" again.
+    if (done && blocks_done_ % kLogInterval != 0) blocks_next_msg = blocks_done_;
+
+    if (blocks_done_ >= blocks_next_msg)
+        LOG(DEBUG) << "Encrypted " << blocks_next_msg << " of " << blocks_to_encrypt_ << " blocks";
+
+    if (!set_progress_properties_) return;
+
+    uint64_t new_pct;
+    if (done) {
+        new_pct = 100;
     } else {
-        data->new_pct = data->blocks_already_done / data->one_pct;
+        new_pct = (blocks_done_ * 100) / std::max<uint64_t>(blocks_to_encrypt_, 1);
+        new_pct = std::min<uint64_t>(new_pct, 99);
+    }
+    if (new_pct > cur_pct_) {
+        cur_pct_ = new_pct;
+        android::base::SetProperty("vold.encrypt_progress", std::to_string(new_pct));
     }
 
-    if (!data->set_progress_properties) return;
-
-    if (data->new_pct > data->cur_pct) {
-        char buf[8];
-        data->cur_pct = data->new_pct;
-        snprintf(buf, sizeof(buf), "%" PRId64, data->cur_pct);
-        android::base::SetProperty("vold.encrypt_progress", buf);
-    }
-
-    if (data->cur_pct >= 5) {
+    if (cur_pct_ >= 5) {
         struct timespec time_now;
         if (clock_gettime(CLOCK_MONOTONIC, &time_now)) {
-            LOG(WARNING) << "Error getting time";
+            PLOG(WARNING) << "Error getting time while updating encryption progress";
         } else {
-            double elapsed_time = difftime(time_now.tv_sec, data->time_started);
-            off64_t remaining_blocks = data->tot_used_blocks - data->used_blocks_already_done;
-            int remaining_time =
-                (int)(elapsed_time * remaining_blocks / data->used_blocks_already_done);
+            double elapsed_time = difftime(time_now.tv_sec, time_started_);
+
+            uint64_t remaining_blocks = 0;
+            if (blocks_done_ < blocks_to_encrypt_)
+                remaining_blocks = blocks_to_encrypt_ - blocks_done_;
+
+            int remaining_time = 0;
+            if (blocks_done_ != 0)
+                remaining_time = (int)(elapsed_time * remaining_blocks / blocks_done_);
 
             // Change time only if not yet set, lower, or a lot higher for
             // best user experience
-            if (data->remaining_time == -1 || remaining_time < data->remaining_time ||
-                remaining_time > data->remaining_time + 60) {
-                char buf[8];
-                snprintf(buf, sizeof(buf), "%d", remaining_time);
-                android::base::SetProperty("vold.encrypt_time_remaining", buf);
-                data->remaining_time = remaining_time;
+            if (remaining_time_ == -1 || remaining_time < remaining_time_ ||
+                remaining_time > remaining_time_ + 60) {
+                remaining_time_ = remaining_time;
+                android::base::SetProperty("vold.encrypt_time_remaining",
+                                           std::to_string(remaining_time));
             }
         }
     }
 }
 
-static void log_progress(struct encryptGroupsData const* data, bool completed) {
-    // Precondition - if completed data = 0 else data != 0
+bool InPlaceEncrypter::EncryptPendingData() {
+    if (blocks_pending_ == 0) return true;
 
-    // Track progress so we can skip logging blocks
-    static off64_t offset = -1;
+    ssize_t bytes = blocks_pending_ * block_size_;
+    uint64_t offset = first_pending_block_ * block_size_;
 
-    // Need to close existing 'Encrypting from' log?
-    if (completed || (offset != -1 && data->offset != offset)) {
-        LOG(INFO) << "Encrypted to sector " << offset / info.block_size * CRYPT_SECTOR_SIZE;
-        offset = -1;
+    if (pread64(realfd_, &io_buffer_[0], bytes, offset) != bytes) {
+        PLOG(ERROR) << "Error reading real_blkdev " << real_blkdev_ << " for inplace encrypt";
+        return false;
     }
 
-    // Need to start new 'Encrypting from' log?
-    if (!completed && offset != data->offset) {
-        LOG(INFO) << "Encrypting from sector " << data->offset / info.block_size * CRYPT_SECTOR_SIZE;
+    if (pwrite64(cryptofd_, &io_buffer_[0], bytes, offset) != bytes) {
+        PLOG(ERROR) << "Error writing crypto_blkdev " << crypto_blkdev_ << " for inplace encrypt";
+        return false;
     }
 
-    // Update offset
-    if (!completed) {
-        offset = data->offset + (off64_t)data->count * info.block_size;
-    }
+    UpdateProgress(blocks_pending_, false);
+
+    blocks_pending_ = 0;
+    return true;
 }
 
-static int flush_outstanding_data(struct encryptGroupsData* data) {
-    if (data->count == 0) {
-        return 0;
+bool InPlaceEncrypter::ProcessUsedBlock(uint64_t block_num) {
+    // Flush if the amount of pending data has reached the I/O buffer size, if
+    // there's a gap between the pending blocks and the next block (due to
+    // block(s) not being used by the filesystem and thus not needing
+    // encryption), or if the next block will be aligned to the I/O buffer size.
+    if (blocks_pending_ * block_size_ == io_buffer_.size() ||
+        block_num != first_pending_block_ + blocks_pending_ ||
+        (block_num * block_size_) % io_buffer_.size() == 0) {
+        if (!EncryptPendingData()) return false;
+        first_pending_block_ = block_num;
+    }
+    blocks_pending_++;
+    return true;
+}
+
+// Reads the block bitmap for block group |group| into |buf|.
+bool InPlaceEncrypter::ReadExt4BlockBitmap(uint32_t group, uint8_t* buf) {
+    uint64_t offset = (uint64_t)aux_info.bg_desc[group].bg_block_bitmap * info.block_size;
+    if (pread64(realfd_, buf, info.block_size, offset) != (ssize_t)info.block_size) {
+        PLOG(ERROR) << "Failed to read block bitmap for block group " << group;
+        return false;
+    }
+    return true;
+}
+
+uint64_t InPlaceEncrypter::FirstBlockInGroup(uint32_t group) {
+    return aux_info.first_data_block + (group * (uint64_t)info.blocks_per_group);
+}
+
+uint32_t InPlaceEncrypter::NumBlocksInGroup(uint32_t group) {
+    uint64_t remaining = aux_info.len_blocks - FirstBlockInGroup(group);
+    return std::min<uint64_t>(info.blocks_per_group, remaining);
+}
+
+// In block groups with an uninitialized block bitmap, we only need to encrypt
+// the backup superblock and the block group descriptors (if they are present).
+uint32_t InPlaceEncrypter::NumBaseMetaBlocksInGroup(uint64_t group) {
+    if (!ext4_bg_has_super_block(group)) return 0;
+    return 1 + aux_info.bg_desc_blocks;
+}
+
+EncryptInPlaceError InPlaceEncrypter::EncryptInPlaceExt4() {
+    if (setjmp(setjmp_env))  // NOLINT
+        return kFilesystemNotFound;
+
+    if (read_ext(realfd_, 0) != 0) return kFilesystemNotFound;
+
+    LOG(DEBUG) << "ext4 filesystem has " << aux_info.groups << " block groups";
+
+    uint64_t blocks_to_encrypt = 0;
+    for (uint32_t group = 0; group < aux_info.groups; group++) {
+        if (aux_info.bg_desc[group].bg_flags & EXT4_BG_BLOCK_UNINIT)
+            blocks_to_encrypt += NumBaseMetaBlocksInGroup(group);
+        else
+            blocks_to_encrypt +=
+                    (NumBlocksInGroup(group) - aux_info.bg_desc[group].bg_free_blocks_count);
     }
 
-    LOG(DEBUG) << "Copying " << data->count << " blocks at offset " << data->offset;
+    InitFs("ext4", blocks_to_encrypt, aux_info.len_blocks, info.block_size);
 
-    if (pread64(data->realfd, data->buffer, info.block_size * data->count, data->offset) <= 0) {
-        LOG(ERROR) << "Error reading real_blkdev " << data->real_blkdev << " for inplace encrypt";
-        return -1;
+    // Encrypt each block group.
+    std::vector<uint8_t> block_bitmap(info.block_size);
+    for (uint32_t group = 0; group < aux_info.groups; group++) {
+        if (!ReadExt4BlockBitmap(group, &block_bitmap[0])) return kFailed;
+
+        uint64_t first_block_num = FirstBlockInGroup(group);
+        bool uninit = (aux_info.bg_desc[group].bg_flags & EXT4_BG_BLOCK_UNINIT);
+        uint32_t block_count = uninit ? NumBaseMetaBlocksInGroup(group) : NumBlocksInGroup(group);
+
+        // Encrypt each used block in the block group.
+        for (uint32_t i = 0; i < block_count; i++) {
+            if (uninit || bitmap_get_bit(&block_bitmap[0], i))
+                ProcessUsedBlock(first_block_num + i);
+        }
     }
+    return kSuccess;
+}
 
-    if (pwrite64(data->cryptofd, data->buffer, info.block_size * data->count, data->offset) <= 0) {
-        LOG(ERROR) << "Error writing crypto_blkdev " << data->crypto_blkdev
-                   << " for inplace encrypt";
-        return -1;
-    } else {
-        log_progress(data, false);
-    }
-
-    data->count = 0;
-    data->last_written_sector =
-        (data->offset + data->count) / info.block_size * CRYPT_SECTOR_SIZE - 1;
+static int encrypt_f2fs_block(uint64_t block_num, void* _encrypter) {
+    InPlaceEncrypter* encrypter = reinterpret_cast<InPlaceEncrypter*>(_encrypter);
+    if (!encrypter->ProcessUsedBlock(block_num)) return -1;
     return 0;
 }
 
-static int encrypt_groups(struct encryptGroupsData* data) {
-    unsigned int i;
-    u8* block_bitmap = 0;
-    unsigned int block;
-    off64_t ret;
-    int rc = -1;
+EncryptInPlaceError InPlaceEncrypter::EncryptInPlaceF2fs() {
+    std::unique_ptr<struct f2fs_info, void (*)(struct f2fs_info*)> fs_info(
+            generate_f2fs_info(realfd_), free_f2fs_info);
+    if (!fs_info) return kFilesystemNotFound;
 
-    data->buffer = (char*)malloc(info.block_size * BLOCKS_AT_A_TIME);
-    if (!data->buffer) {
-        LOG(ERROR) << "Failed to allocate crypto buffer";
-        goto errout;
-    }
-
-    block_bitmap = (u8*)malloc(info.block_size);
-    if (!block_bitmap) {
-        LOG(ERROR) << "failed to allocate block bitmap";
-        goto errout;
-    }
-
-    for (i = 0; i < aux_info.groups; ++i) {
-        LOG(INFO) << "Encrypting group " << i;
-
-        u32 first_block = aux_info.first_data_block + i * info.blocks_per_group;
-        u32 block_count = std::min(info.blocks_per_group, (u32)(aux_info.len_blocks - first_block));
-
-        off64_t offset = (u64)info.block_size * aux_info.bg_desc[i].bg_block_bitmap;
-
-        ret = pread64(data->realfd, block_bitmap, info.block_size, offset);
-        if (ret != (int)info.block_size) {
-            LOG(ERROR) << "failed to read all of block group bitmap " << i;
-            goto errout;
-        }
-
-        offset = (u64)info.block_size * first_block;
-
-        data->count = 0;
-
-        for (block = 0; block < block_count; block++) {
-            int used;
-
-            if (aux_info.bg_desc[i].bg_flags & EXT4_BG_BLOCK_UNINIT) {
-                // In block groups with an uninitialized block bitmap, we only
-                // need to encrypt the backup superblock (if one is present).
-                used = (ext4_bg_has_super_block(i) && block < 1 + aux_info.bg_desc_blocks);
-            } else {
-                used = bitmap_get_bit(block_bitmap, block);
-            }
-
-            update_progress(data, used);
-            if (used) {
-                if (data->count == 0) {
-                    data->offset = offset;
-                }
-                data->count++;
-            } else {
-                if (flush_outstanding_data(data)) {
-                    goto errout;
-                }
-            }
-
-            offset += info.block_size;
-
-            /* Write data if we are aligned or buffer size reached */
-            if (offset % (info.block_size * BLOCKS_AT_A_TIME) == 0 ||
-                data->count == BLOCKS_AT_A_TIME) {
-                if (flush_outstanding_data(data)) {
-                    goto errout;
-                }
-            }
-        }
-        if (flush_outstanding_data(data)) {
-            goto errout;
-        }
-    }
-
-    data->completed = 1;
-    rc = 0;
-
-errout:
-    log_progress(0, true);
-    free(data->buffer);
-    free(block_bitmap);
-    return rc;
+    InitFs("f2fs", get_num_blocks_used(fs_info.get()), fs_info->total_blocks, fs_info->block_size);
+    if (run_on_used_blocks(0, fs_info.get(), encrypt_f2fs_block, this) != 0) return kFailed;
+    return kSuccess;
 }
 
-static int cryptfs_enable_inplace_ext4(const char* crypto_blkdev, const char* real_blkdev,
-                                       off64_t size, off64_t* size_already_done, off64_t tot_size,
-                                       off64_t previously_encrypted_upto,
-                                       bool set_progress_properties) {
-    u32 i;
-    struct encryptGroupsData data;
-    int rc;  // Can't initialize without causing warning -Wclobbered
-    int retries = RETRY_MOUNT_ATTEMPTS;
+bool InPlaceEncrypter::DoEncryptInPlace() {
+    EncryptInPlaceError rc;
+
+    rc = EncryptInPlaceExt4();
+    if (rc != kFilesystemNotFound) return rc == kSuccess;
+
+    rc = EncryptInPlaceF2fs();
+    if (rc != kFilesystemNotFound) return rc == kSuccess;
+
+    LOG(WARNING) << "No recognized filesystem found on " << real_blkdev_
+                 << ".  Falling back to encrypting the full block device.";
+    InitFs("", nr_sec_, nr_sec_, 512);
+    for (uint64_t i = 0; i < nr_sec_; i++) {
+        if (!ProcessUsedBlock(i)) return false;
+    }
+    return true;
+}
+
+bool InPlaceEncrypter::EncryptInPlace(const std::string& crypto_blkdev,
+                                      const std::string& real_blkdev, uint64_t nr_sec,
+                                      bool set_progress_properties) {
     struct timespec time_started = {0};
 
-    if (previously_encrypted_upto > *size_already_done) {
-        LOG(DEBUG) << "Not fast encrypting since resuming part way through";
-        return -1;
-    }
+    real_blkdev_ = real_blkdev;
+    crypto_blkdev_ = crypto_blkdev;
+    nr_sec_ = nr_sec;
+    set_progress_properties_ = set_progress_properties;
 
-    memset(&data, 0, sizeof(data));
-    data.real_blkdev = real_blkdev;
-    data.crypto_blkdev = crypto_blkdev;
-    data.set_progress_properties = set_progress_properties;
-
-    LOG(DEBUG) << "Opening" << real_blkdev;
-    if ((data.realfd = open(real_blkdev, O_RDWR | O_CLOEXEC)) < 0) {
+    realfd_.reset(open64(real_blkdev.c_str(), O_RDONLY | O_CLOEXEC));
+    if (realfd_ < 0) {
         PLOG(ERROR) << "Error opening real_blkdev " << real_blkdev << " for inplace encrypt";
-        rc = -1;
-        goto errout;
+        return false;
     }
 
-    LOG(DEBUG) << "Opening" << crypto_blkdev;
-    // Wait until the block device appears.  Re-use the mount retry values since it is reasonable.
-    while ((data.cryptofd = open(crypto_blkdev, O_WRONLY | O_CLOEXEC)) < 0) {
-        if (--retries) {
-            PLOG(ERROR) << "Error opening crypto_blkdev " << crypto_blkdev
-                        << " for ext4 inplace encrypt, retrying";
-            sleep(RETRY_MOUNT_DELAY_SECONDS);
-        } else {
-            PLOG(ERROR) << "Error opening crypto_blkdev " << crypto_blkdev
-                        << " for ext4 inplace encrypt";
-            rc = ENABLE_INPLACE_ERR_DEV;
-            goto errout;
-        }
-    }
-
-    if (setjmp(setjmp_env)) {  // NOLINT
-        LOG(ERROR) << "Reading ext4 extent caused an exception";
-        rc = -1;
-        goto errout;
-    }
-
-    if (read_ext(data.realfd, 0) != 0) {
-        LOG(ERROR) << "Failed to read ext4 extent";
-        rc = -1;
-        goto errout;
-    }
-
-    data.numblocks = size / CRYPT_SECTORS_PER_BUFSIZE;
-    data.tot_numblocks = tot_size / CRYPT_SECTORS_PER_BUFSIZE;
-    data.blocks_already_done = *size_already_done / CRYPT_SECTORS_PER_BUFSIZE;
-
-    LOG(INFO) << "Encrypting ext4 filesystem in place...";
-
-    data.tot_used_blocks = data.numblocks;
-    for (i = 0; i < aux_info.groups; ++i) {
-        data.tot_used_blocks -= aux_info.bg_desc[i].bg_free_blocks_count;
-    }
-
-    data.one_pct = data.tot_used_blocks / 100;
-    data.cur_pct = 0;
-
-    if (clock_gettime(CLOCK_MONOTONIC, &time_started)) {
-        LOG(WARNING) << "Error getting time at start";
-        // Note - continue anyway - we'll run with 0
-    }
-    data.time_started = time_started.tv_sec;
-    data.remaining_time = -1;
-
-    rc = encrypt_groups(&data);
-    if (rc) {
-        LOG(ERROR) << "Error encrypting groups";
-        goto errout;
-    }
-
-    *size_already_done += data.completed ? size : data.last_written_sector;
-    rc = 0;
-
-errout:
-    close(data.realfd);
-    close(data.cryptofd);
-
-    return rc;
-}
-
-static void log_progress_f2fs(u64 block, bool completed) {
-    // Precondition - if completed data = 0 else data != 0
-
-    // Track progress so we can skip logging blocks
-    static u64 last_block = (u64)-1;
-
-    // Need to close existing 'Encrypting from' log?
-    if (completed || (last_block != (u64)-1 && block != last_block + 1)) {
-        LOG(INFO) << "Encrypted to block " << last_block;
-        last_block = -1;
-    }
-
-    // Need to start new 'Encrypting from' log?
-    if (!completed && (last_block == (u64)-1 || block != last_block + 1)) {
-        LOG(INFO) << "Encrypting from block " << block;
-    }
-
-    // Update offset
-    if (!completed) {
-        last_block = block;
-    }
-}
-
-static int encrypt_one_block_f2fs(u64 pos, void* data) {
-    struct encryptGroupsData* priv_dat = (struct encryptGroupsData*)data;
-
-    priv_dat->blocks_already_done = pos - 1;
-    update_progress(priv_dat, 1);
-
-    off64_t offset = pos * CRYPT_INPLACE_BUFSIZE;
-
-    if (pread64(priv_dat->realfd, priv_dat->buffer, CRYPT_INPLACE_BUFSIZE, offset) <= 0) {
-        LOG(ERROR) << "Error reading real_blkdev " << priv_dat->crypto_blkdev
-                   << " for f2fs inplace encrypt";
-        return -1;
-    }
-
-    if (pwrite64(priv_dat->cryptofd, priv_dat->buffer, CRYPT_INPLACE_BUFSIZE, offset) <= 0) {
-        LOG(ERROR) << "Error writing crypto_blkdev " << priv_dat->crypto_blkdev
-                   << " for f2fs inplace encrypt";
-        return -1;
-    } else {
-        log_progress_f2fs(pos, false);
-    }
-
-    return 0;
-}
-
-static int cryptfs_enable_inplace_f2fs(const char* crypto_blkdev, const char* real_blkdev,
-                                       off64_t size, off64_t* size_already_done, off64_t tot_size,
-                                       off64_t previously_encrypted_upto,
-                                       bool set_progress_properties) {
-    struct encryptGroupsData data;
-    struct f2fs_info* f2fs_info = NULL;
-    int rc = ENABLE_INPLACE_ERR_OTHER;
-    struct timespec time_started = {0};
-
-    if (previously_encrypted_upto > *size_already_done) {
-        LOG(DEBUG) << "Not fast encrypting since resuming part way through";
-        return ENABLE_INPLACE_ERR_OTHER;
-    }
-    memset(&data, 0, sizeof(data));
-    data.real_blkdev = real_blkdev;
-    data.crypto_blkdev = crypto_blkdev;
-    data.set_progress_properties = set_progress_properties;
-    data.realfd = -1;
-    data.cryptofd = -1;
-    if ((data.realfd = open64(real_blkdev, O_RDWR | O_CLOEXEC)) < 0) {
-        PLOG(ERROR) << "Error opening real_blkdev " << real_blkdev << " for f2fs inplace encrypt";
-        goto errout;
-    }
-    if ((data.cryptofd = open64(crypto_blkdev, O_WRONLY | O_CLOEXEC)) < 0) {
-        PLOG(ERROR) << "Error opening crypto_blkdev " << crypto_blkdev
-                    << " for f2fs inplace encrypt";
-        rc = ENABLE_INPLACE_ERR_DEV;
-        goto errout;
-    }
-
-    f2fs_info = generate_f2fs_info(data.realfd);
-    if (!f2fs_info) goto errout;
-
-    data.numblocks = size / CRYPT_SECTORS_PER_BUFSIZE;
-    data.tot_numblocks = tot_size / CRYPT_SECTORS_PER_BUFSIZE;
-    data.blocks_already_done = *size_already_done / CRYPT_SECTORS_PER_BUFSIZE;
-
-    data.tot_used_blocks = get_num_blocks_used(f2fs_info);
-
-    data.one_pct = data.tot_used_blocks / 100;
-    data.cur_pct = 0;
-    if (clock_gettime(CLOCK_MONOTONIC, &time_started)) {
-        LOG(WARNING) << "Error getting time at start";
-        // Note - continue anyway - we'll run with 0
-    }
-    data.time_started = time_started.tv_sec;
-    data.remaining_time = -1;
-
-
-    data.buffer = (char*)malloc(f2fs_info->block_size);
-    if (!data.buffer) {
-        LOG(ERROR) << "Failed to allocate crypto buffer";
-        goto errout;
-    }
-
-    data.count = 0;
-
-    /* Currently, this either runs to completion, or hits a nonrecoverable error */
-    rc = run_on_used_blocks(data.blocks_already_done, f2fs_info, &encrypt_one_block_f2fs, &data);
-
-    if (rc) {
-        LOG(ERROR) << "Error in running over f2fs blocks";
-        rc = ENABLE_INPLACE_ERR_OTHER;
-        goto errout;
-    }
-
-    *size_already_done += size;
-    rc = 0;
-
-errout:
-    if (rc) LOG(ERROR) << "Failed to encrypt f2fs filesystem on " << real_blkdev;
-
-    log_progress_f2fs(0, true);
-    free(f2fs_info);
-    free(data.buffer);
-    close(data.realfd);
-    close(data.cryptofd);
-
-    return rc;
-}
-
-static int cryptfs_enable_inplace_full(const char* crypto_blkdev, const char* real_blkdev,
-                                       off64_t size, off64_t* size_already_done, off64_t tot_size,
-                                       off64_t previously_encrypted_upto,
-                                       bool set_progress_properties) {
-    int realfd, cryptofd;
-    char* buf[CRYPT_INPLACE_BUFSIZE];
-    int rc = ENABLE_INPLACE_ERR_OTHER;
-    off64_t numblocks, i, remainder;
-    off64_t one_pct, cur_pct, new_pct;
-    off64_t blocks_already_done, tot_numblocks;
-
-    if ((realfd = open(real_blkdev, O_RDONLY | O_CLOEXEC)) < 0) {
-        PLOG(ERROR) << "Error opening real_blkdev " << real_blkdev << " for inplace encrypt";
-        return ENABLE_INPLACE_ERR_OTHER;
-    }
-
-    if ((cryptofd = open(crypto_blkdev, O_WRONLY | O_CLOEXEC)) < 0) {
+    cryptofd_.reset(open64(crypto_blkdev.c_str(), O_WRONLY | O_CLOEXEC));
+    if (cryptofd_ < 0) {
         PLOG(ERROR) << "Error opening crypto_blkdev " << crypto_blkdev << " for inplace encrypt";
-        close(realfd);
-        return ENABLE_INPLACE_ERR_DEV;
+        return false;
     }
 
-    /* This is pretty much a simple loop of reading 4K, and writing 4K.
-     * The size passed in is the number of 512 byte sectors in the filesystem.
-     * So compute the number of whole 4K blocks we should read/write,
-     * and the remainder.
-     */
-    numblocks = size / CRYPT_SECTORS_PER_BUFSIZE;
-    remainder = size % CRYPT_SECTORS_PER_BUFSIZE;
-    tot_numblocks = tot_size / CRYPT_SECTORS_PER_BUFSIZE;
-    blocks_already_done = *size_already_done / CRYPT_SECTORS_PER_BUFSIZE;
+    if (clock_gettime(CLOCK_MONOTONIC, &time_started)) {
+        PLOG(WARNING) << "Error getting time at start of in-place encryption";
+        // Note - continue anyway - we'll run with 0
+    }
+    time_started_ = time_started.tv_sec;
+    remaining_time_ = -1;
 
-    LOG(ERROR) << "Encrypting filesystem in place...";
+    bool success = DoEncryptInPlace();
 
-    i = previously_encrypted_upto + 1 - *size_already_done;
+    if (success) success &= EncryptPendingData();
 
-    if (lseek64(realfd, i * CRYPT_SECTOR_SIZE, SEEK_SET) < 0) {
-        PLOG(ERROR) << "Cannot seek to previously encrypted point on " << real_blkdev;
-        goto errout;
+    if (success && fsync(cryptofd_) != 0) {
+        PLOG(ERROR) << "Error syncing " << crypto_blkdev_;
+        success = false;
     }
 
-    if (lseek64(cryptofd, i * CRYPT_SECTOR_SIZE, SEEK_SET) < 0) {
-        PLOG(ERROR) << "Cannot seek to previously encrypted point on " << crypto_blkdev;
-        goto errout;
+    if (!success) {
+        LOG(ERROR) << "In-place encryption of " << DescribeFilesystem() << " failed";
+        return false;
     }
-
-    for (; i < size && i % CRYPT_SECTORS_PER_BUFSIZE != 0; ++i) {
-        if (unix_read(realfd, buf, CRYPT_SECTOR_SIZE) <= 0) {
-            PLOG(ERROR) << "Error reading initial sectors from real_blkdev " << real_blkdev
-                        << " for inplace encrypt";
-            goto errout;
-        }
-        if (unix_write(cryptofd, buf, CRYPT_SECTOR_SIZE) <= 0) {
-            PLOG(ERROR) << "Error writing initial sectors to crypto_blkdev " << crypto_blkdev
-                        << " for inplace encrypt";
-            goto errout;
-        } else {
-            LOG(INFO) << "Encrypted 1 block at " << i;
-        }
+    if (blocks_done_ != blocks_to_encrypt_) {
+        LOG(WARNING) << "blocks_to_encrypt (" << blocks_to_encrypt_
+                     << ") was incorrect; we actually encrypted " << blocks_done_
+                     << " blocks.  Encryption progress was inaccurate";
     }
-
-    one_pct = tot_numblocks / 100;
-    cur_pct = 0;
-    /* process the majority of the filesystem in blocks */
-    for (i /= CRYPT_SECTORS_PER_BUFSIZE; i < numblocks; i++) {
-        new_pct = (i + blocks_already_done) / one_pct;
-        if (set_progress_properties && new_pct > cur_pct) {
-            char property_buf[8];
-
-            cur_pct = new_pct;
-            snprintf(property_buf, sizeof(property_buf), "%" PRId64, cur_pct);
-            android::base::SetProperty("vold.encrypt_progress", property_buf);
-        }
-        if (unix_read(realfd, buf, CRYPT_INPLACE_BUFSIZE) <= 0) {
-            PLOG(ERROR) << "Error reading real_blkdev " << real_blkdev << " for inplace encrypt";
-            goto errout;
-        }
-        if (unix_write(cryptofd, buf, CRYPT_INPLACE_BUFSIZE) <= 0) {
-            PLOG(ERROR) << "Error writing crypto_blkdev " << crypto_blkdev << " for inplace encrypt";
-            goto errout;
-        } else {
-            LOG(DEBUG) << "Encrypted " << CRYPT_SECTORS_PER_BUFSIZE << " block at "
-                       << i * CRYPT_SECTORS_PER_BUFSIZE;
-        }
-    }
-
-    /* Do any remaining sectors */
-    for (i = 0; i < remainder; i++) {
-        if (unix_read(realfd, buf, CRYPT_SECTOR_SIZE) <= 0) {
-            LOG(ERROR) << "Error reading final sectors from real_blkdev " << real_blkdev
-                       << " for inplace encrypt";
-            goto errout;
-        }
-        if (unix_write(cryptofd, buf, CRYPT_SECTOR_SIZE) <= 0) {
-            LOG(ERROR) << "Error writing final sectors to crypto_blkdev " << crypto_blkdev
-                       << " for inplace encrypt";
-            goto errout;
-        } else {
-            LOG(INFO) << "Encrypted 1 block at next location";
-        }
-    }
-
-    *size_already_done += size;
-    rc = 0;
-
-errout:
-    close(realfd);
-    close(cryptofd);
-
-    return rc;
+    // Make sure vold.encrypt_progress gets set to 100.
+    UpdateProgress(0, true);
+    LOG(INFO) << "Successfully encrypted " << DescribeFilesystem();
+    return true;
 }
 
-/* returns on of the ENABLE_INPLACE_* return codes */
-int cryptfs_enable_inplace(const char* crypto_blkdev, const char* real_blkdev, off64_t size,
-                           off64_t* size_already_done, off64_t tot_size,
-                           off64_t previously_encrypted_upto, bool set_progress_properties) {
-    int rc_ext4, rc_f2fs, rc_full;
-    LOG(DEBUG) << "cryptfs_enable_inplace(" << crypto_blkdev << ", " << real_blkdev << ", " << size
-               << ", " << size_already_done << ", " << tot_size << ", " << previously_encrypted_upto
-               << ", " << set_progress_properties << ")";
-    if (previously_encrypted_upto) {
-        LOG(DEBUG) << "Continuing encryption from " << previously_encrypted_upto;
-    }
+// Encrypts |real_blkdev| in-place by reading the data from |real_blkdev| and
+// writing it to |crypto_blkdev|, which should be a dm-crypt or dm-default-key
+// device backed by |real_blkdev|.  The size to encrypt is |nr_sec| 512-byte
+// sectors; however, if a filesystem is detected, then its size will be used
+// instead, and only the in-use blocks of the filesystem will be encrypted.
+bool encrypt_inplace(const std::string& crypto_blkdev, const std::string& real_blkdev,
+                     uint64_t nr_sec, bool set_progress_properties) {
+    LOG(DEBUG) << "encrypt_inplace(" << crypto_blkdev << ", " << real_blkdev << ", " << nr_sec
+               << ", " << (set_progress_properties ? "true" : "false") << ")";
 
-    if (*size_already_done + size < previously_encrypted_upto) {
-        LOG(DEBUG) << "cryptfs_enable_inplace already done";
-        *size_already_done += size;
-        return 0;
-    }
-
-    /* TODO: identify filesystem type.
-     * As is, cryptfs_enable_inplace_ext4 will fail on an f2fs partition, and
-     * then we will drop down to cryptfs_enable_inplace_f2fs.
-     * */
-    if ((rc_ext4 = cryptfs_enable_inplace_ext4(crypto_blkdev, real_blkdev, size, size_already_done,
-                                               tot_size, previously_encrypted_upto,
-                                               set_progress_properties)) == 0) {
-        LOG(DEBUG) << "cryptfs_enable_inplace_ext4 success";
-        return 0;
-    }
-    LOG(DEBUG) << "cryptfs_enable_inplace_ext4()=" << rc_ext4;
-
-    if ((rc_f2fs = cryptfs_enable_inplace_f2fs(crypto_blkdev, real_blkdev, size, size_already_done,
-                                               tot_size, previously_encrypted_upto,
-                                               set_progress_properties)) == 0) {
-        LOG(DEBUG) << "cryptfs_enable_inplace_f2fs success";
-        return 0;
-    }
-    LOG(DEBUG) << "cryptfs_enable_inplace_f2fs()=" << rc_f2fs;
-
-    rc_full =
-        cryptfs_enable_inplace_full(crypto_blkdev, real_blkdev, size, size_already_done, tot_size,
-                                    previously_encrypted_upto, set_progress_properties);
-    LOG(DEBUG) << "cryptfs_enable_inplace_full()=" << rc_full;
-
-    /* Hack for b/17898962, the following is the symptom... */
-    if (rc_ext4 == ENABLE_INPLACE_ERR_DEV && rc_f2fs == ENABLE_INPLACE_ERR_DEV &&
-        rc_full == ENABLE_INPLACE_ERR_DEV) {
-        LOG(DEBUG) << "ENABLE_INPLACE_ERR_DEV";
-        return ENABLE_INPLACE_ERR_DEV;
-    }
-    return rc_full;
+    InPlaceEncrypter encrypter;
+    return encrypter.EncryptInPlace(crypto_blkdev, real_blkdev, nr_sec, set_progress_properties);
 }
diff --git a/EncryptInplace.h b/EncryptInplace.h
index a2b46cf..480a47c 100644
--- a/EncryptInplace.h
+++ b/EncryptInplace.h
@@ -17,20 +17,10 @@
 #ifndef _ENCRYPT_INPLACE_H
 #define _ENCRYPT_INPLACE_H
 
-#include <sys/types.h>
+#include <stdint.h>
+#include <string>
 
-#define CRYPT_INPLACE_BUFSIZE 4096
-#define CRYPT_SECTOR_SIZE 512
-#define RETRY_MOUNT_ATTEMPTS 10
-#define RETRY_MOUNT_DELAY_SECONDS 1
-
-/* Return values for cryptfs_enable_inplace() */
-#define ENABLE_INPLACE_OK 0
-#define ENABLE_INPLACE_ERR_OTHER (-1)
-#define ENABLE_INPLACE_ERR_DEV (-2) /* crypto_blkdev issue */
-
-int cryptfs_enable_inplace(const char* crypto_blkdev, const char* real_blkdev, off64_t size,
-                           off64_t* size_already_done, off64_t tot_size,
-                           off64_t previously_encrypted_upto, bool set_progress_properties);
+bool encrypt_inplace(const std::string& crypto_blkdev, const std::string& real_blkdev,
+                     uint64_t nr_sec, bool set_progress_properties);
 
 #endif
diff --git a/FsCrypt.cpp b/FsCrypt.cpp
index 9bdade5..ff8c1f4 100644
--- a/FsCrypt.cpp
+++ b/FsCrypt.cpp
@@ -67,6 +67,7 @@
 using android::base::StringPrintf;
 using android::fs_mgr::GetEntryForMountPoint;
 using android::vold::BuildDataPath;
+using android::vold::IsDotOrDotDot;
 using android::vold::IsFilesystemSupported;
 using android::vold::kEmptyAuthentication;
 using android::vold::KeyBuffer;
@@ -140,6 +141,7 @@
             }
             break;
         }
+        if (IsDotOrDotDot(*entry)) continue;
         if (entry->d_type != DT_DIR || entry->d_name[0] != 'c') {
             LOG(DEBUG) << "Skipping non-key " << entry->d_name;
             continue;
@@ -391,6 +393,7 @@
             }
             break;
         }
+        if (IsDotOrDotDot(*entry)) continue;
         if (entry->d_type != DT_DIR || !is_numeric(entry->d_name)) {
             LOG(DEBUG) << "Skipping non-de-key " << entry->d_name;
             continue;
@@ -791,6 +794,11 @@
 
 static bool prepare_subdirs(const std::string& action, const std::string& volume_uuid,
                             userid_t user_id, int flags) {
+    // TODO(b/141677108): Remove this & make it the default behavior
+    if (android::base::GetProperty("ro.vold.level_from_user", "0") == "1") {
+        flags |= android::os::IVold::STORAGE_FLAG_LEVEL_FROM_USER;
+    }
+
     if (0 != android::vold::ForkExecvp(
                  std::vector<std::string>{prepare_subdirs_path, action, volume_uuid,
                                           std::to_string(user_id), std::to_string(flags)})) {
@@ -968,6 +976,7 @@
             }
             break;
         }
+        if (IsDotOrDotDot(*entry)) continue;
         if (entry->d_type != DT_DIR || entry->d_name[0] == '.') {
             LOG(DEBUG) << "Skipping non-user " << entry->d_name;
             continue;
diff --git a/IdleMaint.cpp b/IdleMaint.cpp
index 2b5a8f1..e4a1806 100644
--- a/IdleMaint.cpp
+++ b/IdleMaint.cpp
@@ -17,6 +17,7 @@
 #include "IdleMaint.h"
 #include "FileDeviceUtils.h"
 #include "Utils.h"
+#include "VoldUtil.h"
 #include "VolumeManager.h"
 #include "model/PrivateVolume.h"
 
@@ -45,8 +46,6 @@
 using android::base::StringPrintf;
 using android::base::Timer;
 using android::base::WriteStringToFile;
-using android::fs_mgr::Fstab;
-using android::fs_mgr::ReadDefaultFstab;
 using android::hardware::Return;
 using android::hardware::Void;
 using android::hardware::health::storage::V1_0::IStorage;
@@ -104,17 +103,18 @@
 }
 
 static void addFromFstab(std::list<std::string>* paths, PathTypes path_type) {
-    Fstab fstab;
-    ReadDefaultFstab(&fstab);
-
     std::string previous_mount_point;
-    for (const auto& entry : fstab) {
-        // Skip raw partitions.
-        if (entry.fs_type == "emmc" || entry.fs_type == "mtd") {
+    for (const auto& entry : fstab_default) {
+        // Skip raw partitions and swap space.
+        if (entry.fs_type == "emmc" || entry.fs_type == "mtd" || entry.fs_type == "swap") {
             continue;
         }
-        // Skip read-only filesystems
-        if (entry.flags & MS_RDONLY) {
+        // Skip read-only filesystems and bind mounts.
+        if (entry.flags & (MS_RDONLY | MS_BIND)) {
+            continue;
+        }
+        // Skip anything without an underlying block device, e.g. virtiofs.
+        if (entry.blk_device[0] != '/') {
             continue;
         }
         if (entry.fs_mgr_flags.vold_managed) {
@@ -253,11 +253,8 @@
 }
 
 static void runDevGcFstab(void) {
-    Fstab fstab;
-    ReadDefaultFstab(&fstab);
-
     std::string path;
-    for (const auto& entry : fstab) {
+    for (const auto& entry : fstab_default) {
         if (!entry.sysfs_path.empty()) {
             path = entry.sysfs_path;
             break;
diff --git a/KeyStorage.cpp b/KeyStorage.cpp
index 951536b..8147827 100644
--- a/KeyStorage.cpp
+++ b/KeyStorage.cpp
@@ -21,6 +21,7 @@
 #include "ScryptParameters.h"
 #include "Utils.h"
 
+#include <algorithm>
 #include <thread>
 #include <vector>
 
@@ -141,9 +142,12 @@
     if (!keymaster) return false;
     std::string key_temp;
     auto paramBuilder = km::AuthorizationSetBuilder().AesEncryptionKey(AES_KEY_BYTES * 8);
-    paramBuilder.Authorization(km::TAG_ROLLBACK_RESISTANCE);
     paramBuilder.Authorization(km::TAG_STORAGE_KEY);
-    if (!keymaster.generateKey(paramBuilder, &key_temp)) return false;
+    auto paramsWithRollback = paramBuilder;
+    paramsWithRollback.Authorization(km::TAG_ROLLBACK_RESISTANCE);
+    if (!keymaster.generateKey(paramsWithRollback, &key_temp)) {
+        if (!keymaster.generateKey(paramBuilder, &key_temp)) return false;
+    }
     *key = KeyBuffer(key_temp.size());
     memcpy(reinterpret_cast<void*>(key->data()), key_temp.c_str(), key->size());
     return true;
@@ -205,75 +209,161 @@
     return true;
 }
 
-static void deferedKmDeleteKey(const std::string& kmkey) {
-    while (!android::base::WaitForProperty("vold.checkpoint_committed", "1")) {
-        LOG(ERROR) << "Wait for boot timed out";
+static std::mutex key_upgrade_lock;
+
+// List of key directories that have had their Keymaster key upgraded during
+// this boot and written to "keymaster_key_blob_upgraded", but replacing the old
+// key was delayed due to an active checkpoint.  Protected by key_upgrade_lock.
+static std::vector<std::string> key_dirs_to_commit;
+
+// Replaces |dir|/keymaster_key_blob with |dir|/keymaster_key_blob_upgraded and
+// deletes the old key from Keymaster.
+static bool CommitUpgradedKey(Keymaster& keymaster, const std::string& dir) {
+    auto blob_file = dir + "/" + kFn_keymaster_key_blob;
+    auto upgraded_blob_file = dir + "/" + kFn_keymaster_key_blob_upgraded;
+
+    std::string blob;
+    if (!readFileToString(blob_file, &blob)) return false;
+
+    if (rename(upgraded_blob_file.c_str(), blob_file.c_str()) != 0) {
+        PLOG(ERROR) << "Failed to rename " << upgraded_blob_file << " to " << blob_file;
+        return false;
     }
+    // Ensure that the rename is persisted before deleting the Keymaster key.
+    if (!FsyncDirectory(dir)) return false;
+
+    if (!keymaster || !keymaster.deleteKey(blob)) {
+        LOG(WARNING) << "Failed to delete old key " << blob_file
+                     << " from Keymaster; continuing anyway";
+        // Continue on, but the space in Keymaster used by the old key won't be freed.
+    }
+    return true;
+}
+
+static void DeferredCommitKeys() {
+    android::base::WaitForProperty("vold.checkpoint_committed", "1");
+    LOG(INFO) << "Committing upgraded keys";
     Keymaster keymaster;
-    if (!keymaster || !keymaster.deleteKey(kmkey)) {
-        LOG(ERROR) << "Defered Key deletion failed during upgrade";
+    if (!keymaster) {
+        LOG(ERROR) << "Failed to open Keymaster; old keys won't be deleted from Keymaster";
+        // Continue on, but the space in Keymaster used by the old keys won't be freed.
+    }
+    std::lock_guard<std::mutex> lock(key_upgrade_lock);
+    for (auto& dir : key_dirs_to_commit) {
+        LOG(INFO) << "Committing upgraded key " << dir;
+        CommitUpgradedKey(keymaster, dir);
+    }
+    key_dirs_to_commit.clear();
+}
+
+// Returns true if the Keymaster key in |dir| has already been upgraded and is
+// pending being committed.  Assumes that key_upgrade_lock is held.
+static bool IsKeyCommitPending(const std::string& dir) {
+    for (const auto& dir_to_commit : key_dirs_to_commit) {
+        if (IsSameFile(dir, dir_to_commit)) return true;
+    }
+    return false;
+}
+
+// Schedules the upgraded Keymaster key in |dir| to be committed later.
+// Assumes that key_upgrade_lock is held.
+static void ScheduleKeyCommit(const std::string& dir) {
+    if (key_dirs_to_commit.empty()) std::thread(DeferredCommitKeys).detach();
+    key_dirs_to_commit.push_back(dir);
+}
+
+static void CancelPendingKeyCommit(const std::string& dir) {
+    std::lock_guard<std::mutex> lock(key_upgrade_lock);
+    for (auto it = key_dirs_to_commit.begin(); it != key_dirs_to_commit.end(); it++) {
+        if (IsSameFile(*it, dir)) {
+            LOG(DEBUG) << "Cancelling pending commit of upgraded key " << dir
+                       << " because it is being destroyed";
+            key_dirs_to_commit.erase(it);
+            break;
+        }
     }
 }
 
-bool kmDeleteKey(Keymaster& keymaster, const std::string& kmKey) {
-    bool needs_cp = cp_needsCheckpoint();
-
-    if (needs_cp) {
-        std::thread(deferedKmDeleteKey, kmKey).detach();
-        LOG(INFO) << "Deferring Key deletion during upgrade";
-        return true;
-    } else {
-        return keymaster.deleteKey(kmKey);
+// Deletes a leftover upgraded key, if present.  An upgraded key can be left
+// over if an update failed, or if we rebooted before committing the key in a
+// freak accident.  Either way, we can re-upgrade the key if we need to.
+static void DeleteUpgradedKey(Keymaster& keymaster, const std::string& path) {
+    if (pathExists(path)) {
+        LOG(DEBUG) << "Deleting leftover upgraded key " << path;
+        std::string blob;
+        if (!android::base::ReadFileToString(path, &blob)) {
+            LOG(WARNING) << "Failed to read leftover upgraded key " << path
+                         << "; continuing anyway";
+        } else if (!keymaster.deleteKey(blob)) {
+            LOG(WARNING) << "Failed to delete leftover upgraded key " << path
+                         << " from Keymaster; continuing anyway";
+        }
+        if (unlink(path.c_str()) != 0) {
+            LOG(WARNING) << "Failed to unlink leftover upgraded key " << path
+                         << "; continuing anyway";
+        }
     }
 }
 
-static KeymasterOperation begin(Keymaster& keymaster, const std::string& dir,
-                                km::KeyPurpose purpose, const km::AuthorizationSet& keyParams,
-                                const km::AuthorizationSet& opParams,
-                                const km::HardwareAuthToken& authToken,
-                                km::AuthorizationSet* outParams, bool keepOld) {
-    auto kmKeyPath = dir + "/" + kFn_keymaster_key_blob;
-    std::string kmKey;
-    if (!readFileToString(kmKeyPath, &kmKey)) return KeymasterOperation();
+// Begins a Keymaster operation using the key stored in |dir|.
+static KeymasterOperation BeginKeymasterOp(Keymaster& keymaster, const std::string& dir,
+                                           km::KeyPurpose purpose,
+                                           const km::AuthorizationSet& keyParams,
+                                           const km::AuthorizationSet& opParams,
+                                           const km::HardwareAuthToken& authToken,
+                                           km::AuthorizationSet* outParams) {
     km::AuthorizationSet inParams(keyParams);
     inParams.append(opParams.begin(), opParams.end());
-    for (;;) {
-        auto opHandle = keymaster.begin(purpose, kmKey, inParams, authToken, outParams);
-        if (opHandle) {
-            return opHandle;
-        }
-        if (opHandle.errorCode() != km::ErrorCode::KEY_REQUIRES_UPGRADE) return opHandle;
-        LOG(DEBUG) << "Upgrading key: " << dir;
-        std::string newKey;
-        if (!keymaster.upgradeKey(kmKey, keyParams, &newKey)) return KeymasterOperation();
-        auto newKeyPath = dir + "/" + kFn_keymaster_key_blob_upgraded;
-        if (!writeStringToFile(newKey, newKeyPath)) return KeymasterOperation();
-        if (!keepOld) {
-            if (rename(newKeyPath.c_str(), kmKeyPath.c_str()) != 0) {
-                PLOG(ERROR) << "Unable to move upgraded key to location: " << kmKeyPath;
-                return KeymasterOperation();
-            }
-            if (!android::vold::FsyncDirectory(dir)) {
-                LOG(ERROR) << "Key dir sync failed: " << dir;
-                return KeymasterOperation();
-            }
-            if (!kmDeleteKey(keymaster, kmKey)) {
-                LOG(ERROR) << "Key deletion failed during upgrade, continuing anyway: " << dir;
-            }
-        }
-        kmKey = newKey;
-        LOG(INFO) << "Key upgraded: " << dir;
+
+    auto blob_file = dir + "/" + kFn_keymaster_key_blob;
+    auto upgraded_blob_file = dir + "/" + kFn_keymaster_key_blob_upgraded;
+
+    std::lock_guard<std::mutex> lock(key_upgrade_lock);
+
+    std::string blob;
+    bool already_upgraded = IsKeyCommitPending(dir);
+    if (already_upgraded) {
+        LOG(DEBUG)
+                << blob_file
+                << " was already upgraded and is waiting to be committed; using the upgraded blob";
+        if (!readFileToString(upgraded_blob_file, &blob)) return KeymasterOperation();
+    } else {
+        DeleteUpgradedKey(keymaster, upgraded_blob_file);
+        if (!readFileToString(blob_file, &blob)) return KeymasterOperation();
     }
+
+    auto opHandle = keymaster.begin(purpose, blob, inParams, authToken, outParams);
+    if (opHandle) return opHandle;
+    if (opHandle.errorCode() != km::ErrorCode::KEY_REQUIRES_UPGRADE) return opHandle;
+
+    if (already_upgraded) {
+        LOG(ERROR) << "Unexpected case; already-upgraded key " << upgraded_blob_file
+                   << " still requires upgrade";
+        return KeymasterOperation();
+    }
+    LOG(INFO) << "Upgrading key: " << blob_file;
+    if (!keymaster.upgradeKey(blob, keyParams, &blob)) return KeymasterOperation();
+    if (!writeStringToFile(blob, upgraded_blob_file)) return KeymasterOperation();
+    if (cp_needsCheckpoint()) {
+        LOG(INFO) << "Wrote upgraded key to " << upgraded_blob_file
+                  << "; delaying commit due to checkpoint";
+        ScheduleKeyCommit(dir);
+    } else {
+        if (!CommitUpgradedKey(keymaster, dir)) return KeymasterOperation();
+        LOG(INFO) << "Key upgraded: " << blob_file;
+    }
+
+    return keymaster.begin(purpose, blob, inParams, authToken, outParams);
 }
 
 static bool encryptWithKeymasterKey(Keymaster& keymaster, const std::string& dir,
                                     const km::AuthorizationSet& keyParams,
-                                    const km::HardwareAuthToken& authToken, const KeyBuffer& message,
-                                    std::string* ciphertext, bool keepOld) {
+                                    const km::HardwareAuthToken& authToken,
+                                    const KeyBuffer& message, std::string* ciphertext) {
     km::AuthorizationSet opParams;
     km::AuthorizationSet outParams;
-    auto opHandle = begin(keymaster, dir, km::KeyPurpose::ENCRYPT, keyParams, opParams, authToken,
-                          &outParams, keepOld);
+    auto opHandle = BeginKeymasterOp(keymaster, dir, km::KeyPurpose::ENCRYPT, keyParams, opParams,
+                                     authToken, &outParams);
     if (!opHandle) return false;
     auto nonceBlob = outParams.GetTagValue(km::TAG_NONCE);
     if (!nonceBlob.isOk()) {
@@ -297,14 +387,13 @@
 static bool decryptWithKeymasterKey(Keymaster& keymaster, const std::string& dir,
                                     const km::AuthorizationSet& keyParams,
                                     const km::HardwareAuthToken& authToken,
-                                    const std::string& ciphertext, KeyBuffer* message,
-                                    bool keepOld) {
+                                    const std::string& ciphertext, KeyBuffer* message) {
     auto nonce = ciphertext.substr(0, GCM_NONCE_BYTES);
     auto bodyAndMac = ciphertext.substr(GCM_NONCE_BYTES);
     auto opParams = km::AuthorizationSetBuilder().Authorization(km::TAG_NONCE,
                                                                 km::support::blob2hidlVec(nonce));
-    auto opHandle = begin(keymaster, dir, km::KeyPurpose::DECRYPT, keyParams, opParams, authToken,
-                          nullptr, keepOld);
+    auto opHandle = BeginKeymasterOp(keymaster, dir, km::KeyPurpose::DECRYPT, keyParams, opParams,
+                                     authToken, nullptr);
     if (!opHandle) return false;
     if (!opHandle.updateCompletely(bodyAndMac, message)) return false;
     if (!opHandle.finish(nullptr)) return false;
@@ -510,8 +599,7 @@
         km::AuthorizationSet keyParams;
         km::HardwareAuthToken authToken;
         std::tie(keyParams, authToken) = beginParams(auth, appId);
-        if (!encryptWithKeymasterKey(keymaster, dir, keyParams, authToken, key, &encryptedKey,
-                                     false))
+        if (!encryptWithKeymasterKey(keymaster, dir, keyParams, authToken, key, &encryptedKey))
             return false;
     } else {
         if (!encryptWithoutKeymaster(appId, key, &encryptedKey)) return false;
@@ -540,8 +628,7 @@
     return true;
 }
 
-bool retrieveKey(const std::string& dir, const KeyAuthentication& auth, KeyBuffer* key,
-                 bool keepOld) {
+bool retrieveKey(const std::string& dir, const KeyAuthentication& auth, KeyBuffer* key) {
     std::string version;
     if (!readFileToString(dir + "/" + kFn_version, &version)) return false;
     if (version != kCurrentVersion) {
@@ -566,8 +653,7 @@
         km::AuthorizationSet keyParams;
         km::HardwareAuthToken authToken;
         std::tie(keyParams, authToken) = beginParams(auth, appId);
-        if (!decryptWithKeymasterKey(keymaster, dir, keyParams, authToken, encryptedMessage, key,
-                                     keepOld))
+        if (!decryptWithKeymasterKey(keymaster, dir, keyParams, authToken, encryptedMessage, key))
             return false;
     } else {
         if (!decryptWithoutKeymaster(appId, encryptedMessage, key)) return false;
@@ -575,12 +661,13 @@
     return true;
 }
 
-static bool deleteKey(const std::string& dir) {
-    std::string kmKey;
-    if (!readFileToString(dir + "/" + kFn_keymaster_key_blob, &kmKey)) return false;
+static bool DeleteKeymasterKey(const std::string& blob_file) {
+    std::string blob;
+    if (!readFileToString(blob_file, &blob)) return false;
     Keymaster keymaster;
     if (!keymaster) return false;
-    if (!keymaster.deleteKey(kmKey)) return false;
+    LOG(DEBUG) << "Deleting key " << blob_file << " from Keymaster";
+    if (!keymaster.deleteKey(blob)) return false;
     return true;
 }
 
@@ -602,19 +689,23 @@
 
 bool destroyKey(const std::string& dir) {
     bool success = true;
-    // Try each thing, even if previous things failed.
-    bool uses_km = pathExists(dir + "/" + kFn_keymaster_key_blob);
-    if (uses_km) {
-        success &= deleteKey(dir);
-    }
+
+    CancelPendingKeyCommit(dir);
+
     auto secdiscard_cmd = std::vector<std::string>{
         kSecdiscardPath,
         "--",
         dir + "/" + kFn_encrypted_key,
         dir + "/" + kFn_secdiscardable,
     };
-    if (uses_km) {
-        secdiscard_cmd.emplace_back(dir + "/" + kFn_keymaster_key_blob);
+    // Try each thing, even if previous things failed.
+
+    for (auto& fn : {kFn_keymaster_key_blob, kFn_keymaster_key_blob_upgraded}) {
+        auto blob_file = dir + "/" + fn;
+        if (pathExists(blob_file)) {
+            success &= DeleteKeymasterKey(blob_file);
+            secdiscard_cmd.push_back(blob_file);
+        }
     }
     if (ForkExecvp(secdiscard_cmd) != 0) {
         LOG(ERROR) << "secdiscard failed";
diff --git a/KeyStorage.h b/KeyStorage.h
index f9d3ec6..1eb26ae 100644
--- a/KeyStorage.h
+++ b/KeyStorage.h
@@ -61,8 +61,7 @@
                         const KeyAuthentication& auth, const KeyBuffer& key);
 
 // Retrieve the key from the named directory.
-bool retrieveKey(const std::string& dir, const KeyAuthentication& auth, KeyBuffer* key,
-                 bool keepOld = false);
+bool retrieveKey(const std::string& dir, const KeyAuthentication& auth, KeyBuffer* key);
 
 // Securely destroy the key stored in the named directory and delete the directory.
 bool destroyKey(const std::string& dir);
diff --git a/KeyUtil.cpp b/KeyUtil.cpp
index f3a2986..9d01e1e 100644
--- a/KeyUtil.cpp
+++ b/KeyUtil.cpp
@@ -392,10 +392,10 @@
 
 bool retrieveOrGenerateKey(const std::string& key_path, const std::string& tmp_path,
                            const KeyAuthentication& key_authentication, const KeyGeneration& gen,
-                           KeyBuffer* key, bool keepOld) {
+                           KeyBuffer* key) {
     if (pathExists(key_path)) {
         LOG(DEBUG) << "Key exists, using: " << key_path;
-        if (!retrieveKey(key_path, key_authentication, key, keepOld)) return false;
+        if (!retrieveKey(key_path, key_authentication, key)) return false;
     } else {
         if (!gen.allow_gen) {
             LOG(ERROR) << "No key found in " << key_path;
diff --git a/KeyUtil.h b/KeyUtil.h
index 23278c1..73255a3 100644
--- a/KeyUtil.h
+++ b/KeyUtil.h
@@ -74,9 +74,11 @@
 // responsible for dropping caches.
 bool evictKey(const std::string& mountpoint, const EncryptionPolicy& policy);
 
+// Retrieves the key from the named directory, or generates it if it doesn't
+// exist.
 bool retrieveOrGenerateKey(const std::string& key_path, const std::string& tmp_path,
                            const KeyAuthentication& key_authentication, const KeyGeneration& gen,
-                           KeyBuffer* key, bool keepOld = true);
+                           KeyBuffer* key);
 
 // Re-installs a file-based encryption key of fscrypt-provisioning type from the
 // global session keyring back into fs keyring of the mountpoint.
diff --git a/Loop.cpp b/Loop.cpp
index 9fa876c..87f105d 100644
--- a/Loop.cpp
+++ b/Loop.cpp
@@ -150,7 +150,9 @@
 
         struct loop_info64 li;
         if (ioctl(fd.get(), LOOP_GET_STATUS64, &li) < 0) {
-            PLOG(WARNING) << "Failed to LOOP_GET_STATUS64 " << path;
+            if (errno != ENXIO) {
+                PLOG(WARNING) << "Failed to LOOP_GET_STATUS64 " << path;
+            }
             continue;
         }
 
diff --git a/MetadataCrypt.cpp b/MetadataCrypt.cpp
index c61132c..fdee21f 100644
--- a/MetadataCrypt.cpp
+++ b/MetadataCrypt.cpp
@@ -17,17 +17,13 @@
 #include "MetadataCrypt.h"
 #include "KeyBuffer.h"
 
-#include <algorithm>
 #include <string>
-#include <thread>
-#include <vector>
 
 #include <fcntl.h>
 #include <sys/param.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 
-#include <android-base/file.h>
 #include <android-base/logging.h>
 #include <android-base/properties.h>
 #include <android-base/strings.h>
@@ -35,6 +31,7 @@
 #include <cutils/fs.h>
 #include <fs_mgr.h>
 #include <libdm/dm.h>
+#include <libgsi/libgsi.h>
 
 #include "Checkpoint.h"
 #include "CryptoType.h"
@@ -45,8 +42,6 @@
 #include "Utils.h"
 #include "VoldUtil.h"
 
-#define TABLE_LOAD_RETRIES 10
-
 namespace android {
 namespace vold {
 
@@ -65,9 +60,6 @@
 
 static const std::string kDmNameUserdata = "userdata";
 
-static const char* kFn_keymaster_key_blob = "keymaster_key_blob";
-static const char* kFn_keymaster_key_blob_upgraded = "keymaster_key_blob_upgraded";
-
 // The first entry in this table is the default crypto type.
 constexpr CryptoType supported_crypto_types[] = {aes_256_xts, adiantum};
 
@@ -112,31 +104,6 @@
     return true;
 }
 
-// Note: It is possible to orphan a key if it is removed before deleting
-// Update this once keymaster APIs change, and we have a proper commit.
-static void commit_key(const std::string& dir) {
-    while (!android::base::WaitForProperty("vold.checkpoint_committed", "1")) {
-        LOG(ERROR) << "Wait for boot timed out";
-    }
-    Keymaster keymaster;
-    auto keyPath = dir + "/" + kFn_keymaster_key_blob;
-    auto newKeyPath = dir + "/" + kFn_keymaster_key_blob_upgraded;
-    std::string key;
-
-    if (!android::base::ReadFileToString(keyPath, &key)) {
-        LOG(ERROR) << "Failed to read old key: " << dir;
-        return;
-    }
-    if (rename(newKeyPath.c_str(), keyPath.c_str()) != 0) {
-        PLOG(ERROR) << "Unable to move upgraded key to location: " << keyPath;
-        return;
-    }
-    if (!keymaster.deleteKey(key)) {
-        LOG(ERROR) << "Key deletion failed during upgrade, continuing anyway: " << dir;
-    }
-    LOG(INFO) << "Old Key deleted: " << dir;
-}
-
 static bool read_key(const std::string& metadata_key_dir, const KeyGeneration& gen,
                      KeyBuffer* key) {
     if (metadata_key_dir.empty()) {
@@ -151,25 +118,7 @@
         return false;
     }
     auto temp = metadata_key_dir + "/tmp";
-    auto newKeyPath = dir + "/" + kFn_keymaster_key_blob_upgraded;
-    /* If we have a leftover upgraded key, delete it.
-     * We either failed an update and must return to the old key,
-     * or we rebooted before commiting the keys in a freak accident.
-     * Either way, we can re-upgrade the key if we need to.
-     */
-    Keymaster keymaster;
-    if (pathExists(newKeyPath)) {
-        if (!android::base::ReadFileToString(newKeyPath, &sKey))
-            LOG(ERROR) << "Failed to read incomplete key: " << dir;
-        else if (!keymaster.deleteKey(sKey))
-            LOG(ERROR) << "Incomplete key deletion failed, continuing anyway: " << dir;
-        else
-            unlink(newKeyPath.c_str());
-    }
-    bool needs_cp = cp_needsCheckpoint();
-    if (!retrieveOrGenerateKey(dir, temp, kEmptyAuthentication, gen, key, needs_cp)) return false;
-    if (needs_cp && pathExists(newKeyPath)) std::thread(commit_key, dir).detach();
-    return true;
+    return retrieveOrGenerateKey(dir, temp, kEmptyAuthentication, gen, key);
 }
 
 static bool get_number_of_sectors(const std::string& real_blkdev, uint64_t* nr_sec) {
@@ -215,20 +164,8 @@
     table.AddTarget(std::move(target));
 
     auto& dm = DeviceMapper::Instance();
-    for (int i = 0;; i++) {
-        if (dm.CreateDevice(dm_name, table)) {
-            break;
-        }
-        if (i + 1 >= TABLE_LOAD_RETRIES) {
-            PLOG(ERROR) << "Could not create default-key device " << dm_name;
-            return false;
-        }
-        PLOG(INFO) << "Could not create default-key device, retrying";
-        usleep(500000);
-    }
-
-    if (!dm.GetDmDevicePathByName(dm_name, crypto_blkdev)) {
-        LOG(ERROR) << "Cannot retrieve default-key device status " << dm_name;
+    if (!dm.CreateDevice(dm_name, table, crypto_blkdev, std::chrono::seconds(5))) {
+        PLOG(ERROR) << "Could not create default-key device " << dm_name;
         return false;
     }
     return true;
@@ -318,21 +255,7 @@
         return false;
 
     // FIXME handle the corrupt case
-    if (needs_encrypt) {
-        LOG(INFO) << "Beginning inplace encryption, nr_sec: " << nr_sec;
-        off64_t size_already_done = 0;
-        auto rc = cryptfs_enable_inplace(crypto_blkdev.data(), blk_device.data(), nr_sec,
-                                         &size_already_done, nr_sec, 0, false);
-        if (rc != 0) {
-            LOG(ERROR) << "Inplace crypto failed with code: " << rc;
-            return false;
-        }
-        if (static_cast<uint64_t>(size_already_done) != nr_sec) {
-            LOG(ERROR) << "Inplace crypto only got up to sector: " << size_already_done;
-            return false;
-        }
-        LOG(INFO) << "Inplace encryption complete";
-    }
+    if (needs_encrypt && !encrypt_inplace(crypto_blkdev, blk_device, nr_sec, false)) return false;
 
     LOG(DEBUG) << "Mounting metadata-encrypted filesystem:" << mount_point;
     mount_via_fs_mgr(mount_point.c_str(), crypto_blkdev.c_str());
@@ -366,5 +289,44 @@
     return create_crypto_blk_dev(label, blk_device, key, options, out_crypto_blkdev, &nr_sec);
 }
 
+bool destroy_dsu_metadata_key(const std::string& dsu_slot) {
+    LOG(DEBUG) << "destroy_dsu_metadata_key: " << dsu_slot;
+
+    const auto dsu_metadata_key_dir = android::gsi::GetDsuMetadataKeyDir(dsu_slot);
+    if (!pathExists(dsu_metadata_key_dir)) {
+        LOG(DEBUG) << "DSU metadata_key_dir doesn't exist, nothing to remove: "
+                   << dsu_metadata_key_dir;
+        return true;
+    }
+
+    // Ensure that the DSU key directory is different from the host OS'.
+    // Under normal circumstances, this should never happen, but handle it just in case.
+    if (auto data_rec = GetEntryForMountPoint(&fstab_default, "/data")) {
+        if (dsu_metadata_key_dir == data_rec->metadata_key_dir) {
+            LOG(ERROR) << "DSU metadata_key_dir is same as host OS: " << dsu_metadata_key_dir;
+            return false;
+        }
+    }
+
+    bool ok = true;
+    for (auto suffix : {"/key", "/tmp"}) {
+        const auto key_path = dsu_metadata_key_dir + suffix;
+        if (pathExists(key_path)) {
+            LOG(DEBUG) << "Destroy key: " << key_path;
+            if (!android::vold::destroyKey(key_path)) {
+                LOG(ERROR) << "Failed to destroyKey(): " << key_path;
+                ok = false;
+            }
+        }
+    }
+    if (!ok) {
+        return false;
+    }
+
+    LOG(DEBUG) << "Remove DSU metadata_key_dir: " << dsu_metadata_key_dir;
+    // DeleteDirContentsAndDir() already logged any error, so don't log repeatedly.
+    return android::vold::DeleteDirContentsAndDir(dsu_metadata_key_dir) == android::OK;
+}
+
 }  // namespace vold
 }  // namespace android
diff --git a/MetadataCrypt.h b/MetadataCrypt.h
index dc68e7c..7341a08 100644
--- a/MetadataCrypt.h
+++ b/MetadataCrypt.h
@@ -34,6 +34,8 @@
                                  const android::vold::KeyBuffer& key,
                                  std::string* out_crypto_blkdev);
 
+bool destroy_dsu_metadata_key(const std::string& dsu_slot);
+
 }  // namespace vold
 }  // namespace android
 #endif
diff --git a/MoveStorage.cpp b/MoveStorage.cpp
index 2447cce..3f636a2 100644
--- a/MoveStorage.cpp
+++ b/MoveStorage.cpp
@@ -70,9 +70,7 @@
     bool found = false;
     struct dirent* ent;
     while ((ent = readdir(dirp.get())) != NULL) {
-        if ((!strcmp(ent->d_name, ".")) || (!strcmp(ent->d_name, ".."))) {
-            continue;
-        }
+        if (IsDotOrDotDot(*ent)) continue;
         auto subdir = path + "/" + ent->d_name;
         found |= pushBackContents(subdir, cmd, searchLevels - 1);
     }
diff --git a/Utils.cpp b/Utils.cpp
index a9b7440..afb0989 100644
--- a/Utils.cpp
+++ b/Utils.cpp
@@ -199,7 +199,7 @@
 }
 
 int SetQuotaInherit(const std::string& path) {
-    unsigned long flags;
+    unsigned int flags;
 
     android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_CLOEXEC)));
     if (fd == -1) {
@@ -416,7 +416,32 @@
     return OK;
 }
 
-status_t PrepareDir(const std::string& path, mode_t mode, uid_t uid, gid_t gid) {
+int SetAttrs(const std::string& path, unsigned int attrs) {
+    unsigned int flags;
+    android::base::unique_fd fd(
+            TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_NONBLOCK | O_CLOEXEC)));
+
+    if (fd == -1) {
+        PLOG(ERROR) << "Failed to open " << path;
+        return -1;
+    }
+
+    if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) {
+        PLOG(ERROR) << "Failed to get flags for " << path;
+        return -1;
+    }
+
+    if ((flags & attrs) == attrs) return 0;
+    flags |= attrs;
+    if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) {
+        PLOG(ERROR) << "Failed to set flags for " << path << "(0x" << std::hex << attrs << ")";
+        return -1;
+    }
+    return 0;
+}
+
+status_t PrepareDir(const std::string& path, mode_t mode, uid_t uid, gid_t gid,
+                    unsigned int attrs) {
     std::lock_guard<std::mutex> lock(kSecurityLock);
     const char* cpath = path.c_str();
 
@@ -434,6 +459,9 @@
         freecon(secontext);
     }
 
+    if (res) return -errno;
+    if (attrs) res = SetAttrs(path, attrs);
+
     if (res == 0) {
         return OK;
     } else {
@@ -928,10 +956,7 @@
             int subfd;
 
             /* always skip "." and ".." */
-            if (name[0] == '.') {
-                if (name[1] == 0) continue;
-                if ((name[1] == '.') && (name[2] == 0)) continue;
-            }
+            if (IsDotOrDotDot(*de)) continue;
 
             subfd = openat(dfd, name, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
             if (subfd >= 0) {
@@ -1104,6 +1129,13 @@
     }
 }
 
+// Returns true if |path1| names the same existing file or directory as |path2|.
+bool IsSameFile(const std::string& path1, const std::string& path2) {
+    struct stat stbuf1, stbuf2;
+    if (stat(path1.c_str(), &stbuf1) != 0 || stat(path2.c_str(), &stbuf2) != 0) return false;
+    return stbuf1.st_ino == stbuf2.st_ino && stbuf1.st_dev == stbuf2.st_dev;
+}
+
 status_t RestoreconRecursive(const std::string& path) {
     LOG(DEBUG) << "Starting restorecon of " << path;
 
@@ -1228,6 +1260,10 @@
     return OK;
 }
 
+bool IsDotOrDotDot(const struct dirent& ent) {
+    return strcmp(ent.d_name, ".") == 0 || strcmp(ent.d_name, "..") == 0;
+}
+
 static status_t delete_dir_contents(DIR* dir) {
     // Shamelessly borrowed from android::installd
     int dfd = dirfd(dir);
@@ -1241,10 +1277,7 @@
         const char* name = de->d_name;
         if (de->d_type == DT_DIR) {
             /* always skip "." and ".." */
-            if (name[0] == '.') {
-                if (name[1] == 0) continue;
-                if ((name[1] == '.') && (name[2] == 0)) continue;
-            }
+            if (IsDotOrDotDot(*de)) continue;
 
             android::base::unique_fd subfd(
                 openat(dfd, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC));
diff --git a/Utils.h b/Utils.h
index 04cbac4..cf3fd9b 100644
--- a/Utils.h
+++ b/Utils.h
@@ -67,7 +67,8 @@
                           bool fixupExisting);
 
 /* fs_prepare_dir wrapper that creates with SELinux context */
-status_t PrepareDir(const std::string& path, mode_t mode, uid_t uid, gid_t gid);
+status_t PrepareDir(const std::string& path, mode_t mode, uid_t uid, gid_t gid,
+                    unsigned int attrs = 0);
 
 /* Really unmounts the path, killing active processes along the way */
 status_t ForceUnmount(const std::string& path);
@@ -154,6 +155,8 @@
 
 dev_t GetDevice(const std::string& path);
 
+bool IsSameFile(const std::string& path1, const std::string& path2);
+
 status_t EnsureDirExists(const std::string& path, mode_t mode, uid_t uid, gid_t gid);
 
 status_t RestoreconRecursive(const std::string& path);
@@ -167,6 +170,8 @@
 status_t UnmountTreeWithPrefix(const std::string& prefix);
 status_t UnmountTree(const std::string& mountPoint);
 
+bool IsDotOrDotDot(const struct dirent& ent);
+
 status_t DeleteDirContentsAndDir(const std::string& pathname);
 status_t DeleteDirContents(const std::string& pathname);
 
diff --git a/VoldNativeService.cpp b/VoldNativeService.cpp
index d310acd..3fb4e65 100644
--- a/VoldNativeService.cpp
+++ b/VoldNativeService.cpp
@@ -950,5 +950,12 @@
     return translate(incfs::bindMount(sourceDir, targetDir));
 }
 
+binder::Status VoldNativeService::destroyDsuMetadataKey(const std::string& dsuSlot) {
+    ENFORCE_SYSTEM_OR_ROOT;
+    ACQUIRE_LOCK;
+
+    return translateBool(destroy_dsu_metadata_key(dsuSlot));
+}
+
 }  // namespace vold
 }  // namespace android
diff --git a/VoldNativeService.h b/VoldNativeService.h
index f10bf5f..9914879 100644
--- a/VoldNativeService.h
+++ b/VoldNativeService.h
@@ -159,6 +159,8 @@
             const ::android::os::incremental::IncrementalFileSystemControlParcel& control,
             bool enableReadLogs) override;
     binder::Status bindMount(const std::string& sourceDir, const std::string& targetDir) override;
+
+    binder::Status destroyDsuMetadataKey(const std::string& dsuSlot) override;
 };
 
 }  // namespace vold
diff --git a/binder/android/os/IVold.aidl b/binder/android/os/IVold.aidl
index d0e753e..1d6225f 100644
--- a/binder/android/os/IVold.aidl
+++ b/binder/android/os/IVold.aidl
@@ -139,6 +139,8 @@
     void setIncFsMountOptions(in IncrementalFileSystemControlParcel control, boolean enableReadLogs);
     void bindMount(@utf8InCpp String sourceDir, @utf8InCpp String targetDir);
 
+    void destroyDsuMetadataKey(@utf8InCpp String dsuSlot);
+
     const int ENCRYPTION_FLAG_NO_UI = 4;
 
     const int ENCRYPTION_STATE_NONE = 1;
@@ -164,6 +166,7 @@
 
     const int STORAGE_FLAG_DE = 1;
     const int STORAGE_FLAG_CE = 2;
+    const int STORAGE_FLAG_LEVEL_FROM_USER = 4;
 
     const int REMOUNT_MODE_NONE = 0;
     const int REMOUNT_MODE_DEFAULT = 1;
diff --git a/cryptfs.cpp b/cryptfs.cpp
index 8b7ac0a..faed65b 100644
--- a/cryptfs.cpp
+++ b/cryptfs.cpp
@@ -90,6 +90,8 @@
 #define CRYPT_FOOTER_TO_PERSIST_OFFSET 0x1000
 #define CRYPT_PERSIST_DATA_SIZE 0x1000
 
+#define CRYPT_SECTOR_SIZE 512
+
 #define MAX_CRYPTO_TYPE_NAME_LEN 64
 
 #define MAX_KEY_LEN 48
@@ -98,9 +100,7 @@
 
 /* definitions of flags in the structure below */
 #define CRYPT_MNT_KEY_UNENCRYPTED 0x1 /* The key for the partition is not encrypted. */
-#define CRYPT_ENCRYPTION_IN_PROGRESS       \
-    0x2 /* Encryption partially completed, \
-           encrypted_upto valid*/
+#define CRYPT_ENCRYPTION_IN_PROGRESS 0x2 /* no longer used */
 #define CRYPT_INCONSISTENT_STATE                    \
     0x4 /* Set when starting encryption, clear when \
            exit cleanly, either through success or  \
@@ -195,12 +195,8 @@
     __le8 N_factor;        /* (1 << N) */
     __le8 r_factor;        /* (1 << r) */
     __le8 p_factor;        /* (1 << p) */
-    __le64 encrypted_upto; /* If we are in state CRYPT_ENCRYPTION_IN_PROGRESS and
-                              we have to stop (e.g. power low) this is the last
-                              encrypted 512 byte sector.*/
-    __le8 hash_first_block[SHA256_DIGEST_LENGTH]; /* When CRYPT_ENCRYPTION_IN_PROGRESS
-                                                     set, hash of first block, used
-                                                     to validate before continuing*/
+    __le64 encrypted_upto; /* no longer used */
+    __le8 hash_first_block[SHA256_DIGEST_LENGTH]; /* no longer used */
 
     /* key_master key, used to sign the derived key which is then used to generate
      * the intermediate key
@@ -2069,61 +2065,6 @@
 
 #define FRAMEWORK_BOOT_WAIT 60
 
-static int cryptfs_SHA256_fileblock(const char* filename, __le8* buf) {
-    int fd = open(filename, O_RDONLY | O_CLOEXEC);
-    if (fd == -1) {
-        SLOGE("Error opening file %s", filename);
-        return -1;
-    }
-
-    char block[CRYPT_INPLACE_BUFSIZE];
-    memset(block, 0, sizeof(block));
-    if (unix_read(fd, block, sizeof(block)) < 0) {
-        SLOGE("Error reading file %s", filename);
-        close(fd);
-        return -1;
-    }
-
-    close(fd);
-
-    SHA256_CTX c;
-    SHA256_Init(&c);
-    SHA256_Update(&c, block, sizeof(block));
-    SHA256_Final(buf, &c);
-
-    return 0;
-}
-
-static int cryptfs_enable_all_volumes(struct crypt_mnt_ftr* crypt_ftr, const char* crypto_blkdev,
-                                      const char* real_blkdev, int previously_encrypted_upto) {
-    off64_t cur_encryption_done = 0, tot_encryption_size = 0;
-    int rc = -1;
-
-    /* The size of the userdata partition, and add in the vold volumes below */
-    tot_encryption_size = crypt_ftr->fs_size;
-
-    rc = cryptfs_enable_inplace(crypto_blkdev, real_blkdev, crypt_ftr->fs_size, &cur_encryption_done,
-                                tot_encryption_size, previously_encrypted_upto, true);
-
-    if (rc == ENABLE_INPLACE_ERR_DEV) {
-        /* Hack for b/17898962 */
-        SLOGE("cryptfs_enable: crypto block dev failure. Must reboot...\n");
-        cryptfs_reboot(RebootType::reboot);
-    }
-
-    if (!rc) {
-        crypt_ftr->encrypted_upto = cur_encryption_done;
-    }
-
-    if (!rc && crypt_ftr->encrypted_upto == crypt_ftr->fs_size) {
-        /* The inplace routine never actually sets the progress to 100% due
-         * to the round down nature of integer division, so set it here */
-        property_set("vold.encrypt_progress", "100");
-    }
-
-    return rc;
-}
-
 static int vold_unmountAll(void) {
     VolumeManager* vm = VolumeManager::Instance();
     return vm->unmountAll();
@@ -2140,26 +2081,12 @@
     char lockid[32] = {0};
     std::string key_loc;
     int num_vols;
-    off64_t previously_encrypted_upto = 0;
     bool rebootEncryption = false;
     bool onlyCreateHeader = false;
     std::unique_ptr<android::wakelock::WakeLock> wakeLock = nullptr;
 
     if (get_crypt_ftr_and_key(&crypt_ftr) == 0) {
-        if (crypt_ftr.flags & CRYPT_ENCRYPTION_IN_PROGRESS) {
-            /* An encryption was underway and was interrupted */
-            previously_encrypted_upto = crypt_ftr.encrypted_upto;
-            crypt_ftr.encrypted_upto = 0;
-            crypt_ftr.flags &= ~CRYPT_ENCRYPTION_IN_PROGRESS;
-
-            /* At this point, we are in an inconsistent state. Until we successfully
-               complete encryption, a reboot will leave us broken. So mark the
-               encryption failed in case that happens.
-               On successfully completing encryption, remove this flag */
-            crypt_ftr.flags |= CRYPT_INCONSISTENT_STATE;
-
-            put_crypt_ftr_and_key(&crypt_ftr);
-        } else if (crypt_ftr.flags & CRYPT_FORCE_ENCRYPTION) {
+        if (crypt_ftr.flags & CRYPT_FORCE_ENCRYPTION) {
             if (!check_ftr_sha(&crypt_ftr)) {
                 memset(&crypt_ftr, 0, sizeof(crypt_ftr));
                 put_crypt_ftr_and_key(&crypt_ftr);
@@ -2177,7 +2104,7 @@
     }
 
     property_get("ro.crypto.state", encrypted_state, "");
-    if (!strcmp(encrypted_state, "encrypted") && !previously_encrypted_upto) {
+    if (!strcmp(encrypted_state, "encrypted")) {
         SLOGE("Device is already running encrypted, aborting");
         goto error_unencrypted;
     }
@@ -2264,7 +2191,7 @@
 
     /* Start the actual work of making an encrypted filesystem */
     /* Initialize a crypt_mnt_ftr for the partition */
-    if (previously_encrypted_upto == 0 && !rebootEncryption) {
+    if (!rebootEncryption) {
         if (cryptfs_init_crypt_mnt_ftr(&crypt_ftr)) {
             goto error_shutting_down;
         }
@@ -2339,77 +2266,46 @@
     }
 
     decrypt_master_key(passwd, decrypted_master_key, &crypt_ftr, 0, 0);
-    create_crypto_blk_dev(&crypt_ftr, decrypted_master_key, real_blkdev.c_str(), &crypto_blkdev,
-                          CRYPTO_BLOCK_DEVICE, 0);
-
-    /* If we are continuing, check checksums match */
-    rc = 0;
-    if (previously_encrypted_upto) {
-        __le8 hash_first_block[SHA256_DIGEST_LENGTH];
-        rc = cryptfs_SHA256_fileblock(crypto_blkdev.c_str(), hash_first_block);
-
-        if (!rc &&
-            memcmp(hash_first_block, crypt_ftr.hash_first_block, sizeof(hash_first_block)) != 0) {
-            SLOGE("Checksums do not match - trigger wipe");
-            rc = -1;
-        }
-    }
-
+    rc = create_crypto_blk_dev(&crypt_ftr, decrypted_master_key, real_blkdev.c_str(),
+                               &crypto_blkdev, CRYPTO_BLOCK_DEVICE, 0);
     if (!rc) {
-        rc = cryptfs_enable_all_volumes(&crypt_ftr, crypto_blkdev.c_str(), real_blkdev.data(),
-                                        previously_encrypted_upto);
-    }
-
-    /* Calculate checksum if we are not finished */
-    if (!rc && crypt_ftr.encrypted_upto != crypt_ftr.fs_size) {
-        rc = cryptfs_SHA256_fileblock(crypto_blkdev.c_str(), crypt_ftr.hash_first_block);
-        if (rc) {
-            SLOGE("Error calculating checksum for continuing encryption");
+        if (encrypt_inplace(crypto_blkdev, real_blkdev, crypt_ftr.fs_size, true)) {
+            crypt_ftr.encrypted_upto = crypt_ftr.fs_size;
+            rc = 0;
+        } else {
             rc = -1;
         }
+        /* Undo the dm-crypt mapping whether we succeed or not */
+        delete_crypto_blk_dev(CRYPTO_BLOCK_DEVICE);
     }
 
-    /* Undo the dm-crypt mapping whether we succeed or not */
-    delete_crypto_blk_dev(CRYPTO_BLOCK_DEVICE);
-
     if (!rc) {
         /* Success */
         crypt_ftr.flags &= ~CRYPT_INCONSISTENT_STATE;
 
-        if (crypt_ftr.encrypted_upto != crypt_ftr.fs_size) {
-            SLOGD("Encrypted up to sector %lld - will continue after reboot",
-                  crypt_ftr.encrypted_upto);
-            crypt_ftr.flags |= CRYPT_ENCRYPTION_IN_PROGRESS;
-        }
-
         put_crypt_ftr_and_key(&crypt_ftr);
 
-        if (crypt_ftr.encrypted_upto == crypt_ftr.fs_size) {
-            char value[PROPERTY_VALUE_MAX];
-            property_get("ro.crypto.state", value, "");
-            if (!strcmp(value, "")) {
-                /* default encryption - continue first boot sequence */
-                property_set("ro.crypto.state", "encrypted");
-                property_set("ro.crypto.type", "block");
-                wakeLock.reset(nullptr);
-                if (rebootEncryption && crypt_ftr.crypt_type != CRYPT_TYPE_DEFAULT) {
-                    // Bring up cryptkeeper that will check the password and set it
-                    property_set("vold.decrypt", "trigger_shutdown_framework");
-                    sleep(2);
-                    property_set("vold.encrypt_progress", "");
-                    cryptfs_trigger_restart_min_framework();
-                } else {
-                    cryptfs_check_passwd(DEFAULT_PASSWORD);
-                    cryptfs_restart_internal(1);
-                }
-                return 0;
+        char value[PROPERTY_VALUE_MAX];
+        property_get("ro.crypto.state", value, "");
+        if (!strcmp(value, "")) {
+            /* default encryption - continue first boot sequence */
+            property_set("ro.crypto.state", "encrypted");
+            property_set("ro.crypto.type", "block");
+            wakeLock.reset(nullptr);
+            if (rebootEncryption && crypt_ftr.crypt_type != CRYPT_TYPE_DEFAULT) {
+                // Bring up cryptkeeper that will check the password and set it
+                property_set("vold.decrypt", "trigger_shutdown_framework");
+                sleep(2);
+                property_set("vold.encrypt_progress", "");
+                cryptfs_trigger_restart_min_framework();
             } else {
-                sleep(2); /* Give the UI a chance to show 100% progress */
-                cryptfs_reboot(RebootType::reboot);
+                cryptfs_check_passwd(DEFAULT_PASSWORD);
+                cryptfs_restart_internal(1);
             }
+            return 0;
         } else {
-            sleep(2); /* Partially encrypted, ensure writes flushed to ssd */
-            cryptfs_reboot(RebootType::shutdown);
+            sleep(2); /* Give the UI a chance to show 100% progress */
+            cryptfs_reboot(RebootType::reboot);
         }
     } else {
         char value[PROPERTY_VALUE_MAX];
diff --git a/model/PrivateVolume.cpp b/model/PrivateVolume.cpp
index 39a946c..1875b7b 100644
--- a/model/PrivateVolume.cpp
+++ b/model/PrivateVolume.cpp
@@ -166,11 +166,14 @@
 
     RestoreconRecursive(mPath);
 
+    int attrs = 0;
+    if (!IsSdcardfsUsed()) attrs = FS_CASEFOLD_FL;
+
     // Verify that common directories are ready to roll
     if (PrepareDir(mPath + "/app", 0771, AID_SYSTEM, AID_SYSTEM) ||
         PrepareDir(mPath + "/user", 0711, AID_SYSTEM, AID_SYSTEM) ||
         PrepareDir(mPath + "/user_de", 0711, AID_SYSTEM, AID_SYSTEM) ||
-        PrepareDir(mPath + "/media", 0770, AID_MEDIA_RW, AID_MEDIA_RW) ||
+        PrepareDir(mPath + "/media", 0770, AID_MEDIA_RW, AID_MEDIA_RW, attrs) ||
         PrepareDir(mPath + "/media/0", 0770, AID_MEDIA_RW, AID_MEDIA_RW) ||
         PrepareDir(mPath + "/local", 0751, AID_ROOT, AID_ROOT) ||
         PrepareDir(mPath + "/local/tmp", 0771, AID_SHELL, AID_SHELL)) {
diff --git a/vold_prepare_subdirs.cpp b/vold_prepare_subdirs.cpp
index d624d73..0283614 100644
--- a/vold_prepare_subdirs.cpp
+++ b/vold_prepare_subdirs.cpp
@@ -54,20 +54,33 @@
     return s.size() < 40 && s.find_first_not_of("0123456789abcdefABCDEF-_") == std::string::npos;
 }
 
-static bool prepare_dir(struct selabel_handle* sehandle, mode_t mode, uid_t uid, gid_t gid,
-                        const std::string& path) {
+static bool prepare_dir_for_user(struct selabel_handle* sehandle, mode_t mode, uid_t uid, gid_t gid,
+                                 const std::string& path, uid_t user_id) {
     auto clearfscreatecon = android::base::make_scope_guard([] { setfscreatecon(nullptr); });
     auto secontext = std::unique_ptr<char, void (*)(char*)>(nullptr, freecon);
-    char* tmp_secontext;
-    if (sehandle && selabel_lookup(sehandle, &tmp_secontext, path.c_str(), S_IFDIR) == 0) {
-        secontext.reset(tmp_secontext);
+    if (sehandle) {
+        char* tmp_secontext;
+
+        if (selabel_lookup(sehandle, &tmp_secontext, path.c_str(), S_IFDIR) == 0) {
+            secontext.reset(tmp_secontext);
+
+            if (user_id != (uid_t)-1) {
+                if (selinux_android_context_with_level(secontext.get(), &tmp_secontext, user_id,
+                                                       (uid_t)-1) != 0) {
+                    PLOG(ERROR) << "Unable to create context with level for: " << path;
+                    return false;
+                }
+                secontext.reset(tmp_secontext);  // Free the context
+            }
+        }
     }
+
     LOG(DEBUG) << "Setting up mode " << std::oct << mode << std::dec << " uid " << uid << " gid "
                << gid << " context " << (secontext ? secontext.get() : "null")
                << " on path: " << path;
     if (secontext) {
         if (setfscreatecon(secontext.get()) != 0) {
-            PLOG(ERROR) << "Unable to read setfscreatecon for: " << path;
+            PLOG(ERROR) << "Unable to setfscreatecon for: " << path;
             return false;
         }
     }
@@ -93,6 +106,11 @@
     return true;
 }
 
+static bool prepare_dir(struct selabel_handle* sehandle, mode_t mode, uid_t uid, gid_t gid,
+                        const std::string& path) {
+    return prepare_dir_for_user(sehandle, mode, uid, gid, path, (uid_t)-1);
+}
+
 static bool rmrf_contents(const std::string& path) {
     auto dirp = std::unique_ptr<DIR, int (*)(DIR*)>(opendir(path.c_str()), closedir);
     if (!dirp) {
@@ -148,8 +166,17 @@
 static bool prepare_subdirs(const std::string& volume_uuid, int user_id, int flags) {
     struct selabel_handle* sehandle = selinux_android_file_context_handle();
 
-    if (volume_uuid.empty()) {
-        if (flags & android::os::IVold::STORAGE_FLAG_DE) {
+    const uid_t user_for_level =
+            (flags & android::os::IVold::STORAGE_FLAG_LEVEL_FROM_USER) ? user_id : -1;
+
+    if (flags & android::os::IVold::STORAGE_FLAG_DE) {
+        auto user_de_path = android::vold::BuildDataUserDePath(volume_uuid, user_id);
+        if (!prepare_dir_for_user(sehandle, 0771, AID_SYSTEM, AID_SYSTEM, user_de_path,
+                                  user_for_level)) {
+            return false;
+        }
+
+        if (volume_uuid.empty()) {
             auto misc_de_path = android::vold::BuildDataMiscDePath(user_id);
             if (!prepare_dir(sehandle, 0700, 0, 0, misc_de_path + "/vold")) return false;
             if (!prepare_dir(sehandle, 0700, 0, 0, misc_de_path + "/storaged")) return false;
@@ -158,6 +185,12 @@
             prepare_dir(sehandle, 0700, 0, 0, misc_de_path + "/apexrollback");
             prepare_apex_subdirs(sehandle, misc_de_path);
 
+            auto profiles_de_path = android::vold::BuildDataProfilesDePath(user_id);
+            if (!prepare_dir_for_user(sehandle, 0771, AID_SYSTEM, AID_SYSTEM, profiles_de_path,
+                                      user_for_level)) {
+                return false;
+            }
+
             auto vendor_de_path = android::vold::BuildDataVendorDePath(user_id);
             if (!prepare_dir(sehandle, 0700, AID_SYSTEM, AID_SYSTEM, vendor_de_path + "/fpdata")) {
                 return false;
@@ -167,11 +200,20 @@
                 return false;
             }
         }
-        if (flags & android::os::IVold::STORAGE_FLAG_CE) {
+    }
+    if (flags & android::os::IVold::STORAGE_FLAG_CE) {
+        auto user_ce_path = android::vold::BuildDataUserCePath(volume_uuid, user_id);
+        if (!prepare_dir_for_user(sehandle, 0771, AID_SYSTEM, AID_SYSTEM, user_ce_path,
+                                  user_for_level)) {
+            return false;
+        }
+
+        if (volume_uuid.empty()) {
             auto misc_ce_path = android::vold::BuildDataMiscCePath(user_id);
             if (!prepare_dir(sehandle, 0700, 0, 0, misc_ce_path + "/vold")) return false;
             if (!prepare_dir(sehandle, 0700, 0, 0, misc_ce_path + "/storaged")) return false;
             if (!prepare_dir(sehandle, 0700, 0, 0, misc_ce_path + "/rollback")) return false;
+
             // TODO: Return false if this returns false once sure this should succeed.
             prepare_dir(sehandle, 0700, 0, 0, misc_ce_path + "/apexrollback");
             prepare_apex_subdirs(sehandle, misc_ce_path);