diff --git a/Android.bp b/Android.bp
index 861abba..a337b83 100644
--- a/Android.bp
+++ b/Android.bp
@@ -149,6 +149,14 @@
         "model/VolumeEncryption.cpp",
     ],
     product_variables: {
+        arc: {
+            exclude_srcs: [
+                "model/StubVolume.cpp",
+            ],
+            static_libs: [
+                "libarcvolume",
+            ],
+        },
         debuggable: {
             cppflags: ["-D__ANDROID_DEBUGGABLE__"],
         },
@@ -189,6 +197,17 @@
         "android.hardware.health.storage-unstable-ndk_platform",
         "libbinder_ndk",
     ],
+
+    product_variables: {
+        arc: {
+            exclude_srcs: [
+                "model/StubVolume.cpp",
+            ],
+            static_libs: [
+                "libarcvolume",
+            ],
+        },
+	},
 }
 
 cc_binary {
diff --git a/FsCrypt.cpp b/FsCrypt.cpp
index ebb4640..c62eb25 100644
--- a/FsCrypt.cpp
+++ b/FsCrypt.cpp
@@ -74,6 +74,7 @@
 using android::vold::KeyGeneration;
 using android::vold::retrieveKey;
 using android::vold::retrieveOrGenerateKey;
+using android::vold::SetDefaultAcl;
 using android::vold::SetQuotaInherit;
 using android::vold::SetQuotaProjectId;
 using android::vold::writeStringToFile;
@@ -200,7 +201,7 @@
     auto const paths = get_ce_key_paths(directory_path);
     for (auto const ce_key_path : paths) {
         LOG(DEBUG) << "Trying user CE key " << ce_key_path;
-        if (retrieveKey(ce_key_path, auth, ce_key, false)) {
+        if (retrieveKey(ce_key_path, auth, ce_key)) {
             LOG(DEBUG) << "Successfully retrieved key";
             fixate_user_ce_key(directory_path, ce_key_path, paths);
             return true;
@@ -401,7 +402,7 @@
         userid_t user_id = std::stoi(entry->d_name);
         auto key_path = de_dir + "/" + entry->d_name;
         KeyBuffer de_key;
-        if (!retrieveKey(key_path, kEmptyAuthentication, &de_key, false)) return false;
+        if (!retrieveKey(key_path, kEmptyAuthentication, &de_key)) return false;
         EncryptionPolicy de_policy;
         if (!install_storage_key(DATA_MNT_POINT, options, de_key, &de_policy)) return false;
         auto ret = s_de_policies.insert({user_id, de_policy});
@@ -435,7 +436,7 @@
 
     KeyBuffer device_key;
     if (!retrieveOrGenerateKey(device_key_path, device_key_temp, kEmptyAuthentication,
-                               makeGen(options), &device_key, false))
+                               makeGen(options), &device_key))
         return false;
 
     EncryptionPolicy device_policy;
@@ -669,7 +670,7 @@
     EncryptionOptions options;
     if (!get_volume_file_encryption_options(&options)) return false;
     KeyBuffer key;
-    if (!retrieveOrGenerateKey(key_path, key_path + "_tmp", auth, makeGen(options), &key, false))
+    if (!retrieveOrGenerateKey(key_path, key_path + "_tmp", auth, makeGen(options), &key))
         return false;
     if (!install_storage_key(BuildDataPath(volume_uuid), options, key, policy)) return false;
     return true;
@@ -688,12 +689,12 @@
     auto const directory_path = get_ce_key_directory_path(user_id);
     KeyBuffer ce_key;
     std::string ce_key_current_path = get_ce_key_current_path(directory_path);
-    if (retrieveKey(ce_key_current_path, retrieve_auth, &ce_key, false)) {
+    if (retrieveKey(ce_key_current_path, retrieve_auth, &ce_key)) {
         LOG(DEBUG) << "Successfully retrieved key";
         // TODO(147732812): Remove this once Locksettingservice is fixed.
         // Currently it calls fscrypt_clear_user_key_auth with a secret when lockscreen is
         // changed from swipe to none or vice-versa
-    } else if (retrieveKey(ce_key_current_path, kEmptyAuthentication, &ce_key, false)) {
+    } else if (retrieveKey(ce_key_current_path, kEmptyAuthentication, &ce_key)) {
         LOG(DEBUG) << "Successfully retrieved key with empty auth";
     } else {
         LOG(ERROR) << "Failed to retrieve key for user " << user_id;
@@ -794,11 +795,6 @@
 
 static bool prepare_subdirs(const std::string& action, const std::string& volume_uuid,
                             userid_t user_id, int flags) {
-    // TODO(b/141677108): Remove this & make it the default behavior
-    if (android::base::GetProperty("ro.vold.level_from_user", "0") == "1") {
-        flags |= android::os::IVold::STORAGE_FLAG_LEVEL_FROM_USER;
-    }
-
     if (0 != android::vold::ForkExecvp(
                  std::vector<std::string>{prepare_subdirs_path, action, volume_uuid,
                                           std::to_string(user_id), std::to_string(flags)})) {
@@ -867,7 +863,15 @@
             if (!prepare_dir(misc_ce_path, 01771, AID_SYSTEM, AID_MISC)) return false;
             if (!prepare_dir(vendor_ce_path, 0771, AID_ROOT, AID_ROOT)) return false;
         }
-        if (!prepare_dir(media_ce_path, 0770, AID_MEDIA_RW, AID_MEDIA_RW)) return false;
+        if (!prepare_dir(media_ce_path, 02770, AID_MEDIA_RW, AID_MEDIA_RW)) return false;
+        // On devices without sdcardfs (kernel 5.4+), the path permissions aren't fixed
+        // up automatically; therefore, use a default ACL, to ensure apps with MEDIA_RW
+        // can keep reading external storage; in particular, this allows app cloning
+        // scenarios to work correctly on such devices.
+        int ret = SetDefaultAcl(media_ce_path, 02770, AID_MEDIA_RW, AID_MEDIA_RW, {AID_MEDIA_RW});
+        if (ret != android::OK) {
+            return false;
+        }
 
         if (!prepare_dir(user_ce_path, 0771, AID_SYSTEM, AID_SYSTEM)) return false;
 
diff --git a/KeyStorage.cpp b/KeyStorage.cpp
index 533a7cb..8147827 100644
--- a/KeyStorage.cpp
+++ b/KeyStorage.cpp
@@ -21,6 +21,7 @@
 #include "ScryptParameters.h"
 #include "Utils.h"
 
+#include <algorithm>
 #include <thread>
 #include <vector>
 
@@ -208,75 +209,161 @@
     return true;
 }
 
-static void deferedKmDeleteKey(const std::string& kmkey) {
-    while (!android::base::WaitForProperty("vold.checkpoint_committed", "1")) {
-        LOG(ERROR) << "Wait for boot timed out";
+static std::mutex key_upgrade_lock;
+
+// List of key directories that have had their Keymaster key upgraded during
+// this boot and written to "keymaster_key_blob_upgraded", but replacing the old
+// key was delayed due to an active checkpoint.  Protected by key_upgrade_lock.
+static std::vector<std::string> key_dirs_to_commit;
+
+// Replaces |dir|/keymaster_key_blob with |dir|/keymaster_key_blob_upgraded and
+// deletes the old key from Keymaster.
+static bool CommitUpgradedKey(Keymaster& keymaster, const std::string& dir) {
+    auto blob_file = dir + "/" + kFn_keymaster_key_blob;
+    auto upgraded_blob_file = dir + "/" + kFn_keymaster_key_blob_upgraded;
+
+    std::string blob;
+    if (!readFileToString(blob_file, &blob)) return false;
+
+    if (rename(upgraded_blob_file.c_str(), blob_file.c_str()) != 0) {
+        PLOG(ERROR) << "Failed to rename " << upgraded_blob_file << " to " << blob_file;
+        return false;
     }
+    // Ensure that the rename is persisted before deleting the Keymaster key.
+    if (!FsyncDirectory(dir)) return false;
+
+    if (!keymaster || !keymaster.deleteKey(blob)) {
+        LOG(WARNING) << "Failed to delete old key " << blob_file
+                     << " from Keymaster; continuing anyway";
+        // Continue on, but the space in Keymaster used by the old key won't be freed.
+    }
+    return true;
+}
+
+static void DeferredCommitKeys() {
+    android::base::WaitForProperty("vold.checkpoint_committed", "1");
+    LOG(INFO) << "Committing upgraded keys";
     Keymaster keymaster;
-    if (!keymaster || !keymaster.deleteKey(kmkey)) {
-        LOG(ERROR) << "Defered Key deletion failed during upgrade";
+    if (!keymaster) {
+        LOG(ERROR) << "Failed to open Keymaster; old keys won't be deleted from Keymaster";
+        // Continue on, but the space in Keymaster used by the old keys won't be freed.
+    }
+    std::lock_guard<std::mutex> lock(key_upgrade_lock);
+    for (auto& dir : key_dirs_to_commit) {
+        LOG(INFO) << "Committing upgraded key " << dir;
+        CommitUpgradedKey(keymaster, dir);
+    }
+    key_dirs_to_commit.clear();
+}
+
+// Returns true if the Keymaster key in |dir| has already been upgraded and is
+// pending being committed.  Assumes that key_upgrade_lock is held.
+static bool IsKeyCommitPending(const std::string& dir) {
+    for (const auto& dir_to_commit : key_dirs_to_commit) {
+        if (IsSameFile(dir, dir_to_commit)) return true;
+    }
+    return false;
+}
+
+// Schedules the upgraded Keymaster key in |dir| to be committed later.
+// Assumes that key_upgrade_lock is held.
+static void ScheduleKeyCommit(const std::string& dir) {
+    if (key_dirs_to_commit.empty()) std::thread(DeferredCommitKeys).detach();
+    key_dirs_to_commit.push_back(dir);
+}
+
+static void CancelPendingKeyCommit(const std::string& dir) {
+    std::lock_guard<std::mutex> lock(key_upgrade_lock);
+    for (auto it = key_dirs_to_commit.begin(); it != key_dirs_to_commit.end(); it++) {
+        if (IsSameFile(*it, dir)) {
+            LOG(DEBUG) << "Cancelling pending commit of upgraded key " << dir
+                       << " because it is being destroyed";
+            key_dirs_to_commit.erase(it);
+            break;
+        }
     }
 }
 
-bool kmDeleteKey(Keymaster& keymaster, const std::string& kmKey) {
-    bool needs_cp = cp_needsCheckpoint();
-
-    if (needs_cp) {
-        std::thread(deferedKmDeleteKey, kmKey).detach();
-        LOG(INFO) << "Deferring Key deletion during upgrade";
-        return true;
-    } else {
-        return keymaster.deleteKey(kmKey);
+// Deletes a leftover upgraded key, if present.  An upgraded key can be left
+// over if an update failed, or if we rebooted before committing the key in a
+// freak accident.  Either way, we can re-upgrade the key if we need to.
+static void DeleteUpgradedKey(Keymaster& keymaster, const std::string& path) {
+    if (pathExists(path)) {
+        LOG(DEBUG) << "Deleting leftover upgraded key " << path;
+        std::string blob;
+        if (!android::base::ReadFileToString(path, &blob)) {
+            LOG(WARNING) << "Failed to read leftover upgraded key " << path
+                         << "; continuing anyway";
+        } else if (!keymaster.deleteKey(blob)) {
+            LOG(WARNING) << "Failed to delete leftover upgraded key " << path
+                         << " from Keymaster; continuing anyway";
+        }
+        if (unlink(path.c_str()) != 0) {
+            LOG(WARNING) << "Failed to unlink leftover upgraded key " << path
+                         << "; continuing anyway";
+        }
     }
 }
 
-static KeymasterOperation begin(Keymaster& keymaster, const std::string& dir,
-                                km::KeyPurpose purpose, const km::AuthorizationSet& keyParams,
-                                const km::AuthorizationSet& opParams,
-                                const km::HardwareAuthToken& authToken,
-                                km::AuthorizationSet* outParams, bool keepOld) {
-    auto kmKeyPath = dir + "/" + kFn_keymaster_key_blob;
-    std::string kmKey;
-    if (!readFileToString(kmKeyPath, &kmKey)) return KeymasterOperation();
+// Begins a Keymaster operation using the key stored in |dir|.
+static KeymasterOperation BeginKeymasterOp(Keymaster& keymaster, const std::string& dir,
+                                           km::KeyPurpose purpose,
+                                           const km::AuthorizationSet& keyParams,
+                                           const km::AuthorizationSet& opParams,
+                                           const km::HardwareAuthToken& authToken,
+                                           km::AuthorizationSet* outParams) {
     km::AuthorizationSet inParams(keyParams);
     inParams.append(opParams.begin(), opParams.end());
-    for (;;) {
-        auto opHandle = keymaster.begin(purpose, kmKey, inParams, authToken, outParams);
-        if (opHandle) {
-            return opHandle;
-        }
-        if (opHandle.errorCode() != km::ErrorCode::KEY_REQUIRES_UPGRADE) return opHandle;
-        LOG(DEBUG) << "Upgrading key: " << dir;
-        std::string newKey;
-        if (!keymaster.upgradeKey(kmKey, keyParams, &newKey)) return KeymasterOperation();
-        auto newKeyPath = dir + "/" + kFn_keymaster_key_blob_upgraded;
-        if (!writeStringToFile(newKey, newKeyPath)) return KeymasterOperation();
-        if (!keepOld) {
-            if (rename(newKeyPath.c_str(), kmKeyPath.c_str()) != 0) {
-                PLOG(ERROR) << "Unable to move upgraded key to location: " << kmKeyPath;
-                return KeymasterOperation();
-            }
-            if (!android::vold::FsyncDirectory(dir)) {
-                LOG(ERROR) << "Key dir sync failed: " << dir;
-                return KeymasterOperation();
-            }
-            if (!kmDeleteKey(keymaster, kmKey)) {
-                LOG(ERROR) << "Key deletion failed during upgrade, continuing anyway: " << dir;
-            }
-        }
-        kmKey = newKey;
-        LOG(INFO) << "Key upgraded: " << dir;
+
+    auto blob_file = dir + "/" + kFn_keymaster_key_blob;
+    auto upgraded_blob_file = dir + "/" + kFn_keymaster_key_blob_upgraded;
+
+    std::lock_guard<std::mutex> lock(key_upgrade_lock);
+
+    std::string blob;
+    bool already_upgraded = IsKeyCommitPending(dir);
+    if (already_upgraded) {
+        LOG(DEBUG)
+                << blob_file
+                << " was already upgraded and is waiting to be committed; using the upgraded blob";
+        if (!readFileToString(upgraded_blob_file, &blob)) return KeymasterOperation();
+    } else {
+        DeleteUpgradedKey(keymaster, upgraded_blob_file);
+        if (!readFileToString(blob_file, &blob)) return KeymasterOperation();
     }
+
+    auto opHandle = keymaster.begin(purpose, blob, inParams, authToken, outParams);
+    if (opHandle) return opHandle;
+    if (opHandle.errorCode() != km::ErrorCode::KEY_REQUIRES_UPGRADE) return opHandle;
+
+    if (already_upgraded) {
+        LOG(ERROR) << "Unexpected case; already-upgraded key " << upgraded_blob_file
+                   << " still requires upgrade";
+        return KeymasterOperation();
+    }
+    LOG(INFO) << "Upgrading key: " << blob_file;
+    if (!keymaster.upgradeKey(blob, keyParams, &blob)) return KeymasterOperation();
+    if (!writeStringToFile(blob, upgraded_blob_file)) return KeymasterOperation();
+    if (cp_needsCheckpoint()) {
+        LOG(INFO) << "Wrote upgraded key to " << upgraded_blob_file
+                  << "; delaying commit due to checkpoint";
+        ScheduleKeyCommit(dir);
+    } else {
+        if (!CommitUpgradedKey(keymaster, dir)) return KeymasterOperation();
+        LOG(INFO) << "Key upgraded: " << blob_file;
+    }
+
+    return keymaster.begin(purpose, blob, inParams, authToken, outParams);
 }
 
 static bool encryptWithKeymasterKey(Keymaster& keymaster, const std::string& dir,
                                     const km::AuthorizationSet& keyParams,
-                                    const km::HardwareAuthToken& authToken, const KeyBuffer& message,
-                                    std::string* ciphertext, bool keepOld) {
+                                    const km::HardwareAuthToken& authToken,
+                                    const KeyBuffer& message, std::string* ciphertext) {
     km::AuthorizationSet opParams;
     km::AuthorizationSet outParams;
-    auto opHandle = begin(keymaster, dir, km::KeyPurpose::ENCRYPT, keyParams, opParams, authToken,
-                          &outParams, keepOld);
+    auto opHandle = BeginKeymasterOp(keymaster, dir, km::KeyPurpose::ENCRYPT, keyParams, opParams,
+                                     authToken, &outParams);
     if (!opHandle) return false;
     auto nonceBlob = outParams.GetTagValue(km::TAG_NONCE);
     if (!nonceBlob.isOk()) {
@@ -300,14 +387,13 @@
 static bool decryptWithKeymasterKey(Keymaster& keymaster, const std::string& dir,
                                     const km::AuthorizationSet& keyParams,
                                     const km::HardwareAuthToken& authToken,
-                                    const std::string& ciphertext, KeyBuffer* message,
-                                    bool keepOld) {
+                                    const std::string& ciphertext, KeyBuffer* message) {
     auto nonce = ciphertext.substr(0, GCM_NONCE_BYTES);
     auto bodyAndMac = ciphertext.substr(GCM_NONCE_BYTES);
     auto opParams = km::AuthorizationSetBuilder().Authorization(km::TAG_NONCE,
                                                                 km::support::blob2hidlVec(nonce));
-    auto opHandle = begin(keymaster, dir, km::KeyPurpose::DECRYPT, keyParams, opParams, authToken,
-                          nullptr, keepOld);
+    auto opHandle = BeginKeymasterOp(keymaster, dir, km::KeyPurpose::DECRYPT, keyParams, opParams,
+                                     authToken, nullptr);
     if (!opHandle) return false;
     if (!opHandle.updateCompletely(bodyAndMac, message)) return false;
     if (!opHandle.finish(nullptr)) return false;
@@ -513,8 +599,7 @@
         km::AuthorizationSet keyParams;
         km::HardwareAuthToken authToken;
         std::tie(keyParams, authToken) = beginParams(auth, appId);
-        if (!encryptWithKeymasterKey(keymaster, dir, keyParams, authToken, key, &encryptedKey,
-                                     false))
+        if (!encryptWithKeymasterKey(keymaster, dir, keyParams, authToken, key, &encryptedKey))
             return false;
     } else {
         if (!encryptWithoutKeymaster(appId, key, &encryptedKey)) return false;
@@ -543,8 +628,7 @@
     return true;
 }
 
-bool retrieveKey(const std::string& dir, const KeyAuthentication& auth, KeyBuffer* key,
-                 bool keepOld) {
+bool retrieveKey(const std::string& dir, const KeyAuthentication& auth, KeyBuffer* key) {
     std::string version;
     if (!readFileToString(dir + "/" + kFn_version, &version)) return false;
     if (version != kCurrentVersion) {
@@ -569,8 +653,7 @@
         km::AuthorizationSet keyParams;
         km::HardwareAuthToken authToken;
         std::tie(keyParams, authToken) = beginParams(auth, appId);
-        if (!decryptWithKeymasterKey(keymaster, dir, keyParams, authToken, encryptedMessage, key,
-                                     keepOld))
+        if (!decryptWithKeymasterKey(keymaster, dir, keyParams, authToken, encryptedMessage, key))
             return false;
     } else {
         if (!decryptWithoutKeymaster(appId, encryptedMessage, key)) return false;
@@ -578,12 +661,13 @@
     return true;
 }
 
-static bool deleteKey(const std::string& dir) {
-    std::string kmKey;
-    if (!readFileToString(dir + "/" + kFn_keymaster_key_blob, &kmKey)) return false;
+static bool DeleteKeymasterKey(const std::string& blob_file) {
+    std::string blob;
+    if (!readFileToString(blob_file, &blob)) return false;
     Keymaster keymaster;
     if (!keymaster) return false;
-    if (!keymaster.deleteKey(kmKey)) return false;
+    LOG(DEBUG) << "Deleting key " << blob_file << " from Keymaster";
+    if (!keymaster.deleteKey(blob)) return false;
     return true;
 }
 
@@ -605,19 +689,23 @@
 
 bool destroyKey(const std::string& dir) {
     bool success = true;
-    // Try each thing, even if previous things failed.
-    bool uses_km = pathExists(dir + "/" + kFn_keymaster_key_blob);
-    if (uses_km) {
-        success &= deleteKey(dir);
-    }
+
+    CancelPendingKeyCommit(dir);
+
     auto secdiscard_cmd = std::vector<std::string>{
         kSecdiscardPath,
         "--",
         dir + "/" + kFn_encrypted_key,
         dir + "/" + kFn_secdiscardable,
     };
-    if (uses_km) {
-        secdiscard_cmd.emplace_back(dir + "/" + kFn_keymaster_key_blob);
+    // Try each thing, even if previous things failed.
+
+    for (auto& fn : {kFn_keymaster_key_blob, kFn_keymaster_key_blob_upgraded}) {
+        auto blob_file = dir + "/" + fn;
+        if (pathExists(blob_file)) {
+            success &= DeleteKeymasterKey(blob_file);
+            secdiscard_cmd.push_back(blob_file);
+        }
     }
     if (ForkExecvp(secdiscard_cmd) != 0) {
         LOG(ERROR) << "secdiscard failed";
diff --git a/KeyStorage.h b/KeyStorage.h
index 5228f08..1eb26ae 100644
--- a/KeyStorage.h
+++ b/KeyStorage.h
@@ -61,20 +61,7 @@
                         const KeyAuthentication& auth, const KeyBuffer& key);
 
 // Retrieve the key from the named directory.
-//
-// If the key is wrapped by a Keymaster key that requires an upgrade, then that
-// Keymaster key is upgraded.  If |keepOld| is false, then the upgraded
-// Keymaster key replaces the original one.  As part of this, the original is
-// deleted from Keymaster; however, if a user data checkpoint is active, this
-// part is delayed until the checkpoint is committed.
-//
-// If instead |keepOld| is true, then the upgraded key doesn't actually replace
-// the original one.  This is needed *only* if |dir| isn't located in /data and
-// a user data checkpoint is active.  In this case the caller must handle
-// replacing the original key if the checkpoint is committed, and deleting the
-// upgraded key if the checkpoint is rolled back.
-bool retrieveKey(const std::string& dir, const KeyAuthentication& auth, KeyBuffer* key,
-                 bool keepOld);
+bool retrieveKey(const std::string& dir, const KeyAuthentication& auth, KeyBuffer* key);
 
 // Securely destroy the key stored in the named directory and delete the directory.
 bool destroyKey(const std::string& dir);
diff --git a/KeyUtil.cpp b/KeyUtil.cpp
index f3a2986..9d01e1e 100644
--- a/KeyUtil.cpp
+++ b/KeyUtil.cpp
@@ -392,10 +392,10 @@
 
 bool retrieveOrGenerateKey(const std::string& key_path, const std::string& tmp_path,
                            const KeyAuthentication& key_authentication, const KeyGeneration& gen,
-                           KeyBuffer* key, bool keepOld) {
+                           KeyBuffer* key) {
     if (pathExists(key_path)) {
         LOG(DEBUG) << "Key exists, using: " << key_path;
-        if (!retrieveKey(key_path, key_authentication, key, keepOld)) return false;
+        if (!retrieveKey(key_path, key_authentication, key)) return false;
     } else {
         if (!gen.allow_gen) {
             LOG(ERROR) << "No key found in " << key_path;
diff --git a/KeyUtil.h b/KeyUtil.h
index 0f5bc93..73255a3 100644
--- a/KeyUtil.h
+++ b/KeyUtil.h
@@ -75,10 +75,10 @@
 bool evictKey(const std::string& mountpoint, const EncryptionPolicy& policy);
 
 // Retrieves the key from the named directory, or generates it if it doesn't
-// exist.  In most cases |keepOld| must be false; see retrieveKey() for details.
+// exist.
 bool retrieveOrGenerateKey(const std::string& key_path, const std::string& tmp_path,
                            const KeyAuthentication& key_authentication, const KeyGeneration& gen,
-                           KeyBuffer* key, bool keepOld);
+                           KeyBuffer* key);
 
 // Re-installs a file-based encryption key of fscrypt-provisioning type from the
 // global session keyring back into fs keyring of the mountpoint.
diff --git a/MetadataCrypt.cpp b/MetadataCrypt.cpp
index f794ee3..24c7476 100644
--- a/MetadataCrypt.cpp
+++ b/MetadataCrypt.cpp
@@ -17,17 +17,13 @@
 #include "MetadataCrypt.h"
 #include "KeyBuffer.h"
 
-#include <algorithm>
 #include <string>
-#include <thread>
-#include <vector>
 
 #include <fcntl.h>
 #include <sys/param.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 
-#include <android-base/file.h>
 #include <android-base/logging.h>
 #include <android-base/properties.h>
 #include <android-base/strings.h>
@@ -45,6 +41,8 @@
 #include "Keymaster.h"
 #include "Utils.h"
 #include "VoldUtil.h"
+#include "fs/Ext4.h"
+#include "fs/F2fs.h"
 
 namespace android {
 namespace vold {
@@ -64,9 +62,6 @@
 
 static const std::string kDmNameUserdata = "userdata";
 
-static const char* kFn_keymaster_key_blob = "keymaster_key_blob";
-static const char* kFn_keymaster_key_blob_upgraded = "keymaster_key_blob_upgraded";
-
 // The first entry in this table is the default crypto type.
 constexpr CryptoType supported_crypto_types[] = {aes_256_xts, adiantum};
 
@@ -86,10 +81,6 @@
 }
 
 static bool mount_via_fs_mgr(const char* mount_point, const char* blk_device) {
-    // We're about to mount data not verified by verified boot.  Tell Keymaster instances that early
-    // boot has ended.
-    ::android::vold::Keymaster::earlyBootEnded();
-
     // fs_mgr_do_mount runs fsck. Use setexeccon to run trusted
     // partitions in the fsck domain.
     if (setexeccon(android::vold::sFsckContext)) {
@@ -111,31 +102,6 @@
     return true;
 }
 
-// Note: It is possible to orphan a key if it is removed before deleting
-// Update this once keymaster APIs change, and we have a proper commit.
-static void commit_key(const std::string& dir) {
-    while (!android::base::WaitForProperty("vold.checkpoint_committed", "1")) {
-        LOG(ERROR) << "Wait for boot timed out";
-    }
-    Keymaster keymaster;
-    auto keyPath = dir + "/" + kFn_keymaster_key_blob;
-    auto newKeyPath = dir + "/" + kFn_keymaster_key_blob_upgraded;
-    std::string key;
-
-    if (!android::base::ReadFileToString(keyPath, &key)) {
-        LOG(ERROR) << "Failed to read old key: " << dir;
-        return;
-    }
-    if (rename(newKeyPath.c_str(), keyPath.c_str()) != 0) {
-        PLOG(ERROR) << "Unable to move upgraded key to location: " << keyPath;
-        return;
-    }
-    if (!keymaster.deleteKey(key)) {
-        LOG(ERROR) << "Key deletion failed during upgrade, continuing anyway: " << dir;
-    }
-    LOG(INFO) << "Old Key deleted: " << dir;
-}
-
 static bool read_key(const std::string& metadata_key_dir, const KeyGeneration& gen,
                      KeyBuffer* key) {
     if (metadata_key_dir.empty()) {
@@ -150,25 +116,7 @@
         return false;
     }
     auto temp = metadata_key_dir + "/tmp";
-    auto newKeyPath = dir + "/" + kFn_keymaster_key_blob_upgraded;
-    /* If we have a leftover upgraded key, delete it.
-     * We either failed an update and must return to the old key,
-     * or we rebooted before commiting the keys in a freak accident.
-     * Either way, we can re-upgrade the key if we need to.
-     */
-    Keymaster keymaster;
-    if (pathExists(newKeyPath)) {
-        if (!android::base::ReadFileToString(newKeyPath, &sKey))
-            LOG(ERROR) << "Failed to read incomplete key: " << dir;
-        else if (!keymaster.deleteKey(sKey))
-            LOG(ERROR) << "Incomplete key deletion failed, continuing anyway: " << dir;
-        else
-            unlink(newKeyPath.c_str());
-    }
-    bool needs_cp = cp_needsCheckpoint();
-    if (!retrieveOrGenerateKey(dir, temp, kEmptyAuthentication, gen, key, needs_cp)) return false;
-    if (needs_cp && pathExists(newKeyPath)) std::thread(commit_key, dir).detach();
-    return true;
+    return retrieveOrGenerateKey(dir, temp, kEmptyAuthentication, gen, key);
 }
 
 static bool get_number_of_sectors(const std::string& real_blkdev, uint64_t* nr_sec) {
@@ -256,8 +204,11 @@
 }
 
 bool fscrypt_mount_metadata_encrypted(const std::string& blk_device, const std::string& mount_point,
-                                      bool needs_encrypt) {
-    LOG(DEBUG) << "fscrypt_mount_metadata_encrypted: " << mount_point << " " << needs_encrypt;
+                                      bool needs_encrypt, bool should_format,
+                                      const std::string& fs_type) {
+    LOG(DEBUG) << "fscrypt_mount_metadata_encrypted: " << mount_point
+               << " encrypt: " << needs_encrypt << " format: " << should_format << " with "
+               << fs_type;
     auto encrypted_state = android::base::GetProperty("ro.crypto.state", "");
     if (encrypted_state != "" && encrypted_state != "encrypted") {
         LOG(DEBUG) << "fscrypt_enable_crypto got unexpected starting state: " << encrypted_state;
@@ -304,8 +255,24 @@
     if (!create_crypto_blk_dev(kDmNameUserdata, blk_device, key, options, &crypto_blkdev, &nr_sec))
         return false;
 
-    // FIXME handle the corrupt case
-    if (needs_encrypt && !encrypt_inplace(crypto_blkdev, blk_device, nr_sec, false)) return false;
+    if (needs_encrypt) {
+        if (should_format) {
+            status_t error;
+
+            if (fs_type == "ext4") {
+                error = ext4::Format(crypto_blkdev, 0, mount_point);
+            } else if (fs_type == "f2fs") {
+                error = f2fs::Format(crypto_blkdev);
+            } else {
+                LOG(ERROR) << "Unknown filesystem type: " << fs_type;
+                return false;
+            }
+            LOG(DEBUG) << "Format (err=" << error << ") " << crypto_blkdev << " on " << mount_point;
+            if (error != 0) return false;
+        } else {
+            if (!encrypt_inplace(crypto_blkdev, blk_device, nr_sec, false)) return false;
+        }
+    }
 
     LOG(DEBUG) << "Mounting metadata-encrypted filesystem:" << mount_point;
     mount_via_fs_mgr(mount_point.c_str(), crypto_blkdev.c_str());
diff --git a/MetadataCrypt.h b/MetadataCrypt.h
index 7341a08..e482765 100644
--- a/MetadataCrypt.h
+++ b/MetadataCrypt.h
@@ -26,7 +26,8 @@
 namespace vold {
 
 bool fscrypt_mount_metadata_encrypted(const std::string& block_device,
-                                      const std::string& mount_point, bool needs_encrypt);
+                                      const std::string& mount_point, bool needs_encrypt,
+                                      bool should_format, const std::string& fs_type);
 
 bool defaultkey_volume_keygen(KeyGeneration* gen);
 
diff --git a/OWNERS b/OWNERS
index deeceb7..6d8d89f 100644
--- a/OWNERS
+++ b/OWNERS
@@ -1,7 +1,9 @@
+alanstokes@google.com
+drosen@google.com
+ebiggers@google.com
+jeffv@google.com
 jsharkey@android.com
+maco@google.com
 paulcrowley@google.com
 paullawrence@google.com
-ebiggers@google.com
-drosen@google.com
 zezeozue@google.com
-maco@google.com
diff --git a/Utils.cpp b/Utils.cpp
index 7f53a92..98797b2 100644
--- a/Utils.cpp
+++ b/Utils.cpp
@@ -136,8 +136,8 @@
 }
 
 // Sets a default ACL on the directory.
-int SetDefaultAcl(const std::string& path, mode_t mode, uid_t uid, gid_t gid,
-                  std::vector<gid_t> additionalGids) {
+status_t SetDefaultAcl(const std::string& path, mode_t mode, uid_t uid, gid_t gid,
+                       std::vector<gid_t> additionalGids) {
     if (IsSdcardfsUsed()) {
         // sdcardfs magically takes care of this
         return OK;
@@ -1129,6 +1129,13 @@
     }
 }
 
+// Returns true if |path1| names the same existing file or directory as |path2|.
+bool IsSameFile(const std::string& path1, const std::string& path2) {
+    struct stat stbuf1, stbuf2;
+    if (stat(path1.c_str(), &stbuf1) != 0 || stat(path2.c_str(), &stbuf2) != 0) return false;
+    return stbuf1.st_ino == stbuf2.st_ino && stbuf1.st_dev == stbuf2.st_dev;
+}
+
 status_t RestoreconRecursive(const std::string& path) {
     LOG(DEBUG) << "Starting restorecon of " << path;
 
@@ -1581,18 +1588,8 @@
     std::string pass_through_path(
             StringPrintf("/mnt/pass_through/%d/%s", user_id, relative_upper_path.c_str()));
 
-    // Best effort unmount pass_through path
-    sSleepOnUnmount = false;
-    LOG(INFO) << "Unmounting pass_through_path " << pass_through_path;
-    auto status = ForceUnmount(pass_through_path);
-    if (status != android::OK) {
-        LOG(ERROR) << "Failed to unmount " << pass_through_path;
-    }
-    rmdir(pass_through_path.c_str());
-
     LOG(INFO) << "Unmounting fuse path " << fuse_path;
     android::status_t result = ForceUnmount(fuse_path);
-    sSleepOnUnmount = true;
     if (result != android::OK) {
         // TODO(b/135341433): MNT_DETACH is needed for fuse because umount2 can fail with EBUSY.
         // Figure out why we get EBUSY and remove this special casing if possible.
@@ -1606,6 +1603,13 @@
     }
     rmdir(fuse_path.c_str());
 
+    LOG(INFO) << "Unmounting pass_through_path " << pass_through_path;
+    auto status = ForceUnmount(pass_through_path);
+    if (status != android::OK) {
+        LOG(ERROR) << "Failed to unmount " << pass_through_path;
+    }
+    rmdir(pass_through_path.c_str());
+
     return result;
 }
 
diff --git a/Utils.h b/Utils.h
index 27889c6..a727e63 100644
--- a/Utils.h
+++ b/Utils.h
@@ -34,7 +34,6 @@
 namespace android {
 namespace vold {
 
-static const char* kPropFuse = "persist.sys.fuse";
 static const char* kVoldAppDataIsolationEnabled = "persist.sys.vold_app_data_isolation_enabled";
 static const char* kExternalStorageSdcardfs = "external_storage.sdcardfs.enabled";
 
@@ -52,6 +51,9 @@
 status_t CreateDeviceNode(const std::string& path, dev_t dev);
 status_t DestroyDeviceNode(const std::string& path);
 
+status_t SetDefaultAcl(const std::string& path, mode_t mode, uid_t uid, gid_t gid,
+                       std::vector<gid_t> additionalGids);
+
 status_t AbortFuseConnections();
 
 int SetQuotaInherit(const std::string& path);
@@ -155,6 +157,8 @@
 
 dev_t GetDevice(const std::string& path);
 
+bool IsSameFile(const std::string& path1, const std::string& path2);
+
 status_t EnsureDirExists(const std::string& path, mode_t mode, uid_t uid, gid_t gid);
 
 status_t RestoreconRecursive(const std::string& path);
diff --git a/VoldNativeService.cpp b/VoldNativeService.cpp
index 3fb4e65..9f4f7b0 100644
--- a/VoldNativeService.cpp
+++ b/VoldNativeService.cpp
@@ -33,6 +33,7 @@
 #include "Checkpoint.h"
 #include "FsCrypt.h"
 #include "IdleMaint.h"
+#include "Keymaster.h"
 #include "MetadataCrypt.h"
 #include "MoveStorage.h"
 #include "Process.h"
@@ -281,12 +282,6 @@
         return translate(res);
     }
 
-    if ((mountFlags & MOUNT_FLAG_PRIMARY) != 0) {
-        res = VolumeManager::Instance()->setPrimary(vol);
-        if (res != OK) {
-            return translate(res);
-        }
-    }
     return translate(OK);
 }
 
@@ -382,7 +377,17 @@
     ENFORCE_SYSTEM_OR_ROOT;
     ACQUIRE_LOCK;
 
-    return translate(VolumeManager::Instance()->remountAppStorageDirs(uid, pid, packageNames));
+    return translate(VolumeManager::Instance()->handleAppStorageDirs(uid, pid,
+            false /* doUnmount */, packageNames));
+}
+
+binder::Status VoldNativeService::unmountAppStorageDirs(int uid, int pid,
+        const std::vector<std::string>& packageNames) {
+    ENFORCE_SYSTEM_OR_ROOT;
+    ACQUIRE_LOCK;
+
+    return translate(VolumeManager::Instance()->handleAppStorageDirs(uid, pid,
+            true /* doUnmount */, packageNames));
 }
 
 binder::Status VoldNativeService::setupAppDir(const std::string& path, int32_t appUid) {
@@ -393,6 +398,14 @@
     return translate(VolumeManager::Instance()->setupAppDir(path, appUid));
 }
 
+binder::Status VoldNativeService::ensureAppDirsCreated(const std::vector<std::string>& paths,
+        int32_t appUid) {
+    ENFORCE_SYSTEM_OR_ROOT;
+    ACQUIRE_LOCK;
+
+    return translate(VolumeManager::Instance()->ensureAppDirsCreated(paths, appUid));
+}
+
 binder::Status VoldNativeService::fixupAppDir(const std::string& path, int32_t appUid) {
     ENFORCE_SYSTEM_OR_ROOT;
     CHECK_ARGUMENT_PATH(path);
@@ -672,15 +685,18 @@
     ENFORCE_SYSTEM_OR_ROOT;
     ACQUIRE_LOCK;
 
-    return translateBool(fscrypt_mount_metadata_encrypted(blkDevice, mountPoint, false));
+    return translateBool(
+            fscrypt_mount_metadata_encrypted(blkDevice, mountPoint, false, false, "null"));
 }
 
 binder::Status VoldNativeService::encryptFstab(const std::string& blkDevice,
-                                               const std::string& mountPoint) {
+                                               const std::string& mountPoint, bool shouldFormat,
+                                               const std::string& fsType) {
     ENFORCE_SYSTEM_OR_ROOT;
     ACQUIRE_LOCK;
 
-    return translateBool(fscrypt_mount_metadata_encrypted(blkDevice, mountPoint, true));
+    return translateBool(
+            fscrypt_mount_metadata_encrypted(blkDevice, mountPoint, true, shouldFormat, fsType));
 }
 
 binder::Status VoldNativeService::createUserKey(int32_t userId, int32_t userSerial,
@@ -879,6 +895,14 @@
     return Ok();
 }
 
+binder::Status VoldNativeService::earlyBootEnded() {
+    ENFORCE_SYSTEM_OR_ROOT;
+    ACQUIRE_LOCK;
+
+    Keymaster::earlyBootEnded();
+    return Ok();
+}
+
 binder::Status VoldNativeService::incFsEnabled(bool* _aidl_return) {
     ENFORCE_SYSTEM_OR_ROOT;
 
diff --git a/VoldNativeService.h b/VoldNativeService.h
index 9914879..47991c2 100644
--- a/VoldNativeService.h
+++ b/VoldNativeService.h
@@ -66,7 +66,10 @@
     binder::Status remountUid(int32_t uid, int32_t remountMode);
     binder::Status remountAppStorageDirs(int uid, int pid,
                                const std::vector<std::string>& packageNames);
+    binder::Status unmountAppStorageDirs(int uid, int pid,
+                               const std::vector<std::string>& packageNames);
 
+    binder::Status ensureAppDirsCreated(const std::vector<std::string>& paths, int32_t appUid);
     binder::Status setupAppDir(const std::string& path, int32_t appUid);
     binder::Status fixupAppDir(const std::string& path, int32_t appUid);
 
@@ -110,7 +113,8 @@
     binder::Status initUser0();
     binder::Status isConvertibleToFbe(bool* _aidl_return);
     binder::Status mountFstab(const std::string& blkDevice, const std::string& mountPoint);
-    binder::Status encryptFstab(const std::string& blkDevice, const std::string& mountPoint);
+    binder::Status encryptFstab(const std::string& blkDevice, const std::string& mountPoint,
+                                bool shouldFormat, const std::string& fsType);
 
     binder::Status createUserKey(int32_t userId, int32_t userSerial, bool ephemeral);
     binder::Status destroyUserKey(int32_t userId);
@@ -150,6 +154,8 @@
     binder::Status supportsFileCheckpoint(bool* _aidl_return);
     binder::Status resetCheckpoint();
 
+    binder::Status earlyBootEnded();
+
     binder::Status incFsEnabled(bool* _aidl_return) override;
     binder::Status mountIncFs(
             const std::string& backingPath, const std::string& targetDir, int32_t flags,
diff --git a/VolumeManager.cpp b/VolumeManager.cpp
index fb88fa4..f97397d 100644
--- a/VolumeManager.cpp
+++ b/VolumeManager.cpp
@@ -369,21 +369,6 @@
     return success ? 0 : -1;
 }
 
-int VolumeManager::linkPrimary(userid_t userId) {
-    if (!GetBoolProperty(android::vold::kPropFuse, false)) {
-        std::string source(mPrimary->getPath());
-        if (mPrimary->isEmulated()) {
-            source = StringPrintf("%s/%d", source.c_str(), userId);
-            fs_prepare_dir(source.c_str(), 0755, AID_ROOT, AID_ROOT);
-        }
-
-        std::string target(StringPrintf("/mnt/user/%d/primary", userId));
-        LOG(DEBUG) << "Linking " << source << " to " << target;
-        Symlink(source, target);
-    }
-    return 0;
-}
-
 void VolumeManager::destroyEmulatedVolumesForUser(userid_t userId) {
     // Destroy and remove all unstacked EmulatedVolumes for the user
     auto i = mInternalEmulatedVolumes.begin();
@@ -464,18 +449,6 @@
         createEmulatedVolumesForUser(userId);
     }
 
-    if (!GetBoolProperty(android::vold::kPropFuse, false)) {
-        // Note that sometimes the system will spin up processes from Zygote
-        // before actually starting the user, so we're okay if Zygote
-        // already created this directory.
-        std::string path(StringPrintf("%s/%d", kPathUserMount, userId));
-        fs_prepare_dir(path.c_str(), 0755, AID_ROOT, AID_ROOT);
-
-        if (mPrimary) {
-            linkPrimary(userId);
-        }
-    }
-
     mStartedUsers.insert(userId);
 
     createPendingDisksIfNeeded();
@@ -512,14 +485,6 @@
     return 0;
 }
 
-int VolumeManager::setPrimary(const std::shared_ptr<android::vold::VolumeBase>& vol) {
-    mPrimary = vol;
-    for (userid_t userId : mStartedUsers) {
-        linkPrimary(userId);
-    }
-    return 0;
-}
-
 // This code is executed after a fork so it's very important that the set of
 // methods we call here is strictly limited.
 //
@@ -718,15 +683,42 @@
     return true;
 }
 
-int VolumeManager::remountUid(uid_t uid, int32_t mountMode) {
-    if (GetBoolProperty(android::vold::kPropFuse, false)) {
-        // TODO(135341433): Implement fuse specific logic.
-        return 0;
+// In each app's namespace, unmount obb and data dirs
+static bool umountStorageDirs(int nsFd, const char* android_data_dir, const char* android_obb_dir,
+        int uid, const char* targets[], int size) {
+    // This code is executed after a fork so it's very important that the set of
+    // methods we call here is strictly limited.
+    if (setns(nsFd, CLONE_NEWNS) != 0) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to setns %s", strerror(errno));
+        return false;
     }
-    return scanProcProcesses(uid, static_cast<userid_t>(-1),
-            forkAndRemountChild, &mountMode) ? 0 : -1;
-}
 
+    // Unmount of Android/data/foo needs to be done before Android/data below.
+    bool result = true;
+    for (int i = 0; i < size; i++) {
+        if (TEMP_FAILURE_RETRY(umount2(targets[i], MNT_DETACH)) < 0 && errno != EINVAL &&
+                errno != ENOENT) {
+            async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to umount %s: %s",
+                                  targets[i], strerror(errno));
+            result = false;
+        }
+    }
+
+    // Mount tmpfs on Android/data and Android/obb
+    if (TEMP_FAILURE_RETRY(umount2(android_data_dir, MNT_DETACH)) < 0 && errno != EINVAL &&
+                errno != ENOENT) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to umount %s :%s",
+                        android_data_dir, strerror(errno));
+        result = false;
+    }
+    if (TEMP_FAILURE_RETRY(umount2(android_obb_dir, MNT_DETACH)) < 0 && errno != EINVAL &&
+                errno != ENOENT) {
+        async_safe_format_log(ANDROID_LOG_ERROR, "vold", "Failed to umount %s :%s",
+                android_obb_dir, strerror(errno));
+        result = false;
+    }
+    return result;
+}
 
 // In each app's namespace, mount tmpfs on obb and data dir, and bind mount obb and data
 // package dirs.
@@ -786,8 +778,8 @@
             userId, dirName.c_str(), packageName.c_str());
 }
 
-// Fork the process and remount storage
-bool VolumeManager::forkAndRemountStorage(int uid, int pid,
+// Fork the process and remount / unmount app data and obb dirs
+bool VolumeManager::forkAndRemountStorage(int uid, int pid, bool doUnmount,
                                           const std::vector<std::string>& packageNames) {
     userid_t userId = multiuser_get_user_id(uid);
     std::string mnt_path = StringPrintf("/proc/%d/ns/mnt", pid);
@@ -821,17 +813,18 @@
     }
 
     for (int i = 0; i < size; i++) {
-        auto status = EnsureDirExists(sources_cstr[i], 0771, AID_MEDIA_RW, AID_MEDIA_RW);
-        if (status != OK) {
-            PLOG(ERROR) << "Failed to create dir: " << sources_cstr[i];
-            return false;
-        }
         // Make sure /storage/emulated/... paths are setup correctly
-        status = setupAppDir(targets_cstr[i], uid, false /* fixupExistingOnly */);
+        // This needs to be done before EnsureDirExists to ensure Android/ is created.
+        auto status = setupAppDir(targets_cstr[i], uid, false /* fixupExistingOnly */);
         if (status != OK) {
             PLOG(ERROR) << "Failed to create dir: " << targets_cstr[i];
             return false;
         }
+        status = EnsureDirExists(sources_cstr[i], 0771, AID_MEDIA_RW, AID_MEDIA_RW);
+        if (status != OK) {
+            PLOG(ERROR) << "Failed to create dir: " << sources_cstr[i];
+            return false;
+        }
     }
 
     char android_data_dir[PATH_MAX];
@@ -843,11 +836,20 @@
     // Fork a child to mount Android/obb android Android/data dirs, as we don't want it to affect
     // original vold process mount namespace.
     if (!(child = fork())) {
-        if (remountStorageDirs(nsFd, android_data_dir, android_obb_dir, uid,
-                sources_cstr, targets_cstr, size)) {
-            _exit(0);
+        if (doUnmount) {
+            if (umountStorageDirs(nsFd, android_data_dir, android_obb_dir, uid,
+                    targets_cstr, size)) {
+                _exit(0);
+            } else {
+                _exit(1);
+            }
         } else {
-            _exit(1);
+            if (remountStorageDirs(nsFd, android_data_dir, android_obb_dir, uid,
+                    sources_cstr, targets_cstr, size)) {
+                _exit(0);
+            } else {
+                _exit(1);
+            }
         }
     }
 
@@ -872,11 +874,8 @@
     return true;
 }
 
-int VolumeManager::remountAppStorageDirs(int uid, int pid,
-        const std::vector<std::string>& packageNames) {
-    if (!GetBoolProperty(android::vold::kPropFuse, false)) {
-        return 0;
-    }
+int VolumeManager::handleAppStorageDirs(int uid, int pid,
+        bool doUnmount, const std::vector<std::string>& packageNames) {
     // Only run the remount if fuse is mounted for that user.
     userid_t userId = multiuser_get_user_id(uid);
     bool fuseMounted = false;
@@ -890,7 +889,7 @@
         }
     }
     if (fuseMounted) {
-        forkAndRemountStorage(uid, pid, packageNames);
+        forkAndRemountStorage(uid, pid, doUnmount, packageNames);
     }
     return 0;
 }
@@ -984,7 +983,20 @@
     return 0;
 }
 
-int VolumeManager::setupAppDir(const std::string& path, int32_t appUid, bool fixupExistingOnly) {
+int VolumeManager::ensureAppDirsCreated(const std::vector<std::string>& paths, int32_t appUid) {
+    int size = paths.size();
+    for (int i = 0; i < size; i++) {
+        int result = setupAppDir(paths[i], appUid, false /* fixupExistingOnly */,
+                true /* skipIfDirExists */);
+        if (result != OK) {
+            return result;
+        }
+    }
+    return OK;
+}
+
+int VolumeManager::setupAppDir(const std::string& path, int32_t appUid, bool fixupExistingOnly,
+        bool skipIfDirExists) {
     // Only offer to create directories for paths managed by vold
     if (!StartsWith(path, "/storage/")) {
         LOG(ERROR) << "Failed to find mounted volume for " << path;
@@ -1029,11 +1041,18 @@
 
     const std::string volumeRoot = volume->getRootPath();  // eg /data/media/0
 
-    if (fixupExistingOnly && (access(lowerPath.c_str(), F_OK) != 0)) {
+    const int access_result = access(lowerPath.c_str(), F_OK);
+    if (fixupExistingOnly && access_result != 0) {
         // Nothing to fixup
         return OK;
     }
 
+    if (skipIfDirExists && access_result == 0) {
+        // It's safe to assume it's ok as it will be used for zygote to bind mount dir only,
+        // which the dir doesn't need to have correct permission for now yet.
+        return OK;
+    }
+
     if (volume->getType() == VolumeBase::Type::kPublic) {
         // On public volumes, we don't need to setup permissions, as everything goes through
         // FUSE; just create the dirs and be done with it.
@@ -1056,8 +1075,42 @@
                              int32_t ownerGid, std::string* outVolId) {
     int id = mNextObbId++;
 
+    std::string lowerSourcePath;
+
+    // Convert to lower filesystem path
+    if (StartsWith(sourcePath, "/storage/")) {
+        auto filter_fn = [&](const VolumeBase& vol) {
+            if (vol.getState() != VolumeBase::State::kMounted) {
+                // The volume must be mounted
+                return false;
+            }
+            if ((vol.getMountFlags() & VolumeBase::MountFlags::kVisible) == 0) {
+                // and visible
+                return false;
+            }
+            if (vol.getInternalPath().empty()) {
+                return false;
+            }
+            if (!sourcePath.empty() && StartsWith(sourcePath, vol.getPath())) {
+                return true;
+            }
+
+            return false;
+        };
+        auto volume = findVolumeWithFilter(filter_fn);
+        if (volume == nullptr) {
+            LOG(ERROR) << "Failed to find mounted volume for " << sourcePath;
+            return -EINVAL;
+        } else {
+            lowerSourcePath =
+                    volume->getInternalPath() + sourcePath.substr(volume->getPath().length());
+        }
+    } else {
+        lowerSourcePath = sourcePath;
+    }
+
     auto vol = std::shared_ptr<android::vold::VolumeBase>(
-        new android::vold::ObbVolume(id, sourcePath, sourceKey, ownerGid));
+            new android::vold::ObbVolume(id, lowerSourcePath, sourceKey, ownerGid));
     vol->create();
 
     mObbVolumes.push_back(vol);
@@ -1086,9 +1139,14 @@
     auto vol = std::shared_ptr<android::vold::StubVolume>(
             new android::vold::StubVolume(stubId, sourcePath, mountPath, fsType, fsUuid, fsLabel));
 
-    int32_t passedFlags = android::vold::Disk::Flags::kStub;
+    int32_t passedFlags = 0;
     passedFlags |= (flags & android::vold::Disk::Flags::kUsb);
     passedFlags |= (flags & android::vold::Disk::Flags::kSd);
+    if (flags & android::vold::Disk::Flags::kStubVisible) {
+        passedFlags |= (flags & android::vold::Disk::Flags::kStubVisible);
+    } else {
+        passedFlags |= (flags & android::vold::Disk::Flags::kStubInvisible);
+    }
     // StubDisk doesn't have device node corresponds to it. So, a fake device
     // number is used.
     auto disk = std::shared_ptr<android::vold::Disk>(
diff --git a/VolumeManager.h b/VolumeManager.h
index 54a2443..3573b1a 100644
--- a/VolumeManager.h
+++ b/VolumeManager.h
@@ -114,10 +114,9 @@
     void createPendingDisksIfNeeded();
     int onSecureKeyguardStateChanged(bool isShowing);
 
-    int setPrimary(const std::shared_ptr<android::vold::VolumeBase>& vol);
-
-    int remountUid(uid_t uid, int32_t remountMode);
-    int remountAppStorageDirs(int uid, int pid, const std::vector<std::string>& packageNames);
+    int remountUid(uid_t uid, int32_t remountMode) { return 0; }
+    int handleAppStorageDirs(int uid, int pid,
+            bool doUnmount, const std::vector<std::string>& packageNames);
 
     /* Aborts all FUSE filesystems, in case the FUSE daemon is no longer up. */
     int abortFuse();
@@ -131,7 +130,8 @@
     int updateVirtualDisk();
     int setDebug(bool enable);
 
-    bool forkAndRemountStorage(int uid, int pid, const std::vector<std::string>& packageNames);
+    bool forkAndRemountStorage(int uid, int pid, bool doUnmount,
+        const std::vector<std::string>& packageNames);
 
     static VolumeManager* Instance();
 
@@ -165,12 +165,16 @@
      * files in the passed in path, but only if that path exists; if it doesn't
      * exist, this function doesn't create them.
      *
+     * If skipIfDirExists is set, we will not fix any existing dirs, we will
+     * only create app dirs if it doesn't exist.
+     *
      * Validates that given paths are absolute and that they contain no relative
      * "." or ".." paths or symlinks.  Last path segment is treated as filename
      * and ignored, unless the path ends with "/".  Also ensures that path
      * belongs to a volume managed by vold.
      */
-    int setupAppDir(const std::string& path, int32_t appUid, bool fixupExistingOnly = false);
+    int setupAppDir(const std::string& path, int32_t appUid, bool fixupExistingOnly = false,
+            bool skipIfDirExists = false);
 
     /**
      * Fixes up an existing application directory, as if it was created with
@@ -179,6 +183,9 @@
      */
     int fixupAppDir(const std::string& path, int32_t appUid);
 
+    // Called before zygote starts to ensure dir exists so zygote can bind mount them.
+    int ensureAppDirsCreated(const std::vector<std::string>& paths, int32_t appUid);
+
     int createObb(const std::string& path, const std::string& key, int32_t ownerGid,
                   std::string* outVolId);
     int destroyObb(const std::string& volId);
diff --git a/binder/android/os/IVold.aidl b/binder/android/os/IVold.aidl
index 1d6225f..19ce9ba 100644
--- a/binder/android/os/IVold.aidl
+++ b/binder/android/os/IVold.aidl
@@ -54,9 +54,11 @@
 
     void remountUid(int uid, int remountMode);
     void remountAppStorageDirs(int uid, int pid, in @utf8InCpp String[] packageNames);
+    void unmountAppStorageDirs(int uid, int pid, in @utf8InCpp String[] packageNames);
 
     void setupAppDir(@utf8InCpp String path, int appUid);
     void fixupAppDir(@utf8InCpp String path, int appUid);
+    void ensureAppDirsCreated(in @utf8InCpp String[] paths, int appUid);
 
     @utf8InCpp String createObb(@utf8InCpp String sourcePath, @utf8InCpp String sourceKey,
                                 int ownerGid);
@@ -87,7 +89,7 @@
     void initUser0();
     boolean isConvertibleToFbe();
     void mountFstab(@utf8InCpp String blkDevice, @utf8InCpp String mountPoint);
-    void encryptFstab(@utf8InCpp String blkDevice, @utf8InCpp String mountPoint);
+    void encryptFstab(@utf8InCpp String blkDevice, @utf8InCpp String mountPoint, boolean shouldFormat, @utf8InCpp String fsType);
 
     void createUserKey(int userId, int userSerial, boolean ephemeral);
     void destroyUserKey(int userId);
@@ -126,6 +128,7 @@
     boolean supportsFileCheckpoint();
     void resetCheckpoint();
 
+    void earlyBootEnded();
     @utf8InCpp String createStubVolume(@utf8InCpp String sourcePath,
             @utf8InCpp String mountPath, @utf8InCpp String fsType,
             @utf8InCpp String fsUuid, @utf8InCpp String fsLabel, int flags);
@@ -166,7 +169,6 @@
 
     const int STORAGE_FLAG_DE = 1;
     const int STORAGE_FLAG_CE = 2;
-    const int STORAGE_FLAG_LEVEL_FROM_USER = 4;
 
     const int REMOUNT_MODE_NONE = 0;
     const int REMOUNT_MODE_DEFAULT = 1;
diff --git a/fs/Exfat.cpp b/fs/Exfat.cpp
index 34f1024..7782dd3 100644
--- a/fs/Exfat.cpp
+++ b/fs/Exfat.cpp
@@ -41,7 +41,7 @@
 status_t Check(const std::string& source) {
     std::vector<std::string> cmd;
     cmd.push_back(kFsckPath);
-    cmd.push_back("-a");
+    cmd.push_back("-y");
     cmd.push_back(source);
 
     int rc = ForkExecvp(cmd, nullptr, sFsckUntrustedContext);
diff --git a/model/Disk.h b/model/Disk.h
index 99c98fc..16476dc 100644
--- a/model/Disk.h
+++ b/model/Disk.h
@@ -53,9 +53,12 @@
         kUsb = 1 << 3,
         /* Flag that disk is EMMC internal */
         kEmmc = 1 << 4,
-        /* Flag that disk is Stub disk, i.e., disk that is managed from outside
-         * Android (e.g., ARC++). */
-        kStub = 1 << 5,
+        /* Flag that disk is an invisible Stub disk, i.e., disk that is managed from outside
+         * Android (e.g., ARC++) and invisible to apps. */
+        kStubInvisible = 1 << 5,
+        /* Flag that disk is a visible Stub disk, i.e., disk that is managed from outside
+         * Android (e.g., ARC++) and visible to apps. */
+        kStubVisible = 1 << 6,
     };
 
     const std::string& getId() const { return mId; }
@@ -120,7 +123,7 @@
 
     int getMaxMinors();
 
-    bool isStub() { return mFlags & kStub; }
+    bool isStub() { return (mFlags & kStubInvisible) || (mFlags & kStubVisible); }
 
     DISALLOW_COPY_AND_ASSIGN(Disk);
 };
diff --git a/model/EmulatedVolume.cpp b/model/EmulatedVolume.cpp
index db93bc2..4a77846 100644
--- a/model/EmulatedVolume.cpp
+++ b/model/EmulatedVolume.cpp
@@ -301,8 +301,6 @@
 
     dev_t before = GetDevice(mSdcardFsFull);
 
-    bool isFuse = base::GetBoolProperty(kPropFuse, false);
-
     // Mount sdcardfs regardless of FUSE, since we need it to bind-mount on top of the
     // FUSE volume for various reasons.
     if (mUseSdcardFs && getMountUserId() == 0) {
@@ -350,7 +348,7 @@
         sdcardFsPid = 0;
     }
 
-    if (isFuse && isVisible) {
+    if (isVisible) {
         // Make sure we unmount sdcardfs if we bail out with an error below
         auto sdcardfs_unmounter = [&]() {
             LOG(INFO) << "sdcardfs_unmounter scope_guard running";
diff --git a/model/PublicVolume.cpp b/model/PublicVolume.cpp
index d40e3e3..12e31ff 100644
--- a/model/PublicVolume.cpp
+++ b/model/PublicVolume.cpp
@@ -227,39 +227,36 @@
         TEMP_FAILURE_RETRY(waitpid(sdcardFsPid, nullptr, 0));
     }
 
-    bool isFuse = base::GetBoolProperty(kPropFuse, false);
-    if (isFuse) {
-        // We need to mount FUSE *after* sdcardfs, since the FUSE daemon may depend
-        // on sdcardfs being up.
-        LOG(INFO) << "Mounting public fuse volume";
-        android::base::unique_fd fd;
-        int user_id = getMountUserId();
-        int result = MountUserFuse(user_id, getInternalPath(), stableName, &fd);
+    // We need to mount FUSE *after* sdcardfs, since the FUSE daemon may depend
+    // on sdcardfs being up.
+    LOG(INFO) << "Mounting public fuse volume";
+    android::base::unique_fd fd;
+    int user_id = getMountUserId();
+    int result = MountUserFuse(user_id, getInternalPath(), stableName, &fd);
 
-        if (result != 0) {
-            LOG(ERROR) << "Failed to mount public fuse volume";
-            doUnmount();
-            return -result;
-        }
-
-        mFuseMounted = true;
-        auto callback = getMountCallback();
-        if (callback) {
-            bool is_ready = false;
-            callback->onVolumeChecking(std::move(fd), getPath(), getInternalPath(), &is_ready);
-            if (!is_ready) {
-                LOG(ERROR) << "Failed to complete public volume mount";
-                doUnmount();
-                return -EIO;
-            }
-        }
-
-        ConfigureReadAheadForFuse(GetFuseMountPathForUser(user_id, stableName), 256u);
-
-        // See comment in model/EmulatedVolume.cpp
-        ConfigureMaxDirtyRatioForFuse(GetFuseMountPathForUser(user_id, stableName), 40u);
+    if (result != 0) {
+        LOG(ERROR) << "Failed to mount public fuse volume";
+        doUnmount();
+        return -result;
     }
 
+    mFuseMounted = true;
+    auto callback = getMountCallback();
+    if (callback) {
+        bool is_ready = false;
+        callback->onVolumeChecking(std::move(fd), getPath(), getInternalPath(), &is_ready);
+        if (!is_ready) {
+            LOG(ERROR) << "Failed to complete public volume mount";
+            doUnmount();
+            return -EIO;
+        }
+    }
+
+    ConfigureReadAheadForFuse(GetFuseMountPathForUser(user_id, stableName), 256u);
+
+    // See comment in model/EmulatedVolume.cpp
+    ConfigureMaxDirtyRatioForFuse(GetFuseMountPathForUser(user_id, stableName), 40u);
+
     return OK;
 }
 
diff --git a/vdc.cpp b/vdc.cpp
index 11562e7..47d98de 100644
--- a/vdc.cpp
+++ b/vdc.cpp
@@ -31,9 +31,10 @@
 #include "android/os/IVold.h"
 
 #include <android-base/logging.h>
+#include <android-base/parsebool.h>
 #include <android-base/parseint.h>
-#include <android-base/strings.h>
 #include <android-base/stringprintf.h>
+#include <android-base/strings.h>
 #include <binder/IServiceManager.h>
 #include <binder/Status.h>
 
@@ -107,8 +108,12 @@
         checkStatus(args, vold->reset());
     } else if (args[0] == "cryptfs" && args[1] == "mountFstab" && args.size() == 4) {
         checkStatus(args, vold->mountFstab(args[2], args[3]));
-    } else if (args[0] == "cryptfs" && args[1] == "encryptFstab" && args.size() == 4) {
-        checkStatus(args, vold->encryptFstab(args[2], args[3]));
+    } else if (args[0] == "cryptfs" && args[1] == "encryptFstab" && args.size() == 6) {
+        auto shouldFormat = android::base::ParseBool(args[4]);
+        if (shouldFormat == android::base::ParseBoolResult::kError) exit(EINVAL);
+        checkStatus(args, vold->encryptFstab(args[2], args[3],
+                                             shouldFormat == android::base::ParseBoolResult::kTrue,
+                                             args[5]));
     } else if (args[0] == "checkpoint" && args[1] == "supportsCheckpoint" && args.size() == 2) {
         bool supported = false;
         checkStatus(args, vold->supportsCheckpoint(&supported));
@@ -152,6 +157,8 @@
         checkStatus(args, vold->abortChanges(args[2], retry != 0));
     } else if (args[0] == "checkpoint" && args[1] == "resetCheckpoint") {
         checkStatus(args, vold->resetCheckpoint());
+    } else if (args[0] == "keymaster" && args[1] == "earlyBootEnded") {
+        checkStatus(args, vold->earlyBootEnded());
     } else {
         LOG(ERROR) << "Raw commands are no longer supported";
         exit(EINVAL);
diff --git a/vold_prepare_subdirs.cpp b/vold_prepare_subdirs.cpp
index 0283614..e2afb81 100644
--- a/vold_prepare_subdirs.cpp
+++ b/vold_prepare_subdirs.cpp
@@ -166,13 +166,9 @@
 static bool prepare_subdirs(const std::string& volume_uuid, int user_id, int flags) {
     struct selabel_handle* sehandle = selinux_android_file_context_handle();
 
-    const uid_t user_for_level =
-            (flags & android::os::IVold::STORAGE_FLAG_LEVEL_FROM_USER) ? user_id : -1;
-
     if (flags & android::os::IVold::STORAGE_FLAG_DE) {
         auto user_de_path = android::vold::BuildDataUserDePath(volume_uuid, user_id);
-        if (!prepare_dir_for_user(sehandle, 0771, AID_SYSTEM, AID_SYSTEM, user_de_path,
-                                  user_for_level)) {
+        if (!prepare_dir_for_user(sehandle, 0771, AID_SYSTEM, AID_SYSTEM, user_de_path, user_id)) {
             return false;
         }
 
@@ -187,7 +183,7 @@
 
             auto profiles_de_path = android::vold::BuildDataProfilesDePath(user_id);
             if (!prepare_dir_for_user(sehandle, 0771, AID_SYSTEM, AID_SYSTEM, profiles_de_path,
-                                      user_for_level)) {
+                                      user_id)) {
                 return false;
             }
 
@@ -203,8 +199,7 @@
     }
     if (flags & android::os::IVold::STORAGE_FLAG_CE) {
         auto user_ce_path = android::vold::BuildDataUserCePath(volume_uuid, user_id);
-        if (!prepare_dir_for_user(sehandle, 0771, AID_SYSTEM, AID_SYSTEM, user_ce_path,
-                                  user_for_level)) {
+        if (!prepare_dir_for_user(sehandle, 0771, AID_SYSTEM, AID_SYSTEM, user_ce_path, user_id)) {
             return false;
         }