Merge changes from topic "fuse_bindmount"

* changes:
  Use a regular set for started users.
  When unmounting, only kill user-specific paths.
  Bind mount Android/ directory in FUSE.
  Use sdcardfs for pass-through mounts.
diff --git a/Utils.cpp b/Utils.cpp
index af93824..d483418 100644
--- a/Utils.cpp
+++ b/Utils.cpp
@@ -995,16 +995,11 @@
     std::string pass_through_path(
             StringPrintf("%s/%s", pre_pass_through_path.c_str(), relative_upper_path.c_str()));
 
-    // Force remove the existing mount before we attempt to prepare the
-    // directory. If we have a dangling mount, then PrepareDir may fail if the
-    // indirection to FUSE doesn't work.
-    android::status_t result = UnmountUserFuse(pass_through_path, fuse_path);
-    if (result != android::OK) {
-        return -1;
-    }
+    std::string sdcardfs_path(
+            StringPrintf("/mnt/runtime/full/%s", relative_upper_path.c_str()));
 
     // Create directories.
-    result = PrepareDir(pre_fuse_path, 0700, AID_ROOT, AID_ROOT);
+    auto result = PrepareDir(pre_fuse_path, 0700, AID_ROOT, AID_ROOT);
     if (result != android::OK) {
         PLOG(ERROR) << "Failed to prepare directory " << pre_fuse_path;
         return -1;
@@ -1063,14 +1058,27 @@
         PLOG(ERROR) << "Failed to mount " << fuse_path;
         return -errno;
     }
-    LOG(INFO) << "Bind mounting to " << absolute_lower_path;
-    return BindMount(absolute_lower_path, pass_through_path);
+
+    LOG(INFO) << "Bind mounting " << sdcardfs_path << " to " << pass_through_path;
+    return BindMount(sdcardfs_path, pass_through_path);
 }
 
-status_t UnmountUserFuse(const std::string& pass_through_path, const std::string& fuse_path) {
+status_t UnmountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
+                         const std::string& relative_upper_path) {
+    std::string fuse_path(StringPrintf("/mnt/user/%d/%s", user_id, relative_upper_path.c_str()));
+    std::string pass_through_path(
+            StringPrintf("/mnt/pass_through/%d/%s", user_id, relative_upper_path.c_str()));
+
     // Best effort unmount pass_through path
     sSleepOnUnmount = false;
-    ForceUnmount(pass_through_path);
+    LOG(INFO) << "Unmounting pass_through_path " << pass_through_path;
+    auto status = ForceUnmount(pass_through_path);
+    if (status != android::OK) {
+        LOG(ERROR) << "Failed to unmount " << pass_through_path;
+    }
+    rmdir(pass_through_path.c_str());
+
+    LOG(INFO) << "Unmounting fuse path " << fuse_path;
     android::status_t result = ForceUnmount(fuse_path);
     sSleepOnUnmount = true;
     if (result != android::OK) {
@@ -1082,8 +1090,10 @@
             PLOG(ERROR) << "Failed to unmount with MNT_DETACH " << fuse_path;
             return -errno;
         }
-        return android::OK;
+        result = android::OK;
     }
+    rmdir(fuse_path.c_str());
+
     return result;
 }
 
diff --git a/Utils.h b/Utils.h
index 375e175..5bb2855 100644
--- a/Utils.h
+++ b/Utils.h
@@ -154,7 +154,8 @@
 status_t MountUserFuse(userid_t user_id, const std::string& absolute_lower_path,
                        const std::string& relative_upper_path, android::base::unique_fd* fuse_fd);
 
-status_t UnmountUserFuse(const std::string& pass_through_path, const std::string& fuse_path);
+status_t UnmountUserFuse(userid_t userId, const std::string& absolute_lower_path,
+                         const std::string& relative_upper_path);
 
 }  // namespace vold
 }  // namespace android
diff --git a/VolumeManager.h b/VolumeManager.h
index aff5aaf..fad3c00 100644
--- a/VolumeManager.h
+++ b/VolumeManager.h
@@ -23,6 +23,7 @@
 
 #include <list>
 #include <mutex>
+#include <set>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -84,7 +85,7 @@
 
     void listVolumes(android::vold::VolumeBase::Type type, std::list<std::string>& list) const;
 
-    const std::unordered_set<userid_t>& getStartedUsers() const { return mStartedUsers; }
+    const std::set<userid_t>& getStartedUsers() const { return mStartedUsers; }
 
     int forgetPartition(const std::string& partGuid, const std::string& fsUuid);
 
@@ -159,7 +160,9 @@
     std::list<std::shared_ptr<android::vold::VolumeBase>> mInternalEmulatedVolumes;
 
     std::unordered_map<userid_t, int> mAddedUsers;
-    std::unordered_set<userid_t> mStartedUsers;
+    // This needs to be a regular set because we care about the ordering here;
+    // user 0 should always go first, because it is responsible for sdcardfs.
+    std::set<userid_t> mStartedUsers;
 
     std::string mVirtualDiskPath;
     std::shared_ptr<android::vold::Disk> mVirtualDisk;
diff --git a/model/EmulatedVolume.cpp b/model/EmulatedVolume.cpp
index 705bf79..7fbfcec 100644
--- a/model/EmulatedVolume.cpp
+++ b/model/EmulatedVolume.cpp
@@ -59,13 +59,59 @@
 
 EmulatedVolume::~EmulatedVolume() {}
 
-status_t EmulatedVolume::doMount() {
+std::string EmulatedVolume::getLabel() {
     // We could have migrated storage to an adopted private volume, so always
     // call primary storage "emulated" to avoid media rescans.
-    std::string label = mLabel;
     if (getMountFlags() & MountFlags::kPrimary) {
-        label = "emulated";
+        return "emulated";
+    } else {
+        return mLabel;
     }
+}
+
+static status_t mountFuseBindMounts(int userId, const std::string& label) {
+    // TODO(b/134706060) we don't actually want to mount the "write" view by
+    // default, since it gives write access to all OBB dirs.
+    std::string androidSource(
+            StringPrintf("/mnt/runtime/write/%s/%d/Android", label.c_str(), userId));
+    std::string androidTarget(
+            StringPrintf("/mnt/user/%d/%s/%d/Android", userId, label.c_str(), userId));
+
+    if (access(androidSource.c_str(), F_OK) != 0) {
+        // Android path may not exist yet if users has just been created; create it on
+        // the lower fs.
+        if (fs_prepare_dir(androidSource.c_str(), 0771, AID_ROOT, AID_ROOT) != 0) {
+            PLOG(ERROR) << "Failed to create " << androidSource;
+            return -errno;
+        }
+    }
+    LOG(INFO) << "Bind mounting " << androidSource << " on " << androidTarget;
+    auto status = BindMount(androidSource, androidTarget);
+    if (status != OK) {
+        return status;
+    }
+    LOG(INFO) << "Bind mounted " << androidSource << " on " << androidTarget;
+
+    return OK;
+}
+
+static status_t unmountFuseBindMounts(int userId, const std::string& label) {
+    std::string androidTarget(
+            StringPrintf("/mnt/user/%d/%s/%d/Android", userId, label.c_str(), userId));
+
+    LOG(INFO) << "Unmounting " << androidTarget;
+    auto status = UnmountTree(androidTarget);
+    if (status != OK) {
+        return status;
+    }
+    LOG(INFO) << "Unmounted " << androidTarget;
+
+    return OK;
+}
+
+status_t EmulatedVolume::doMount() {
+    std::string label = getLabel();
+    bool isVisible = getMountFlags() & MountFlags::kVisible;
 
     mSdcardFsDefault = StringPrintf("/mnt/runtime/default/%s", label.c_str());
     mSdcardFsRead = StringPrintf("/mnt/runtime/read/%s", label.c_str());
@@ -87,7 +133,53 @@
 
     bool isFuse = base::GetBoolProperty(kPropFuseSnapshot, false);
 
-    if (isFuse) {
+    // Mount sdcardfs regardless of FUSE, since we need it to bind-mount on top of the
+    // FUSE volume for various reasons.
+    if (getMountUserId() == 0) {
+        LOG(INFO) << "Executing sdcardfs";
+        int sdcardFsPid;
+        if (!(sdcardFsPid = fork())) {
+            // clang-format off
+            if (execl(kSdcardFsPath, kSdcardFsPath,
+                    "-u", "1023", // AID_MEDIA_RW
+                    "-g", "1023", // AID_MEDIA_RW
+                    "-m",
+                    "-w",
+                    "-G",
+                    "-i",
+                    "-o",
+                    mRawPath.c_str(),
+                    label.c_str(),
+                    NULL)) {
+                // clang-format on
+                PLOG(ERROR) << "Failed to exec";
+            }
+
+            LOG(ERROR) << "sdcardfs exiting";
+            _exit(1);
+        }
+
+        if (sdcardFsPid == -1) {
+            PLOG(ERROR) << getId() << " failed to fork";
+            return -errno;
+        }
+
+        nsecs_t start = systemTime(SYSTEM_TIME_BOOTTIME);
+        while (before == GetDevice(mSdcardFsFull)) {
+            LOG(DEBUG) << "Waiting for sdcardfs to spin up...";
+            usleep(50000);  // 50ms
+
+            nsecs_t now = systemTime(SYSTEM_TIME_BOOTTIME);
+            if (nanoseconds_to_milliseconds(now - start) > 5000) {
+                LOG(WARNING) << "Timed out while waiting for sdcardfs to spin up";
+                return -ETIMEDOUT;
+            }
+        }
+        /* sdcardfs will have exited already. The filesystem will still be running */
+        TEMP_FAILURE_RETRY(waitpid(sdcardFsPid, nullptr, 0));
+        sdcardFsPid = 0;
+    }
+    if (isFuse && isVisible) {
         LOG(INFO) << "Mounting emulated fuse volume";
         android::base::unique_fd fd;
         int user_id = getMountUserId();
@@ -98,6 +190,7 @@
             return -result;
         }
 
+        mFuseMounted = true;
         auto callback = getMountCallback();
         if (callback) {
             bool is_ready = false;
@@ -107,87 +200,44 @@
             }
         }
 
-        return OK;
-    } else if (getMountUserId() != 0) {
-        // For sdcardfs, only mount for user 0, since user 0 will always be running
-        // and the paths don't change for different users. Trying to double mount
-        // will cause sepolicy to scream since sdcardfs prevents 'mounton'
-        return OK;
+        // Only do the bind-mounts when we know for sure the FUSE daemon can resolve the path.
+        return mountFuseBindMounts(user_id, label);
     }
 
-    LOG(INFO) << "Executing sdcardfs";
-    int sdcardFsPid;
-    if (!(sdcardFsPid = fork())) {
-        // clang-format off
-        if (execl(kSdcardFsPath, kSdcardFsPath,
-                "-u", "1023", // AID_MEDIA_RW
-                "-g", "1023", // AID_MEDIA_RW
-                "-m",
-                "-w",
-                "-G",
-                "-i",
-                "-o",
-                mRawPath.c_str(),
-                label.c_str(),
-                NULL)) {
-            // clang-format on
-            PLOG(ERROR) << "Failed to exec";
-        }
-
-        LOG(ERROR) << "sdcardfs exiting";
-        _exit(1);
-    }
-
-    if (sdcardFsPid == -1) {
-        PLOG(ERROR) << getId() << " failed to fork";
-        return -errno;
-    }
-
-    nsecs_t start = systemTime(SYSTEM_TIME_BOOTTIME);
-    while (before == GetDevice(mSdcardFsFull)) {
-        LOG(DEBUG) << "Waiting for sdcardfs to spin up...";
-        usleep(50000);  // 50ms
-
-        nsecs_t now = systemTime(SYSTEM_TIME_BOOTTIME);
-        if (nanoseconds_to_milliseconds(now - start) > 5000) {
-            LOG(WARNING) << "Timed out while waiting for sdcardfs to spin up";
-            return -ETIMEDOUT;
-        }
-    }
-    /* sdcardfs will have exited already. The filesystem will still be running */
-    TEMP_FAILURE_RETRY(waitpid(sdcardFsPid, nullptr, 0));
-
     return OK;
 }
 
 status_t EmulatedVolume::doUnmount() {
-    // Unmount the storage before we kill the FUSE process. If we kill
-    // the FUSE process first, most file system operations will return
+    int userId = getMountUserId();
+
+    // Kill all processes using the filesystem before we unmount it. If we
+    // unmount the filesystem first, most file system operations will return
     // ENOTCONN until the unmount completes. This is an exotic and unusual
     // error code and might cause broken behaviour in applications.
-    KillProcessesUsingPath(getPath());
+    if (mFuseMounted) {
+        // For FUSE specifically, we have an emulated volume per user, so only kill
+        // processes using files from this particular user.
+        std::string user_path(StringPrintf("%s/%d", getPath().c_str(), getMountUserId()));
+        LOG(INFO) << "Killing all processes referencing " << user_path;
+        KillProcessesUsingPath(user_path);
+    } else {
+        KillProcessesUsingPath(getPath());
+    }
 
-    bool isFuse = base::GetBoolProperty(kPropFuseSnapshot, false);
-    if (isFuse) {
-        // We could have migrated storage to an adopted private volume, so always
-        // call primary storage "emulated" to avoid media rescans.
-        std::string label = mLabel;
-        if (getMountFlags() & MountFlags::kPrimary) {
-            label = "emulated";
-        }
+    if (mFuseMounted) {
+        std::string label = getLabel();
+        // Ignoring unmount return status because we do want to try to unmount
+        // the rest cleanly.
 
-        std::string fuse_path(StringPrintf("/mnt/user/%d/%s", getMountUserId(), label.c_str()));
-        std::string pass_through_path(
-                StringPrintf("/mnt/pass_through/%d/%s", getMountUserId(), label.c_str()));
-        if (UnmountUserFuse(pass_through_path, fuse_path) != OK) {
+        unmountFuseBindMounts(userId, label);
+        if (UnmountUserFuse(userId, getInternalPath(), label) != OK) {
             PLOG(INFO) << "UnmountUserFuse failed on emulated fuse volume";
             return -errno;
         }
 
-        rmdir(fuse_path.c_str());
-        rmdir(pass_through_path.c_str());
-        return OK;
-    } else if (getMountUserId() != 0) {
+        mFuseMounted = false;
+    }
+    if (getMountUserId() != 0) {
         // For sdcardfs, only unmount for user 0, since user 0 will always be running
         // and the paths don't change for different users.
         return OK;
diff --git a/model/EmulatedVolume.h b/model/EmulatedVolume.h
index 6692b23..131761c 100644
--- a/model/EmulatedVolume.h
+++ b/model/EmulatedVolume.h
@@ -46,6 +46,7 @@
     status_t doUnmount() override;
 
   private:
+    std::string getLabel();
     std::string mRawPath;
     std::string mLabel;
 
@@ -54,6 +55,9 @@
     std::string mSdcardFsWrite;
     std::string mSdcardFsFull;
 
+    /* Whether we mounted FUSE for this volume */
+    bool mFuseMounted;
+
     DISALLOW_COPY_AND_ASSIGN(EmulatedVolume);
 };
 
diff --git a/model/PublicVolume.cpp b/model/PublicVolume.cpp
index 007fd60..3b5e6f0 100644
--- a/model/PublicVolume.cpp
+++ b/model/PublicVolume.cpp
@@ -95,6 +95,7 @@
 }
 
 status_t PublicVolume::doMount() {
+    bool isVisible = getMountFlags() & MountFlags::kVisible;
     readMetadata();
 
     if (mFsType == "vfat" && vfat::IsSupported()) {
@@ -126,7 +127,7 @@
     mSdcardFsFull = StringPrintf("/mnt/runtime/full/%s", stableName.c_str());
 
     setInternalPath(mRawPath);
-    if (getMountFlags() & MountFlags::kVisible) {
+    if (isVisible) {
         setPath(StringPrintf("/storage/%s", stableName.c_str()));
     } else {
         setPath(mRawPath);
@@ -154,7 +155,7 @@
         initAsecStage();
     }
 
-    if (!(getMountFlags() & MountFlags::kVisible)) {
+    if (!isVisible) {
         // Not visible to apps, so no need to spin up sdcardfs or FUSE
         return OK;
     }
@@ -169,30 +170,6 @@
 
     dev_t before = GetDevice(mSdcardFsFull);
 
-    bool isFuse = base::GetBoolProperty(kPropFuseSnapshot, false);
-    if (isFuse) {
-        LOG(INFO) << "Mounting public fuse volume";
-        android::base::unique_fd fd;
-        int user_id = getMountUserId();
-        int result = MountUserFuse(user_id, getInternalPath(), stableName, &fd);
-
-        if (result != 0) {
-            LOG(ERROR) << "Failed to mount public fuse volume";
-            return -result;
-        }
-
-        auto callback = getMountCallback();
-        if (callback) {
-            bool is_ready = false;
-            callback->onVolumeChecking(std::move(fd), getPath(), getInternalPath(), &is_ready);
-            if (!is_ready) {
-                return -EIO;
-            }
-        }
-
-        return OK;
-    }
-
     int sdcardFsPid;
     if (!(sdcardFsPid = fork())) {
         if (getMountFlags() & MountFlags::kPrimary) {
@@ -245,6 +222,31 @@
     /* sdcardfs will have exited already. The filesystem will still be running */
     TEMP_FAILURE_RETRY(waitpid(sdcardFsPid, nullptr, 0));
 
+    bool isFuse = base::GetBoolProperty(kPropFuseSnapshot, false);
+    if (isFuse) {
+        // We need to mount FUSE *after* sdcardfs, since the FUSE daemon may depend
+        // on sdcardfs being up.
+        LOG(INFO) << "Mounting public fuse volume";
+        android::base::unique_fd fd;
+        int user_id = getMountUserId();
+        int result = MountUserFuse(user_id, getInternalPath(), stableName, &fd);
+
+        if (result != 0) {
+            LOG(ERROR) << "Failed to mount public fuse volume";
+            return -result;
+        }
+
+        mFuseMounted = true;
+        auto callback = getMountCallback();
+        if (callback) {
+            bool is_ready = false;
+            callback->onVolumeChecking(std::move(fd), getPath(), getInternalPath(), &is_ready);
+            if (!is_ready) {
+                return -EIO;
+            }
+        }
+    }
+
     return OK;
 }
 
@@ -255,30 +257,19 @@
     // error code and might cause broken behaviour in applications.
     KillProcessesUsingPath(getPath());
 
-    bool isFuse = base::GetBoolProperty(kPropFuseSnapshot, false);
-    if (isFuse) {
+    if (mFuseMounted) {
         // Use UUID as stable name, if available
         std::string stableName = getId();
         if (!mFsUuid.empty()) {
             stableName = mFsUuid;
         }
 
-        std::string fuse_path(
-                StringPrintf("/mnt/user/%d/%s", getMountUserId(), stableName.c_str()));
-        std::string pass_through_path(
-                StringPrintf("/mnt/pass_through/%d/%s", getMountUserId(), stableName.c_str()));
-        if (UnmountUserFuse(pass_through_path, fuse_path) != OK) {
+        if (UnmountUserFuse(getMountUserId(), getInternalPath(), stableName) != OK) {
             PLOG(INFO) << "UnmountUserFuse failed on public fuse volume";
             return -errno;
         }
-        ForceUnmount(kAsecPath);
-        ForceUnmount(mRawPath);
 
-        rmdir(fuse_path.c_str());
-        rmdir(pass_through_path.c_str());
-        rmdir(mRawPath.c_str());
-        mRawPath.clear();
-        return OK;
+        mFuseMounted = false;
     }
 
     ForceUnmount(kAsecPath);
diff --git a/model/PublicVolume.h b/model/PublicVolume.h
index a9bc1ab..dd76373 100644
--- a/model/PublicVolume.h
+++ b/model/PublicVolume.h
@@ -65,6 +65,9 @@
     std::string mSdcardFsWrite;
     std::string mSdcardFsFull;
 
+    /* Whether we mounted FUSE for this volume */
+    bool mFuseMounted;
+
     /* Filesystem type */
     std::string mFsType;
     /* Filesystem UUID */