Create separate api for reconciling per-sdk storage

Sdks should get their own storage area under the sdk package path
at: `/data/misc_ce/0/sdksandbox/<package>/<sdk-name>@<random-suffix>`

The random suffix is to prevent sdks from guessing each others storage
paths. It is separated by '@' which is an illegal character for package
names.

The new API prepares the sdk-level directories as follows:
- creates them if they do not exist.
- on occasssions where they already exist, it skips creating them.
- on occasssions where it finds an per-sdk directory that shouldn't be
  there, it deletes them.
- it also creates the package path if its missing for any reason.

Additional Changes:
- For package-level sdk data directory, we now delete it if FLAG_STORAGE_SDK
flag is not passed. This is done so that we clean up properly when an
app that consumes sdk updates to a state where it no longer consumes
sdks.

Bug: 211763739
Test: atest installd_service_tes
Test: atest SdkSandboxStorageHostTest
Ignore-AOSP-First: Feature is being developed in internal branch
Change-Id: I7f144f5da4c4e10851f78d731de5dfb429fbac4d
Merged-In: I7f144f5da4c4e10851f78d731de5dfb429fbac4d
(cherry picked from commit 92f5cf0d0b943315d475dfbe9273a6118095bdbf)
diff --git a/cmds/installd/InstalldNativeService.cpp b/cmds/installd/InstalldNativeService.cpp
index 2705505..be56f83 100644
--- a/cmds/installd/InstalldNativeService.cpp
+++ b/cmds/installd/InstalldNativeService.cpp
@@ -41,6 +41,7 @@
 #include <fstream>
 #include <functional>
 #include <regex>
+#include <unordered_set>
 
 #include <android-base/file.h>
 #include <android-base/logging.h>
@@ -81,6 +82,7 @@
 // #define GRANULAR_LOCKS
 
 using android::base::ParseUint;
+using android::base::Split;
 using android::base::StringPrintf;
 using std::endl;
 
@@ -723,21 +725,26 @@
     }
 
     if (flags & FLAG_STORAGE_SDK) {
-        auto status = createSdkSandboxDataDirectory(uuid, packageName, userId, appId, previousAppId,
-                                                    seInfo, flags);
-        if (!status.isOk()) {
-            return status;
+        // Safe to ignore status since we can retry creating this by calling reconcileSdkData
+        auto ignore = createSdkSandboxDataPackageDirectory(uuid, packageName, userId, appId,
+                                                           previousAppId, seInfo, flags);
+        if (!ignore.isOk()) {
+            PLOG(WARNING) << "Failed to create sdk data package directory for " << packageName;
         }
+
+    } else {
+        // Package does not need sdk storage. Remove it.
+        deleteSdkSandboxDataPackageDirectory(uuid, packageName, userId, flags);
     }
 
     return ok();
 }
 
 /**
- * Responsible for creating /data/misc_{ce|de}/user/0/sdksandbox/<app-name> directory and other
+ * Responsible for creating /data/misc_{ce|de}/user/0/sdksandbox/<package-name> directory and other
  * app level sub directories, such as ./shared
  */
-binder::Status InstalldNativeService::createSdkSandboxDataDirectory(
+binder::Status InstalldNativeService::createSdkSandboxDataPackageDirectory(
         const std::optional<std::string>& uuid, const std::string& packageName, int32_t userId,
         int32_t appId, int32_t previousAppId, const std::string& seInfo, int32_t flags) {
     int32_t sdkSandboxUid = multiuser_get_sdk_sandbox_uid(userId, appId);
@@ -746,7 +753,6 @@
         return ok();
     }
 
-    // TODO(b/211763739): what if uuid is not nullptr or TEST?
     const char* uuid_ = uuid ? uuid->c_str() : nullptr;
 
     constexpr int storageFlags[2] = {FLAG_STORAGE_CE, FLAG_STORAGE_DE};
@@ -761,10 +767,14 @@
         // during user creation
 
         // Prepare the app directory
-        auto appPath = create_data_misc_sdk_sandbox_package_path(uuid_, isCeData, userId,
-                                                                 packageName.c_str());
-        if (prepare_app_dir(appPath, 0751, AID_SYSTEM)) {
-            return error("Failed to prepare " + appPath);
+        auto packagePath = create_data_misc_sdk_sandbox_package_path(uuid_, isCeData, userId,
+                                                                     packageName.c_str());
+#if SDK_DEBUG
+        LOG(DEBUG) << "Creating app-level sdk data directory: " << packagePath;
+#endif
+
+        if (prepare_app_dir(packagePath, 0751, AID_SYSTEM)) {
+            return error("Failed to prepare " + packagePath);
         }
 
         // Now prepare the shared directory which will be accessible by all codes
@@ -792,6 +802,32 @@
     return ok();
 }
 
+/**
+ * Responsible for deleting /data/misc_{ce|de}/user/0/sdksandbox/<package-name> directory
+ */
+binder::Status InstalldNativeService::deleteSdkSandboxDataPackageDirectory(
+        const std::optional<std::string>& uuid, const std::string& packageName, int32_t userId,
+        int32_t flags) {
+    const char* uuid_ = uuid ? uuid->c_str() : nullptr;
+    auto res = ok();
+
+    constexpr int storageFlags[2] = {FLAG_STORAGE_CE, FLAG_STORAGE_DE};
+    for (int currentFlag : storageFlags) {
+        if ((flags & currentFlag) == 0) {
+            continue;
+        }
+        bool isCeData = (currentFlag == FLAG_STORAGE_CE);
+
+        const auto& packagePath = create_data_misc_sdk_sandbox_package_path(uuid_, isCeData, userId,
+                                                                            packageName.c_str());
+        if (delete_dir_contents_and_dir(packagePath, /*ignore_if_missing=*/true) != 0) {
+            res = error("Failed to delete sdk package directory: " + packagePath);
+        }
+    }
+
+    return res;
+}
+
 binder::Status InstalldNativeService::createAppData(
         const std::optional<std::string>& uuid, const std::string& packageName, int32_t userId,
         int32_t flags, int32_t appId, int32_t previousAppId, const std::string& seInfo,
@@ -835,6 +871,140 @@
     return ok();
 }
 
+binder::Status InstalldNativeService::reconcileSdkData(
+        const android::os::ReconcileSdkDataArgs& args) {
+    ENFORCE_UID(AID_SYSTEM);
+    // Locking is performed depeer in the callstack.
+
+    return reconcileSdkData(args.uuid, args.packageName, args.sdkPackageNames, args.randomSuffixes,
+                            args.userId, args.appId, args.previousAppId, args.seInfo, args.flags);
+}
+
+/**
+ * Reconciles per-sdk directory under app-level sdk data directory.
+
+ * E.g. `/data/misc_ce/0/sdksandbox/<package-name>/<sdkPackageName>-<randomSuffix>
+ *
+ * - If the sdk data package directory is missing, we create it first.
+ * - If sdkPackageNames is empty, we delete sdk package directory since it's not needed anymore.
+ * - If a sdk level directory we need to prepare already exist, we skip creating it again. This
+ *   is to avoid having same per-sdk directory with different suffix.
+ * - If a sdk level directory exist which is absent from sdkPackageNames, we remove it.
+ */
+binder::Status InstalldNativeService::reconcileSdkData(
+        const std::optional<std::string>& uuid, const std::string& packageName,
+        const std::vector<std::string>& sdkPackageNames,
+        const std::vector<std::string>& randomSuffixes, int userId, int appId, int previousAppId,
+        const std::string& seInfo, int flags) {
+    CHECK_ARGUMENT_UUID(uuid);
+    CHECK_ARGUMENT_PACKAGE_NAME(packageName);
+    for (const auto& sdkPackageName : sdkPackageNames) {
+        CHECK_ARGUMENT_PACKAGE_NAME(sdkPackageName);
+    }
+    LOCK_PACKAGE_USER();
+
+#if SDK_DEBUG
+    LOG(DEBUG) << "Creating per sdk data directory for: " << packageName;
+#endif
+
+    const char* uuid_ = uuid ? uuid->c_str() : nullptr;
+
+    // Validate we have enough randomSuffixStrings
+    if (randomSuffixes.size() != sdkPackageNames.size()) {
+        return exception(binder::Status::EX_ILLEGAL_ARGUMENT,
+                         StringPrintf("Not enough random suffix. Required %d, received %d.",
+                                      (int)sdkPackageNames.size(), (int)randomSuffixes.size()));
+    }
+
+    // Prepare the sdk package directory in case it's missing
+    const auto status = createSdkSandboxDataPackageDirectory(uuid, packageName, userId, appId,
+                                                             previousAppId, seInfo, flags);
+    if (!status.isOk()) {
+        return status;
+    }
+
+    auto res = ok();
+    // We have to create sdk data for CE and DE storage
+    const int storageFlags[2] = {FLAG_STORAGE_CE, FLAG_STORAGE_DE};
+    for (int currentFlag : storageFlags) {
+        if ((flags & currentFlag) == 0) {
+            continue;
+        }
+        const bool isCeData = (currentFlag == FLAG_STORAGE_CE);
+
+        // Since random suffix provided will be random every time, we need to ensure we don't end up
+        // creating multuple directories for same sdk package with different suffixes. This
+        // is ensured by fetching all the existing sub directories and storing them so that we can
+        // check for existence later. We also remove unconsumed sdk directories in this check.
+        const auto packagePath = create_data_misc_sdk_sandbox_package_path(uuid_, isCeData, userId,
+                                                                           packageName.c_str());
+        const std::unordered_set<std::string> expectedSdkNames(sdkPackageNames.begin(),
+                                                               sdkPackageNames.end());
+        // Store paths of per-sdk directory for sdk that already exists
+        std::unordered_map<std::string, std::string> sdkNamesThatExist;
+
+        const auto subDirHandler = [&packagePath, &expectedSdkNames, &sdkNamesThatExist,
+                                    &res](const std::string& filename) {
+            auto filepath = packagePath + "/" + filename;
+            auto tokens = Split(filename, "@");
+            if (tokens.size() != 2) {
+                // Not a per-sdk directory with random suffix
+                return;
+            }
+            auto sdkName = tokens[0];
+
+            // Remove the per-sdk directory if it is not referred in
+            // expectedSdkNames
+            if (expectedSdkNames.find(sdkName) == expectedSdkNames.end()) {
+                if (delete_dir_contents_and_dir(filepath) != 0) {
+                    res = error("Failed to delete " + filepath);
+                    return;
+                }
+            } else {
+                // Otherwise, store it as existing sdk level directory
+                sdkNamesThatExist[sdkName] = filepath;
+            }
+        };
+        const int ec = foreach_subdir(packagePath, subDirHandler);
+        if (ec != 0) {
+            res = error("Failed to process subdirs for " + packagePath);
+            continue;
+        }
+
+        // Create sdksandbox data directory for each sdksandbox package
+        for (int i = 0, size = sdkPackageNames.size(); i < size; i++) {
+            const std::string& sdkName = sdkPackageNames[i];
+            const std::string& randomSuffix = randomSuffixes[i];
+            std::string path;
+            if (const auto& it = sdkNamesThatExist.find(sdkName); it != sdkNamesThatExist.end()) {
+                // Already exists. Use existing path instead of creating a new one
+                path = it->second;
+            } else {
+                path = create_data_misc_sdk_sandbox_sdk_path(uuid_, isCeData, userId,
+                                                             packageName.c_str(), sdkName.c_str(),
+                                                             randomSuffix.c_str());
+            }
+
+            // Create the directory along with cache and code_cache
+            const int32_t cacheGid = multiuser_get_cache_gid(userId, appId);
+            if (cacheGid == -1) {
+                return exception(binder::Status::EX_ILLEGAL_STATE,
+                                 StringPrintf("cacheGid cannot be -1 for sdk data"));
+            }
+            const int32_t sandboxUid = multiuser_get_sdk_sandbox_uid(userId, appId);
+            int32_t previousSandboxUid = multiuser_get_sdk_sandbox_uid(userId, previousAppId);
+            auto status = createAppDataDirs(path, sandboxUid, &previousSandboxUid, cacheGid, seInfo,
+                                            0700);
+            if (!status.isOk()) {
+                res = status;
+                continue;
+            }
+        }
+    }
+
+    return res;
+}
+
 binder::Status InstalldNativeService::migrateAppData(const std::optional<std::string>& uuid,
         const std::string& packageName, int32_t userId, int32_t flags) {
     ENFORCE_UID(AID_SYSTEM);
diff --git a/cmds/installd/InstalldNativeService.h b/cmds/installd/InstalldNativeService.h
index b2bad1d..0e9b0cf 100644
--- a/cmds/installd/InstalldNativeService.h
+++ b/cmds/installd/InstalldNativeService.h
@@ -58,6 +58,8 @@
             const std::vector<android::os::CreateAppDataArgs>& args,
             std::vector<android::os::CreateAppDataResult>* _aidl_return);
 
+    binder::Status reconcileSdkData(const android::os::ReconcileSdkDataArgs& args);
+
     binder::Status restoreconAppData(const std::optional<std::string>& uuid,
             const std::string& packageName, int32_t userId, int32_t flags, int32_t appId,
             const std::string& seInfo);
@@ -204,10 +206,22 @@
                                        const std::string& seInfo, int32_t targetSdkVersion,
                                        int64_t* _aidl_return);
 
-    binder::Status createSdkSandboxDataDirectory(const std::optional<std::string>& uuid,
-                                                 const std::string& packageName, int32_t userId,
-                                                 int32_t appId, int32_t previousAppId,
-                                                 const std::string& seInfo, int32_t flags);
+    binder::Status createSdkSandboxDataPackageDirectory(const std::optional<std::string>& uuid,
+                                                        const std::string& packageName,
+                                                        int32_t userId, int32_t appId,
+                                                        int32_t previousAppId,
+                                                        const std::string& seInfo, int32_t flags);
+
+    binder::Status deleteSdkSandboxDataPackageDirectory(const std::optional<std::string>& uuid,
+                                                        const std::string& packageName,
+                                                        int32_t userId, int32_t flags);
+
+    binder::Status reconcileSdkData(const std::optional<std::string>& uuid,
+                                    const std::string& packageName,
+                                    const std::vector<std::string>& sdkPackageNames,
+                                    const std::vector<std::string>& randomSuffixes, int32_t userId,
+                                    int32_t appId, int32_t previousAppId, const std::string& seInfo,
+                                    int flags);
 };
 
 }  // namespace installd
diff --git a/cmds/installd/binder/android/os/IInstalld.aidl b/cmds/installd/binder/android/os/IInstalld.aidl
index 1e22f48..d76ee54 100644
--- a/cmds/installd/binder/android/os/IInstalld.aidl
+++ b/cmds/installd/binder/android/os/IInstalld.aidl
@@ -24,6 +24,8 @@
     android.os.CreateAppDataResult createAppData(in android.os.CreateAppDataArgs args);
     android.os.CreateAppDataResult[] createAppDataBatched(in android.os.CreateAppDataArgs[] args);
 
+    void reconcileSdkData(in android.os.ReconcileSdkDataArgs args);
+
     void restoreconAppData(@nullable @utf8InCpp String uuid, @utf8InCpp String packageName,
             int userId, int flags, int appId, @utf8InCpp String seInfo);
     void migrateAppData(@nullable @utf8InCpp String uuid, @utf8InCpp String packageName,
diff --git a/cmds/installd/binder/android/os/ReconcileSdkDataArgs.aidl b/cmds/installd/binder/android/os/ReconcileSdkDataArgs.aidl
new file mode 100644
index 0000000..2f794b1
--- /dev/null
+++ b/cmds/installd/binder/android/os/ReconcileSdkDataArgs.aidl
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.os;
+
+/** {@hide} */
+parcelable ReconcileSdkDataArgs {
+    @nullable @utf8InCpp String uuid;
+    @utf8InCpp String packageName;
+    @utf8InCpp List<String> sdkPackageNames;
+    @utf8InCpp List<String> randomSuffixes;
+    int userId;
+    int appId;
+    int previousAppId;
+    @utf8InCpp String seInfo;
+    int flags;
+}
diff --git a/cmds/installd/tests/Android.bp b/cmds/installd/tests/Android.bp
index e390bab..2e63dd6 100644
--- a/cmds/installd/tests/Android.bp
+++ b/cmds/installd/tests/Android.bp
@@ -26,6 +26,7 @@
         "libasync_safe",
         "libdiskusage",
         "libext2_uuid",
+        "libgmock",
         "libinstalld",
         "liblog",
     ],
diff --git a/cmds/installd/tests/installd_service_test.cpp b/cmds/installd/tests/installd_service_test.cpp
index 31f5dce..30a23eb 100644
--- a/cmds/installd/tests/installd_service_test.cpp
+++ b/cmds/installd/tests/installd_service_test.cpp
@@ -36,6 +36,7 @@
 #include <android/content/pm/IPackageManagerNative.h>
 #include <binder/IServiceManager.h>
 #include "InstalldNativeService.h"
+#include "binder/Status.h"
 #include "binder_test_utils.h"
 #include "dexopt.h"
 #include "globals.h"
@@ -965,10 +966,10 @@
     bool exists(const char* path) { return ::access(path, F_OK) == 0; }
 
     // Creates a default CreateAppDataArgs object
-    android::os::CreateAppDataArgs createAppDataArgs() {
+    android::os::CreateAppDataArgs createAppDataArgs(std::string packageName) {
         android::os::CreateAppDataArgs args;
         args.uuid = kTestUuid;
-        args.packageName = "com.foo";
+        args.packageName = packageName;
         args.userId = kTestUserId;
         args.appId = kTestAppId;
         args.seInfo = "default";
@@ -976,6 +977,26 @@
         return args;
     }
 
+    android::os::ReconcileSdkDataArgs reconcileSdkDataArgs(
+            std::string packageName, std::vector<std::string> codeNames,
+            std::vector<std::string> randomSuffixes) {
+        android::os::ReconcileSdkDataArgs args;
+        args.uuid = kTestUuid;
+        args.packageName = packageName;
+        for (const auto& codeName : codeNames) {
+            args.sdkPackageNames.push_back(codeName);
+        }
+        for (const auto& randomSuffix : randomSuffixes) {
+            args.randomSuffixes.push_back(randomSuffix);
+        }
+        args.userId = kTestUserId;
+        args.appId = kTestAppId;
+        args.previousAppId = -1;
+        args.seInfo = "default";
+        args.flags = FLAG_STORAGE_CE | FLAG_STORAGE_DE;
+        return args;
+    }
+
 protected:
     InstalldNativeService* service;
 
@@ -1000,41 +1021,37 @@
 
 private:
     void clearAppData() {
+        ASSERT_EQ(0, delete_dir_contents_and_dir("/data/local/tmp/user", true));
         ASSERT_EQ(0, delete_dir_contents_and_dir("/data/local/tmp/user_de", true));
         ASSERT_EQ(0, delete_dir_contents_and_dir("/data/local/tmp/misc_ce", true));
         ASSERT_EQ(0, delete_dir_contents_and_dir("/data/local/tmp/misc_de", true));
-        ASSERT_EQ(0, delete_dir_contents_and_dir("/data/local/tmp/user_de", true));
     }
 };
 
 TEST_F(SdkSandboxDataTest, CreateAppData_CreatesSdkAppLevelData) {
     android::os::CreateAppDataResult result;
-    android::os::CreateAppDataArgs args = createAppDataArgs();
-    args.packageName = "com.foo";
+    android::os::CreateAppDataArgs args = createAppDataArgs("com.foo");
     args.flags = FLAG_STORAGE_CE | FLAG_STORAGE_DE | FLAG_STORAGE_SDK;
 
     // Create the app user data.
     ASSERT_BINDER_SUCCESS(service->createAppData(args, &result));
 
-    CheckFileAccess("misc_ce/0/sdksandbox/com.foo", kSystemUid, S_IFDIR | 0751);
-    CheckFileAccess("misc_ce/0/sdksandbox/com.foo/shared", kTestSdkSandboxUid, S_IFDIR | 0700);
-    CheckFileAccess("misc_ce/0/sdksandbox/com.foo/shared/cache", kTestSdkSandboxUid,
-                    S_IFDIR | S_ISGID | 0771);
-    CheckFileAccess("misc_ce/0/sdksandbox/com.foo/shared/code_cache", kTestSdkSandboxUid,
-                    S_IFDIR | S_ISGID | 0771);
+    const std::string fooCePath = "misc_ce/0/sdksandbox/com.foo";
+    CheckFileAccess(fooCePath, kSystemUid, S_IFDIR | 0751);
+    CheckFileAccess(fooCePath + "/shared", kTestSdkSandboxUid, S_IFDIR | 0700);
+    CheckFileAccess(fooCePath + "/shared/cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
+    CheckFileAccess(fooCePath + "/shared/code_cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
 
-    CheckFileAccess("misc_de/0/sdksandbox/com.foo", kSystemUid, S_IFDIR | 0751);
-    CheckFileAccess("misc_de/0/sdksandbox/com.foo/shared", kTestSdkSandboxUid, S_IFDIR | 0700);
-    CheckFileAccess("misc_de/0/sdksandbox/com.foo/shared/cache", kTestSdkSandboxUid,
-                    S_IFDIR | S_ISGID | 0771);
-    CheckFileAccess("misc_de/0/sdksandbox/com.foo/shared/code_cache", kTestSdkSandboxUid,
-                    S_IFDIR | S_ISGID | 0771);
+    const std::string fooDePath = "misc_de/0/sdksandbox/com.foo";
+    CheckFileAccess(fooDePath, kSystemUid, S_IFDIR | 0751);
+    CheckFileAccess(fooDePath + "/shared", kTestSdkSandboxUid, S_IFDIR | 0700);
+    CheckFileAccess(fooDePath + "/shared/cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
+    CheckFileAccess(fooDePath + "/shared/code_cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
 }
 
-TEST_F(SdkSandboxDataTest, CreateAppData_CreatesSdkAppLeveleData_WithoutSdkFlag) {
+TEST_F(SdkSandboxDataTest, CreateAppData_CreatesSdkAppLevelData_WithoutSdkFlag) {
     android::os::CreateAppDataResult result;
-    android::os::CreateAppDataArgs args = createAppDataArgs();
-    args.packageName = "com.foo";
+    android::os::CreateAppDataArgs args = createAppDataArgs("com.foo");
     args.flags = FLAG_STORAGE_CE | FLAG_STORAGE_DE;
 
     // Create the app user data.
@@ -1044,10 +1061,24 @@
     ASSERT_FALSE(exists("/data/local/tmp/misc_de/0/sdksandbox/com.foo"));
 }
 
-TEST_F(SdkSandboxDataTest, CreateAppData_CreatesSdkAppLeveleData_WithoutDeFlag) {
+TEST_F(SdkSandboxDataTest, CreateAppData_CreatesSdkAppLevelData_WithoutSdkFlagDeletesExisting) {
     android::os::CreateAppDataResult result;
-    android::os::CreateAppDataArgs args = createAppDataArgs();
-    args.packageName = "com.foo";
+    android::os::CreateAppDataArgs args = createAppDataArgs("com.foo");
+    args.flags = FLAG_STORAGE_CE | FLAG_STORAGE_DE | FLAG_STORAGE_SDK;
+    // Create the app user data.
+    ASSERT_BINDER_SUCCESS(service->createAppData(args, &result));
+    ASSERT_TRUE(exists("/data/local/tmp/misc_ce/0/sdksandbox/com.foo"));
+    ASSERT_TRUE(exists("/data/local/tmp/misc_de/0/sdksandbox/com.foo"));
+
+    args.flags = FLAG_STORAGE_CE | FLAG_STORAGE_DE;
+    ASSERT_BINDER_SUCCESS(service->createAppData(args, &result));
+    ASSERT_FALSE(exists("/data/local/tmp/misc_ce/0/sdksandbox/com.foo"));
+    ASSERT_FALSE(exists("/data/local/tmp/misc_de/0/sdksandbox/com.foo"));
+}
+
+TEST_F(SdkSandboxDataTest, CreateAppData_CreatesSdkAppLevelData_WithoutDeFlag) {
+    android::os::CreateAppDataResult result;
+    android::os::CreateAppDataArgs args = createAppDataArgs("com.foo");
     args.flags = FLAG_STORAGE_CE | FLAG_STORAGE_SDK;
 
     // Create the app user data.
@@ -1060,10 +1091,9 @@
     ASSERT_FALSE(exists("/data/local/tmp/misc_de/0/sdksandbox/com.foo"));
 }
 
-TEST_F(SdkSandboxDataTest, CreateAppData_CreatesSdkAppLeveleData_WithoutCeFlag) {
+TEST_F(SdkSandboxDataTest, CreateAppData_CreatesSdkAppLevelData_WithoutCeFlag) {
     android::os::CreateAppDataResult result;
-    android::os::CreateAppDataArgs args = createAppDataArgs();
-    args.packageName = "com.foo";
+    android::os::CreateAppDataArgs args = createAppDataArgs("com.foo");
     args.flags = FLAG_STORAGE_DE | FLAG_STORAGE_SDK;
 
     // Create the app user data.
@@ -1076,5 +1106,96 @@
     CheckFileAccess("misc_de/0/sdksandbox/com.foo", kSystemUid, S_IFDIR | 0751);
 }
 
+TEST_F(SdkSandboxDataTest, ReconcileSdkData) {
+    android::os::ReconcileSdkDataArgs args =
+            reconcileSdkDataArgs("com.foo", {"bar", "baz"}, {"random1", "random2"});
+
+    // Create the sdk data.
+    ASSERT_BINDER_SUCCESS(service->reconcileSdkData(args));
+
+    const std::string barCePath = "misc_ce/0/sdksandbox/com.foo/bar@random1";
+    CheckFileAccess(barCePath, kTestSdkSandboxUid, S_IFDIR | 0700);
+    CheckFileAccess(barCePath + "/cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
+    CheckFileAccess(barCePath + "/code_cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
+
+    const std::string bazCePath = "misc_ce/0/sdksandbox/com.foo/baz@random2";
+    CheckFileAccess(bazCePath, kTestSdkSandboxUid, S_IFDIR | 0700);
+    CheckFileAccess(bazCePath + "/cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
+    CheckFileAccess(bazCePath + "/code_cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
+
+    const std::string barDePath = "misc_de/0/sdksandbox/com.foo/bar@random1";
+    CheckFileAccess(barDePath, kTestSdkSandboxUid, S_IFDIR | 0700);
+    CheckFileAccess(barDePath + "/cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
+    CheckFileAccess(barDePath + "/code_cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
+
+    const std::string bazDePath = "misc_de/0/sdksandbox/com.foo/baz@random2";
+    CheckFileAccess(bazDePath, kTestSdkSandboxUid, S_IFDIR | 0700);
+    CheckFileAccess(bazDePath + "/cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
+    CheckFileAccess(bazDePath + "/code_cache", kTestSdkSandboxUid, S_IFDIR | S_ISGID | 0771);
+}
+
+TEST_F(SdkSandboxDataTest, ReconcileSdkData_PackageNameCannotUseRandomSuffixSeparator) {
+    android::os::ReconcileSdkDataArgs args =
+            reconcileSdkDataArgs("com.foo", {"bar@illegal"}, {"random1"});
+
+    // Create the sdksandbox data.
+    auto status = service->reconcileSdkData(args);
+    ASSERT_EQ(status.exceptionCode(), binder::Status::EX_ILLEGAL_ARGUMENT);
+    ASSERT_EQ(status.exceptionMessage(), "Package name bar@illegal is malformed");
+}
+
+TEST_F(SdkSandboxDataTest, ReconcileSdkData_NotEnoughRandomSuffix) {
+    android::os::ReconcileSdkDataArgs args =
+            reconcileSdkDataArgs("com.foo", {"bar", "baz"}, {"random1"});
+
+    // Create the sdksandbox data.
+    auto status = service->reconcileSdkData(args);
+    ASSERT_EQ(status.exceptionCode(), binder::Status::EX_ILLEGAL_ARGUMENT);
+    ASSERT_EQ(status.exceptionMessage(), "Not enough random suffix. Required 2, received 1.");
+}
+
+TEST_F(SdkSandboxDataTest, ReconcileSdkData_DirectoryNotCreatedIfAlreadyExistsIgnoringSuffix) {
+    android::os::ReconcileSdkDataArgs args =
+            reconcileSdkDataArgs("com.foo", {"bar", "baz"}, {"random1", "random2"});
+
+    // Create the sdksandbox data.
+    ASSERT_BINDER_SUCCESS(service->reconcileSdkData(args));
+
+    // Retry with different random suffix
+    args.randomSuffixes[0] = "r10";
+    args.randomSuffixes[1] = "r20";
+
+    // Create the sdksandbox data again
+    ASSERT_BINDER_SUCCESS(service->reconcileSdkData(args));
+
+    // Previous directories from first attempt should exist
+    CheckFileAccess("misc_ce/0/sdksandbox/com.foo/bar@random1", kTestSdkSandboxUid, S_IFDIR | 0700);
+    CheckFileAccess("misc_ce/0/sdksandbox/com.foo/baz@random2", kTestSdkSandboxUid, S_IFDIR | 0700);
+    // No new directories should be created on second attempt
+    ASSERT_FALSE(exists("/data/local/tmp/misc_ce/0/sdksandbox/com.foo/bar@r10"));
+    ASSERT_FALSE(exists("/data/local/tmp/misc_de/0/sdksandbox/com.foo/bar@r20"));
+}
+
+TEST_F(SdkSandboxDataTest, ReconcileSdkData_ExtraCodeDirectoriesAreDeleted) {
+    android::os::ReconcileSdkDataArgs args =
+            reconcileSdkDataArgs("com.foo", {"bar", "baz"}, {"random1", "random2"});
+
+    // Create the sdksandbox data.
+    ASSERT_BINDER_SUCCESS(service->reconcileSdkData(args));
+
+    // Retry with different package name
+    args.sdkPackageNames[0] = "bar.diff";
+
+    // Create the sdksandbox data again
+    ASSERT_BINDER_SUCCESS(service->reconcileSdkData(args));
+
+    // New directoris should exist
+    CheckFileAccess("misc_ce/0/sdksandbox/com.foo/bar.diff@random1", kTestSdkSandboxUid,
+                    S_IFDIR | 0700);
+    CheckFileAccess("misc_ce/0/sdksandbox/com.foo/baz@random2", kTestSdkSandboxUid, S_IFDIR | 0700);
+    // Directory for old unreferred sdksandbox package name should be removed
+    ASSERT_FALSE(exists("/data/local/tmp/misc_ce/0/sdksandbox/com.foo/bar@random1"));
+}
+
 }  // namespace installd
 }  // namespace android
diff --git a/cmds/installd/tests/installd_utils_test.cpp b/cmds/installd/tests/installd_utils_test.cpp
index 17802a3..38c1c05 100644
--- a/cmds/installd/tests/installd_utils_test.cpp
+++ b/cmds/installd/tests/installd_utils_test.cpp
@@ -21,6 +21,7 @@
 
 #include <android-base/logging.h>
 #include <android-base/scopeguard.h>
+#include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
 #include "InstalldNativeService.h"
@@ -47,6 +48,8 @@
 namespace android {
 namespace installd {
 
+using ::testing::UnorderedElementsAre;
+
 class UtilsTest : public testing::Test {
 protected:
     virtual void SetUp() {
@@ -658,6 +661,23 @@
     ASSERT_NE(0, create_dir_if_needed("/data/local/tmp/user/0/bar/baz", 0700));
 }
 
+TEST_F(UtilsTest, TestForEachSubdir) {
+    auto deleter = [&]() {
+        delete_dir_contents_and_dir("/data/local/tmp/user/0", true /* ignore_if_missing */);
+    };
+    auto scope_guard = android::base::make_scope_guard(deleter);
+
+    system("mkdir -p /data/local/tmp/user/0/com.foo");
+    system("mkdir -p /data/local/tmp/user/0/com.bar");
+    system("touch /data/local/tmp/user/0/some-file");
+
+    std::vector<std::string> result;
+    foreach_subdir("/data/local/tmp/user/0",
+                   [&](const std::string &filename) { result.push_back(filename); });
+
+    EXPECT_THAT(result, UnorderedElementsAre("com.foo", "com.bar"));
+}
+
 TEST_F(UtilsTest, TestSdkSandboxDataPaths) {
     // Ce data paths
     EXPECT_EQ("/data/misc_ce/0/sdksandbox",
@@ -673,6 +693,8 @@
               create_data_misc_sdk_sandbox_shared_path(nullptr, true, 0, "com.foo"));
     EXPECT_EQ("/data/misc_ce/10/sdksandbox/com.foo/shared",
               create_data_misc_sdk_sandbox_shared_path(nullptr, true, 10, "com.foo"));
+    EXPECT_EQ("/data/misc_ce/10/sdksandbox/com.foo/bar@random",
+              create_data_misc_sdk_sandbox_sdk_path(nullptr, true, 10, "com.foo", "bar", "random"));
 
     // De data paths
     EXPECT_EQ("/data/misc_de/0/sdksandbox",
@@ -688,6 +710,9 @@
               create_data_misc_sdk_sandbox_shared_path(nullptr, false, 0, "com.foo"));
     EXPECT_EQ("/data/misc_de/10/sdksandbox/com.foo/shared",
               create_data_misc_sdk_sandbox_shared_path(nullptr, false, 10, "com.foo"));
+    EXPECT_EQ("/data/misc_de/10/sdksandbox/com.foo/bar@random",
+              create_data_misc_sdk_sandbox_sdk_path(nullptr, false, 10, "com.foo", "bar",
+                                                    "random"));
 }
 
 TEST_F(UtilsTest, WaitChild) {
diff --git a/cmds/installd/utils.cpp b/cmds/installd/utils.cpp
index 992425d..8cfd123 100644
--- a/cmds/installd/utils.cpp
+++ b/cmds/installd/utils.cpp
@@ -212,7 +212,7 @@
 
 /**
  * Create the path name where code data for all codes in a particular app will be stored.
- * E.g. /data/misc_ce/0/sdksandbox/<app-name>
+ * E.g. /data/misc_ce/0/sdksandbox/<package-name>
  */
 std::string create_data_misc_sdk_sandbox_package_path(const char* volume_uuid, bool isCeData,
                                                       userid_t user, const char* package_name) {
@@ -224,7 +224,7 @@
 
 /**
  * Create the path name where shared code data for a particular app will be stored.
- * E.g. /data/misc_ce/0/sdksandbox/<app-name>/shared
+ * E.g. /data/misc_ce/0/sdksandbox/<package-name>/shared
  */
 std::string create_data_misc_sdk_sandbox_shared_path(const char* volume_uuid, bool isCeData,
                                                      userid_t user, const char* package_name) {
@@ -234,6 +234,19 @@
                                 .c_str());
 }
 
+/**
+ * Create the path name where per-code level data for a particular app will be stored.
+ * E.g. /data/misc_ce/0/sdksandbox/<package-name>/<sdk-name>-<random-suffix>
+ */
+std::string create_data_misc_sdk_sandbox_sdk_path(const char* volume_uuid, bool isCeData,
+                                                  userid_t user, const char* package_name,
+                                                  const char* sdk_name, const char* randomSuffix) {
+    check_package_name(sdk_name);
+    auto package_path =
+            create_data_misc_sdk_sandbox_package_path(volume_uuid, isCeData, user, package_name);
+    return StringPrintf("%s/%s@%s", package_path.c_str(), sdk_name, randomSuffix);
+}
+
 std::string create_data_misc_ce_rollback_base_path(const char* volume_uuid, userid_t user) {
     return StringPrintf("%s/misc_ce/%u/rollback", create_data_path(volume_uuid).c_str(), user);
 }
@@ -696,6 +709,34 @@
     return std::unique_ptr<DIR, DirCloser>(::opendir(dir));
 }
 
+// Collects filename of subdirectories of given directory and passes it to the function
+int foreach_subdir(const std::string& pathname, const std::function<void(const std::string&)> fn) {
+    auto dir = open_dir(pathname.c_str());
+    if (!dir) return -1;
+
+    int dfd = dirfd(dir.get());
+    if (dfd < 0) {
+        ALOGE("Couldn't dirfd %s: %s\n", pathname.c_str(), strerror(errno));
+        return -1;
+    }
+
+    struct dirent* de;
+    while ((de = readdir(dir.get()))) {
+        if (de->d_type != DT_DIR) {
+            continue;
+        }
+
+        std::string name{de->d_name};
+        // always skip "." and ".."
+        if (name == "." || name == "..") {
+            continue;
+        }
+        fn(name);
+    }
+
+    return 0;
+}
+
 void cleanup_invalid_package_dirs_under_path(const std::string& pathname) {
     auto dir = open_dir(pathname.c_str());
     if (!dir) {
diff --git a/cmds/installd/utils.h b/cmds/installd/utils.h
index 4b56f99..54d77f9 100644
--- a/cmds/installd/utils.h
+++ b/cmds/installd/utils.h
@@ -32,6 +32,7 @@
 
 #define MEASURE_DEBUG 0
 #define FIXUP_DEBUG 0
+#define SDK_DEBUG 1
 
 #define BYPASS_QUOTA 0
 #define BYPASS_SDCARDFS 0
@@ -66,6 +67,9 @@
                                                       userid_t userid, const char* package_name);
 std::string create_data_misc_sdk_sandbox_shared_path(const char* volume_uuid, bool isCeData,
                                                      userid_t userid, const char* package_name);
+std::string create_data_misc_sdk_sandbox_sdk_path(const char* volume_uuid, bool isCeData,
+                                                  userid_t userid, const char* package_name,
+                                                  const char* sdk_name, const char* randomSuffix);
 
 std::string create_data_misc_ce_rollback_base_path(const char* volume_uuid, userid_t user);
 std::string create_data_misc_de_rollback_base_path(const char* volume_uuid, userid_t user);
@@ -130,6 +134,8 @@
 bool is_renamed_deleted_dir(const std::string& path);
 int rename_delete_dir_contents_and_dir(const std::string& pathname, bool ignore_if_missing = true);
 
+int foreach_subdir(const std::string& pathname, std::function<void(const std::string&)> fn);
+
 void cleanup_invalid_package_dirs_under_path(const std::string& pathname);
 
 int delete_dir_contents(const char *pathname,