DataLoader lifecycle.

- restarting DataLoaders for not fully downloaded mounts,
- exponential backoff retry on DataLoader restart.

Bug: 173223115
Bug: 160634487
Test: atest PackageManagerShellCommandTest PackageManagerShellCommandIncrementalTest IncrementalServiceTest PackageManagerServiceTest ChecksumsTest
Change-Id: I5a93fd515ef0fec09452ce7bb4cd79fbd283d45f
diff --git a/services/incremental/test/IncrementalServiceTest.cpp b/services/incremental/test/IncrementalServiceTest.cpp
index 8713f9d..ab491ef 100644
--- a/services/incremental/test/IncrementalServiceTest.cpp
+++ b/services/incremental/test/IncrementalServiceTest.cpp
@@ -208,8 +208,9 @@
         EXPECT_TRUE(mDataLoaderHolder != nullptr);
     }
 
-    MOCK_CONST_METHOD4(bindToDataLoader,
+    MOCK_CONST_METHOD5(bindToDataLoader,
                        binder::Status(int32_t mountId, const DataLoaderParamsParcel& params,
+                                      int bindDelayMs,
                                       const sp<IDataLoaderStatusListener>& listener,
                                       bool* _aidl_return));
     MOCK_CONST_METHOD2(getDataLoader,
@@ -217,11 +218,11 @@
     MOCK_CONST_METHOD1(unbindFromDataLoader, binder::Status(int32_t mountId));
 
     void bindToDataLoaderSuccess() {
-        ON_CALL(*this, bindToDataLoader(_, _, _, _))
+        ON_CALL(*this, bindToDataLoader(_, _, _, _, _))
                 .WillByDefault(Invoke(this, &MockDataLoaderManager::bindToDataLoaderOk));
     }
     void bindToDataLoaderFails() {
-        ON_CALL(*this, bindToDataLoader(_, _, _, _))
+        ON_CALL(*this, bindToDataLoader(_, _, _, _, _))
                 .WillByDefault(Return(
                         (binder::Status::fromExceptionCode(1, String8("failed to prepare")))));
     }
@@ -234,6 +235,7 @@
                 .WillByDefault(Invoke(this, &MockDataLoaderManager::unbindFromDataLoaderOk));
     }
     binder::Status bindToDataLoaderOk(int32_t mountId, const DataLoaderParamsParcel& params,
+                                      int bindDelayMs,
                                       const sp<IDataLoaderStatusListener>& listener,
                                       bool* _aidl_return) {
         mId = mountId;
@@ -245,6 +247,40 @@
         }
         return binder::Status::ok();
     }
+    binder::Status bindToDataLoaderOkWith10sDelay(int32_t mountId,
+                                                  const DataLoaderParamsParcel& params,
+                                                  int bindDelayMs,
+                                                  const sp<IDataLoaderStatusListener>& listener,
+                                                  bool* _aidl_return) {
+        CHECK(1000 * 9 <= bindDelayMs && bindDelayMs <= 1000 * 11) << bindDelayMs;
+        return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
+    }
+    binder::Status bindToDataLoaderOkWith100sDelay(int32_t mountId,
+                                                   const DataLoaderParamsParcel& params,
+                                                   int bindDelayMs,
+                                                   const sp<IDataLoaderStatusListener>& listener,
+                                                   bool* _aidl_return) {
+        CHECK(1000 * 9 * 9 < bindDelayMs && bindDelayMs < 1000 * 11 * 11) << bindDelayMs;
+        return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
+    }
+    binder::Status bindToDataLoaderOkWith1000sDelay(int32_t mountId,
+                                                    const DataLoaderParamsParcel& params,
+                                                    int bindDelayMs,
+                                                    const sp<IDataLoaderStatusListener>& listener,
+                                                    bool* _aidl_return) {
+        CHECK(1000 * 9 * 9 * 9 < bindDelayMs && bindDelayMs < 1000 * 11 * 11 * 11) << bindDelayMs;
+        return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
+    }
+    binder::Status bindToDataLoaderOkWith10000sDelay(int32_t mountId,
+                                                     const DataLoaderParamsParcel& params,
+                                                     int bindDelayMs,
+                                                     const sp<IDataLoaderStatusListener>& listener,
+                                                     bool* _aidl_return) {
+        CHECK(1000 * 9 * 9 * 9 * 9 < bindDelayMs && bindDelayMs < 1000 * 11 * 11 * 11 * 11)
+                << bindDelayMs;
+        return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
+    }
+
     binder::Status getDataLoaderOk(int32_t mountId, sp<IDataLoader>* _aidl_return) {
         *_aidl_return = mDataLoader;
         return binder::Status::ok();
@@ -261,6 +297,9 @@
     void setDataLoaderStatusUnavailable() {
         mListener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_UNAVAILABLE);
     }
+    void setDataLoaderStatusUnrecoverable() {
+        mListener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_UNRECOVERABLE);
+    }
     binder::Status unbindFromDataLoaderOk(int32_t id) {
         if (mDataLoader) {
             if (auto status = mDataLoader->destroy(id); !status.isOk()) {
@@ -676,7 +715,7 @@
 
 TEST_F(IncrementalServiceTest, testCreateStorageMountIncFsFails) {
     mVold->mountIncFsFails();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(0);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
     TemporaryDir tempDir;
     int storageId =
             mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
@@ -686,7 +725,7 @@
 
 TEST_F(IncrementalServiceTest, testCreateStorageMountIncFsInvalidControlParcel) {
     mVold->mountIncFsInvalidControlParcel();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(0);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
     TemporaryDir tempDir;
     int storageId =
@@ -698,7 +737,7 @@
 TEST_F(IncrementalServiceTest, testCreateStorageMakeFileFails) {
     mVold->mountIncFsSuccess();
     mIncFs->makeFileFails();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(0);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
     EXPECT_CALL(*mVold, unmountIncFs(_));
     TemporaryDir tempDir;
@@ -712,7 +751,7 @@
     mVold->mountIncFsSuccess();
     mIncFs->makeFileSuccess();
     mVold->bindMountFails();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(0);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
     EXPECT_CALL(*mVold, unmountIncFs(_));
     TemporaryDir tempDir;
@@ -727,7 +766,7 @@
     mIncFs->makeFileSuccess();
     mVold->bindMountSuccess();
     mDataLoaderManager->bindToDataLoaderFails();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(0);
     EXPECT_CALL(*mDataLoader, start(_)).Times(0);
@@ -755,11 +794,11 @@
     mIncrementalService->deleteStorage(storageId);
 }
 
-TEST_F(IncrementalServiceTest, testDataLoaderDestroyed) {
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(2);
+TEST_F(IncrementalServiceTest, testDataLoaderDestroyedAndDelayed) {
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(6);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
-    EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(2);
-    EXPECT_CALL(*mDataLoader, start(_)).Times(2);
+    EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(6);
+    EXPECT_CALL(*mDataLoader, start(_)).Times(6);
     EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
     EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
     TemporaryDir tempDir;
@@ -768,13 +807,38 @@
                                                IncrementalService::CreateOptions::CreateNew);
     ASSERT_GE(storageId, 0);
     mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {}, {}, {});
+
     // Simulated crash/other connection breakage.
+
+    ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
+            .WillByDefault(Invoke(mDataLoaderManager,
+                                  &MockDataLoaderManager::bindToDataLoaderOkWith10sDelay));
+    mDataLoaderManager->setDataLoaderStatusDestroyed();
+
+    ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
+            .WillByDefault(Invoke(mDataLoaderManager,
+                                  &MockDataLoaderManager::bindToDataLoaderOkWith100sDelay));
+    mDataLoaderManager->setDataLoaderStatusDestroyed();
+
+    ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
+            .WillByDefault(Invoke(mDataLoaderManager,
+                                  &MockDataLoaderManager::bindToDataLoaderOkWith1000sDelay));
+    mDataLoaderManager->setDataLoaderStatusDestroyed();
+
+    ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
+            .WillByDefault(Invoke(mDataLoaderManager,
+                                  &MockDataLoaderManager::bindToDataLoaderOkWith10000sDelay));
+    mDataLoaderManager->setDataLoaderStatusDestroyed();
+
+    ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
+            .WillByDefault(Invoke(mDataLoaderManager,
+                                  &MockDataLoaderManager::bindToDataLoaderOkWith10000sDelay));
     mDataLoaderManager->setDataLoaderStatusDestroyed();
 }
 
 TEST_F(IncrementalServiceTest, testStartDataLoaderCreate) {
     mDataLoader->initializeCreateOkNoStatus();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoader, start(_)).Times(1);
@@ -793,7 +857,7 @@
 
 TEST_F(IncrementalServiceTest, testStartDataLoaderPendingStart) {
     mDataLoader->initializeCreateOkNoStatus();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoader, start(_)).Times(1);
@@ -811,7 +875,7 @@
 
 TEST_F(IncrementalServiceTest, testStartDataLoaderCreateUnavailable) {
     mDataLoader->initializeCreateOkNoStatus();
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoader, start(_)).Times(0);
@@ -827,12 +891,30 @@
     mDataLoaderManager->setDataLoaderStatusUnavailable();
 }
 
+TEST_F(IncrementalServiceTest, testStartDataLoaderCreateUnrecoverable) {
+    mDataLoader->initializeCreateOkNoStatus();
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
+    EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoader, start(_)).Times(0);
+    EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
+    EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
+    TemporaryDir tempDir;
+    int storageId =
+            mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
+                                               IncrementalService::CreateOptions::CreateNew);
+    ASSERT_GE(storageId, 0);
+    ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
+                                                  {}, {}));
+    mDataLoaderManager->setDataLoaderStatusUnrecoverable();
+}
+
 TEST_F(IncrementalServiceTest, testStartDataLoaderRecreateOnPendingReads) {
     mIncFs->waitForPendingReadsSuccess();
     mIncFs->openMountSuccess();
     mDataLoader->initializeCreateOkNoStatus();
 
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(2);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(2);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(2);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(2);
     EXPECT_CALL(*mDataLoader, start(_)).Times(0);
@@ -856,7 +938,7 @@
 TEST_F(IncrementalServiceTest, testStartDataLoaderUnhealthyStorage) {
     mIncFs->openMountSuccess();
 
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoader, start(_)).Times(1);
@@ -1406,7 +1488,7 @@
 }
 
 TEST_F(IncrementalServiceTest, testPerUidTimeoutsTooShort) {
-    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
     EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
     EXPECT_CALL(*mDataLoader, start(_)).Times(1);