Add recoverability detection to RescueParty and PackageWatchdog
PackageWatchdog will apply high user impact actions only in case of
network failure.
RescueParty updated the list of rescue levels and their corresponding
impact.
Test: unit and manual tests
Bug: 310236690
Change-Id: Iff34f6051b801a0d45149c317a7c9f344a2bd984
diff --git a/tests/PackageWatchdog/Android.bp b/tests/PackageWatchdog/Android.bp
index e0e6c4c..2c5fdd3 100644
--- a/tests/PackageWatchdog/Android.bp
+++ b/tests/PackageWatchdog/Android.bp
@@ -28,8 +28,10 @@
static_libs: [
"junit",
"mockito-target-extended-minus-junit4",
+ "flag-junit",
"frameworks-base-testutils",
"androidx.test.rules",
+ "PlatformProperties",
"services.core",
"services.net",
"truth",
diff --git a/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java b/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java
new file mode 100644
index 0000000..081da11
--- /dev/null
+++ b/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java
@@ -0,0 +1,644 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server;
+
+import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;
+
+import static com.android.dx.mockito.inline.extended.ExtendedMockito.doAnswer;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import static org.junit.Assert.assertFalse;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import android.Manifest;
+import android.content.Context;
+import android.content.pm.PackageInfo;
+import android.content.pm.PackageManager;
+import android.content.pm.VersionedPackage;
+import android.content.rollback.PackageRollbackInfo;
+import android.content.rollback.RollbackInfo;
+import android.content.rollback.RollbackManager;
+import android.crashrecovery.flags.Flags;
+import android.net.ConnectivityModuleConnector;
+import android.net.ConnectivityModuleConnector.ConnectivityModuleHealthListener;
+import android.os.Handler;
+import android.os.SystemProperties;
+import android.os.test.TestLooper;
+import android.platform.test.flag.junit.SetFlagsRule;
+import android.provider.DeviceConfig;
+import android.util.AtomicFile;
+
+import androidx.test.InstrumentationRegistry;
+
+import com.android.dx.mockito.inline.extended.ExtendedMockito;
+import com.android.server.RescueParty.RescuePartyObserver;
+import com.android.server.pm.ApexManager;
+import com.android.server.rollback.RollbackPackageHealthObserver;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Answers;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.mockito.MockitoSession;
+import org.mockito.quality.Strictness;
+import org.mockito.stubbing.Answer;
+
+import java.io.File;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+/**
+ * Test CrashRecovery, integration tests that include PackageWatchdog, RescueParty and
+ * RollbackPackageHealthObserver
+ */
+public class CrashRecoveryTest {
+ private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG =
+ "persist.device_config.configuration.disable_rescue_party";
+
+ private static final String APP_A = "com.package.a";
+ private static final String APP_B = "com.package.b";
+ private static final String APP_C = "com.package.c";
+ private static final long VERSION_CODE = 1L;
+ private static final long SHORT_DURATION = TimeUnit.SECONDS.toMillis(1);
+
+ private static final RollbackInfo ROLLBACK_INFO_LOW = getRollbackInfo(APP_A, VERSION_CODE, 1,
+ PackageManager.ROLLBACK_USER_IMPACT_LOW);
+ private static final RollbackInfo ROLLBACK_INFO_HIGH = getRollbackInfo(APP_B, VERSION_CODE, 2,
+ PackageManager.ROLLBACK_USER_IMPACT_HIGH);
+ private static final RollbackInfo ROLLBACK_INFO_MANUAL = getRollbackInfo(APP_C, VERSION_CODE, 3,
+ PackageManager.ROLLBACK_USER_IMPACT_ONLY_MANUAL);
+
+ @Rule
+ public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();
+
+ private final TestClock mTestClock = new TestClock();
+ private TestLooper mTestLooper;
+ private Context mSpyContext;
+ // Keep track of all created watchdogs to apply device config changes
+ private List<PackageWatchdog> mAllocatedWatchdogs;
+ @Mock
+ private ConnectivityModuleConnector mConnectivityModuleConnector;
+ @Mock
+ private PackageManager mMockPackageManager;
+ @Mock(answer = Answers.RETURNS_DEEP_STUBS)
+ private ApexManager mApexManager;
+ @Mock
+ RollbackManager mRollbackManager;
+ // Mock only sysprop apis
+ private PackageWatchdog.BootThreshold mSpyBootThreshold;
+ @Captor
+ private ArgumentCaptor<ConnectivityModuleHealthListener> mConnectivityModuleCallbackCaptor;
+ private MockitoSession mSession;
+ private HashMap<String, String> mSystemSettingsMap;
+ private HashMap<String, String> mCrashRecoveryPropertiesMap;
+
+ @Before
+ public void setUp() throws Exception {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ MockitoAnnotations.initMocks(this);
+ new File(InstrumentationRegistry.getContext().getFilesDir(),
+ "package-watchdog.xml").delete();
+ adoptShellPermissions(Manifest.permission.READ_DEVICE_CONFIG,
+ Manifest.permission.WRITE_DEVICE_CONFIG);
+ mTestLooper = new TestLooper();
+ mSpyContext = spy(InstrumentationRegistry.getContext());
+ when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager);
+ when(mMockPackageManager.getPackageInfo(anyString(), anyInt())).then(inv -> {
+ final PackageInfo res = new PackageInfo();
+ res.packageName = inv.getArgument(0);
+ res.setLongVersionCode(VERSION_CODE);
+ return res;
+ });
+ mSession = ExtendedMockito.mockitoSession()
+ .initMocks(this)
+ .strictness(Strictness.LENIENT)
+ .spyStatic(SystemProperties.class)
+ .spyStatic(RescueParty.class)
+ .startMocking();
+ mSystemSettingsMap = new HashMap<>();
+
+ // Mock SystemProperties setter and various getters
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ String value = invocationOnMock.getArgument(1);
+
+ mSystemSettingsMap.put(key, value);
+ return null;
+ }
+ ).when(() -> SystemProperties.set(anyString(), anyString()));
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ int defaultValue = invocationOnMock.getArgument(1);
+
+ String storedValue = mSystemSettingsMap.get(key);
+ return storedValue == null ? defaultValue : Integer.parseInt(storedValue);
+ }
+ ).when(() -> SystemProperties.getInt(anyString(), anyInt()));
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ long defaultValue = invocationOnMock.getArgument(1);
+
+ String storedValue = mSystemSettingsMap.get(key);
+ return storedValue == null ? defaultValue : Long.parseLong(storedValue);
+ }
+ ).when(() -> SystemProperties.getLong(anyString(), anyLong()));
+
+ doAnswer((Answer<Boolean>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ boolean defaultValue = invocationOnMock.getArgument(1);
+
+ String storedValue = mSystemSettingsMap.get(key);
+ return storedValue == null ? defaultValue : Boolean.parseBoolean(storedValue);
+ }
+ ).when(() -> SystemProperties.getBoolean(anyString(), anyBoolean()));
+
+ SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));
+ SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(false));
+
+ DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
+ PackageWatchdog.PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED,
+ Boolean.toString(true), false);
+
+ DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
+ PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
+ Integer.toString(PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT), false);
+
+ mAllocatedWatchdogs = new ArrayList<>();
+ RescuePartyObserver.reset();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ dropShellPermissions();
+ mSession.finishMocking();
+ // Clean up listeners since too many listeners will delay notifications significantly
+ for (PackageWatchdog watchdog : mAllocatedWatchdogs) {
+ watchdog.removePropertyChangedListener();
+ }
+ mAllocatedWatchdogs.clear();
+ }
+
+ @Test
+ public void testBootLoopWithRescueParty() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ RescuePartyObserver rescuePartyObserver = setUpRescuePartyObserver(watchdog);
+
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);
+ int bootCounter = 0;
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(1);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(2);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
+
+ int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;
+ for (int i = 0; i < bootLoopThreshold; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(3);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(4);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(5);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(6);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);
+ }
+
+ @Test
+ public void testBootLoopWithRollbackPackageHealthObserver() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ RollbackPackageHealthObserver rollbackObserver =
+ setUpRollbackPackageHealthObserver(watchdog);
+
+ verify(rollbackObserver, never()).executeBootLoopMitigation(1);
+ int bootCounter = 0;
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rollbackObserver).executeBootLoopMitigation(1);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ // Update the list of available rollbacks after executing bootloop mitigation once
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,
+ ROLLBACK_INFO_MANUAL));
+
+ int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;
+ for (int i = 0; i < bootLoopThreshold; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rollbackObserver).executeBootLoopMitigation(2);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+
+ // Update the list of available rollbacks after executing bootloop mitigation once
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+ }
+
+ @Test
+ public void testBootLoopWithRescuePartyAndRollbackPackageHealthObserver() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ RescuePartyObserver rescuePartyObserver = setUpRescuePartyObserver(watchdog);
+ RollbackPackageHealthObserver rollbackObserver =
+ setUpRollbackPackageHealthObserver(watchdog);
+
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(1);
+ int bootCounter = 0;
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(1);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(1);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(2);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
+ verify(rollbackObserver).executeBootLoopMitigation(1);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+ // Update the list of available rollbacks after executing bootloop mitigation once
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,
+ ROLLBACK_INFO_MANUAL));
+
+ int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;
+ for (int i = 0; i < bootLoopThreshold; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(3);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(4);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(5);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
+ verify(rollbackObserver).executeBootLoopMitigation(2);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+ // Update the list of available rollbacks after executing bootloop mitigation
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(6);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+ }
+
+ RollbackPackageHealthObserver setUpRollbackPackageHealthObserver(PackageWatchdog watchdog) {
+ RollbackPackageHealthObserver rollbackObserver =
+ spy(new RollbackPackageHealthObserver(mSpyContext, mApexManager));
+ when(mSpyContext.getSystemService(RollbackManager.class)).thenReturn(mRollbackManager);
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_LOW,
+ ROLLBACK_INFO_HIGH, ROLLBACK_INFO_MANUAL));
+ when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager);
+
+ watchdog.registerHealthObserver(rollbackObserver);
+ return rollbackObserver;
+ }
+
+ RescuePartyObserver setUpRescuePartyObserver(PackageWatchdog watchdog) {
+ setCrashRecoveryPropRescueBootCount(0);
+ RescuePartyObserver rescuePartyObserver = spy(RescuePartyObserver.getInstance(mSpyContext));
+ assertFalse(RescueParty.isRebootPropertySet());
+ watchdog.registerHealthObserver(rescuePartyObserver);
+ return rescuePartyObserver;
+ }
+
+ private static RollbackInfo getRollbackInfo(String packageName, long versionCode,
+ int rollbackId, int rollbackUserImpact) {
+ VersionedPackage appFrom = new VersionedPackage(packageName, versionCode + 1);
+ VersionedPackage appTo = new VersionedPackage(packageName, versionCode);
+ PackageRollbackInfo packageRollbackInfo = new PackageRollbackInfo(appFrom, appTo, null,
+ null, false, false, null);
+ RollbackInfo rollbackInfo = new RollbackInfo(rollbackId, List.of(packageRollbackInfo),
+ false, null, 111, rollbackUserImpact);
+ return rollbackInfo;
+ }
+
+ private void adoptShellPermissions(String... permissions) {
+ androidx.test.platform.app.InstrumentationRegistry
+ .getInstrumentation()
+ .getUiAutomation()
+ .adoptShellPermissionIdentity(permissions);
+ }
+
+ private void dropShellPermissions() {
+ androidx.test.platform.app.InstrumentationRegistry
+ .getInstrumentation()
+ .getUiAutomation()
+ .dropShellPermissionIdentity();
+ }
+
+
+ private PackageWatchdog createWatchdog() {
+ return createWatchdog(new TestController(), true /* withPackagesReady */);
+ }
+
+ private PackageWatchdog createWatchdog(TestController controller, boolean withPackagesReady) {
+ AtomicFile policyFile =
+ new AtomicFile(new File(mSpyContext.getFilesDir(), "package-watchdog.xml"));
+ Handler handler = new Handler(mTestLooper.getLooper());
+ PackageWatchdog watchdog =
+ new PackageWatchdog(mSpyContext, policyFile, handler, handler, controller,
+ mConnectivityModuleConnector, mTestClock);
+ mockCrashRecoveryProperties(watchdog);
+
+ // Verify controller is not automatically started
+ assertThat(controller.mIsEnabled).isFalse();
+ if (withPackagesReady) {
+ // Only capture the NetworkStack callback for the latest registered watchdog
+ reset(mConnectivityModuleConnector);
+ watchdog.onPackagesReady();
+ // Verify controller by default is started when packages are ready
+ assertThat(controller.mIsEnabled).isTrue();
+
+ verify(mConnectivityModuleConnector).registerHealthListener(
+ mConnectivityModuleCallbackCaptor.capture());
+ }
+ mAllocatedWatchdogs.add(watchdog);
+ return watchdog;
+ }
+
+ // Mock CrashRecoveryProperties as they cannot be accessed due to SEPolicy restrictions
+ private void mockCrashRecoveryProperties(PackageWatchdog watchdog) {
+ mCrashRecoveryPropertiesMap = new HashMap<>();
+
+ // mock properties in RescueParty
+ try {
+
+ doAnswer((Answer<Boolean>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.attempting_factory_reset", "false");
+ return Boolean.parseBoolean(storedValue);
+ }).when(() -> RescueParty.isFactoryResetPropertySet());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ boolean value = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.attempting_factory_reset",
+ Boolean.toString(value));
+ return null;
+ }).when(() -> RescueParty.setFactoryResetProperty(anyBoolean()));
+
+ doAnswer((Answer<Boolean>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.attempting_reboot", "false");
+ return Boolean.parseBoolean(storedValue);
+ }).when(() -> RescueParty.isRebootPropertySet());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ boolean value = invocationOnMock.getArgument(0);
+ setCrashRecoveryPropAttemptingReboot(value);
+ return null;
+ }).when(() -> RescueParty.setRebootProperty(anyBoolean()));
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("persist.crashrecovery.last_factory_reset", "0");
+ return Long.parseLong(storedValue);
+ }).when(() -> RescueParty.getLastFactoryResetTimeMs());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ long value = invocationOnMock.getArgument(0);
+ setCrashRecoveryPropLastFactoryReset(value);
+ return null;
+ }).when(() -> RescueParty.setLastFactoryResetTimeMs(anyLong()));
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.max_rescue_level_attempted", "0");
+ return Integer.parseInt(storedValue);
+ }).when(() -> RescueParty.getMaxRescueLevelAttempted());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ int value = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.max_rescue_level_attempted",
+ Integer.toString(value));
+ return null;
+ }).when(() -> RescueParty.setMaxRescueLevelAttempted(anyInt()));
+
+ } catch (Exception e) {
+ // tests will fail, just printing the error
+ System.out.println("Error while mocking crashrecovery properties " + e.getMessage());
+ }
+
+ try {
+ if (Flags.recoverabilityDetection()) {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+ } else {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
+ }
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.rescue_boot_count", "0");
+ return Integer.parseInt(storedValue);
+ }).when(mSpyBootThreshold).getCount();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ int count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_count",
+ Integer.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setCount(anyInt());
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.boot_mitigation_count", "0");
+ return Integer.parseInt(storedValue);
+ }).when(mSpyBootThreshold).getMitigationCount();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ int count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.boot_mitigation_count",
+ Integer.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setMitigationCount(anyInt());
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.rescue_boot_start", "0");
+ return Long.parseLong(storedValue);
+ }).when(mSpyBootThreshold).getStart();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ long count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_start",
+ Long.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setStart(anyLong());
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.boot_mitigation_start", "0");
+ return Long.parseLong(storedValue);
+ }).when(mSpyBootThreshold).getMitigationStart();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ long count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.boot_mitigation_start",
+ Long.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setMitigationStart(anyLong());
+
+ Field mBootThresholdField = watchdog.getClass().getDeclaredField("mBootThreshold");
+ mBootThresholdField.setAccessible(true);
+ mBootThresholdField.set(watchdog, mSpyBootThreshold);
+ } catch (Exception e) {
+ // tests will fail, just printing the error
+ System.out.println("Error detected while spying BootThreshold" + e.getMessage());
+ }
+ }
+
+ private void setCrashRecoveryPropRescueBootCount(int count) {
+ mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_count",
+ Integer.toString(count));
+ }
+
+ private void setCrashRecoveryPropAttemptingReboot(boolean value) {
+ mCrashRecoveryPropertiesMap.put("crashrecovery.attempting_reboot",
+ Boolean.toString(value));
+ }
+
+ private void setCrashRecoveryPropLastFactoryReset(long value) {
+ mCrashRecoveryPropertiesMap.put("persist.crashrecovery.last_factory_reset",
+ Long.toString(value));
+ }
+
+ private static class TestController extends ExplicitHealthCheckController {
+ TestController() {
+ super(null /* controller */);
+ }
+
+ private boolean mIsEnabled;
+ private List<String> mSupportedPackages = new ArrayList<>();
+ private List<String> mRequestedPackages = new ArrayList<>();
+ private Consumer<List<PackageConfig>> mSupportedConsumer;
+ private List<Set> mSyncRequests = new ArrayList<>();
+
+ @Override
+ public void setEnabled(boolean enabled) {
+ mIsEnabled = enabled;
+ if (!mIsEnabled) {
+ mSupportedPackages.clear();
+ }
+ }
+
+ @Override
+ public void setCallbacks(Consumer<String> passedConsumer,
+ Consumer<List<PackageConfig>> supportedConsumer, Runnable notifySyncRunnable) {
+ mSupportedConsumer = supportedConsumer;
+ }
+
+ @Override
+ public void syncRequests(Set<String> packages) {
+ mSyncRequests.add(packages);
+ mRequestedPackages.clear();
+ if (mIsEnabled) {
+ packages.retainAll(mSupportedPackages);
+ mRequestedPackages.addAll(packages);
+ List<PackageConfig> packageConfigs = new ArrayList<>();
+ for (String packageName: packages) {
+ packageConfigs.add(new PackageConfig(packageName, SHORT_DURATION));
+ }
+ mSupportedConsumer.accept(packageConfigs);
+ } else {
+ mSupportedConsumer.accept(Collections.emptyList());
+ }
+ }
+ }
+
+ private static class TestClock implements PackageWatchdog.SystemClock {
+ // Note 0 is special to the internal clock of PackageWatchdog. We need to start from
+ // a non-zero value in order not to disrupt the logic of PackageWatchdog.
+ private long mUpTimeMillis = 1;
+ @Override
+ public long uptimeMillis() {
+ return mUpTimeMillis;
+ }
+ }
+}
diff --git a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
index 75284c7..4f27e06 100644
--- a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
+++ b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
@@ -36,11 +36,13 @@
import android.content.pm.PackageInfo;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
+import android.crashrecovery.flags.Flags;
import android.net.ConnectivityModuleConnector;
import android.net.ConnectivityModuleConnector.ConnectivityModuleHealthListener;
import android.os.Handler;
import android.os.SystemProperties;
import android.os.test.TestLooper;
+import android.platform.test.flag.junit.SetFlagsRule;
import android.provider.DeviceConfig;
import android.util.AtomicFile;
import android.util.Xml;
@@ -54,11 +56,13 @@
import com.android.modules.utils.TypedXmlSerializer;
import com.android.server.PackageWatchdog.HealthCheckState;
import com.android.server.PackageWatchdog.MonitoredPackage;
+import com.android.server.PackageWatchdog.ObserverInternal;
import com.android.server.PackageWatchdog.PackageHealthObserver;
import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
@@ -99,6 +103,10 @@
private static final String OBSERVER_NAME_4 = "observer4";
private static final long SHORT_DURATION = TimeUnit.SECONDS.toMillis(1);
private static final long LONG_DURATION = TimeUnit.SECONDS.toMillis(5);
+
+ @Rule
+ public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();
+
private final TestClock mTestClock = new TestClock();
private TestLooper mTestLooper;
private Context mSpyContext;
@@ -128,6 +136,7 @@
@Before
public void setUp() throws Exception {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
MockitoAnnotations.initMocks(this);
new File(InstrumentationRegistry.getContext().getFilesDir(),
"package-watchdog.xml").delete();
@@ -444,6 +453,7 @@
*/
@Test
public void testPackageFailureNotifyAllDifferentImpacts() throws Exception {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver observerNone = new TestObserver(OBSERVER_NAME_1,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
@@ -488,6 +498,52 @@
assertThat(observerLowPackages).containsExactly(APP_A);
}
+ @Test
+ public void testPackageFailureNotifyAllDifferentImpactsRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver observerNone = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
+ TestObserver observerHigh = new TestObserver(OBSERVER_NAME_2,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+ TestObserver observerMid = new TestObserver(OBSERVER_NAME_3,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ TestObserver observerLow = new TestObserver(OBSERVER_NAME_4,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+
+ // Start observing for all impact observers
+ watchdog.startObservingHealth(observerNone, Arrays.asList(APP_A, APP_B, APP_C, APP_D),
+ SHORT_DURATION);
+ watchdog.startObservingHealth(observerHigh, Arrays.asList(APP_A, APP_B, APP_C),
+ SHORT_DURATION);
+ watchdog.startObservingHealth(observerMid, Arrays.asList(APP_A, APP_B),
+ SHORT_DURATION);
+ watchdog.startObservingHealth(observerLow, Arrays.asList(APP_A),
+ SHORT_DURATION);
+
+ // Then fail all apps above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE),
+ new VersionedPackage(APP_B, VERSION_CODE),
+ new VersionedPackage(APP_C, VERSION_CODE),
+ new VersionedPackage(APP_D, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify least impact observers are notifed of package failures
+ List<String> observerNonePackages = observerNone.mMitigatedPackages;
+ List<String> observerHighPackages = observerHigh.mMitigatedPackages;
+ List<String> observerMidPackages = observerMid.mMitigatedPackages;
+ List<String> observerLowPackages = observerLow.mMitigatedPackages;
+
+ // APP_D failure observed by only observerNone is not caught cos its impact is none
+ assertThat(observerNonePackages).isEmpty();
+ // APP_C failure is caught by observerHigh cos it's the lowest impact observer
+ assertThat(observerHighPackages).containsExactly(APP_C);
+ // APP_B failure is caught by observerMid cos it's the lowest impact observer
+ assertThat(observerMidPackages).containsExactly(APP_B);
+ // APP_A failure is caught by observerLow cos it's the lowest impact observer
+ assertThat(observerLowPackages).containsExactly(APP_A);
+ }
+
/**
* Test package failure and least impact observers are notified successively.
* State transistions:
@@ -501,6 +557,7 @@
*/
@Test
public void testPackageFailureNotifyLeastImpactSuccessively() throws Exception {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver observerFirst = new TestObserver(OBSERVER_NAME_1,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
@@ -563,11 +620,76 @@
assertThat(observerSecond.mMitigatedPackages).isEmpty();
}
+ @Test
+ public void testPackageFailureNotifyLeastImpactSuccessivelyRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver observerFirst = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+ TestObserver observerSecond = new TestObserver(OBSERVER_NAME_2,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+
+ // Start observing for observerFirst and observerSecond with failure handling
+ watchdog.startObservingHealth(observerFirst, Arrays.asList(APP_A), LONG_DURATION);
+ watchdog.startObservingHealth(observerSecond, Arrays.asList(APP_A), LONG_DURATION);
+
+ // Then fail APP_A above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only observerFirst is notifed
+ assertThat(observerFirst.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observerSecond.mMitigatedPackages).isEmpty();
+
+ // After observerFirst handles failure, next action it has is high impact
+ observerFirst.mImpact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
+ observerFirst.mMitigatedPackages.clear();
+ observerSecond.mMitigatedPackages.clear();
+
+ // Then fail APP_A again above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only observerSecond is notifed cos it has least impact
+ assertThat(observerSecond.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observerFirst.mMitigatedPackages).isEmpty();
+
+ // After observerSecond handles failure, it has no further actions
+ observerSecond.mImpact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ observerFirst.mMitigatedPackages.clear();
+ observerSecond.mMitigatedPackages.clear();
+
+ // Then fail APP_A again above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only observerFirst is notifed cos it has the only action
+ assertThat(observerFirst.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observerSecond.mMitigatedPackages).isEmpty();
+
+ // After observerFirst handles failure, it too has no further actions
+ observerFirst.mImpact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ observerFirst.mMitigatedPackages.clear();
+ observerSecond.mMitigatedPackages.clear();
+
+ // Then fail APP_A again above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify no observer is notified cos no actions left
+ assertThat(observerFirst.mMitigatedPackages).isEmpty();
+ assertThat(observerSecond.mMitigatedPackages).isEmpty();
+ }
+
/**
* Test package failure and notifies only one observer even with observer impact tie.
*/
@Test
public void testPackageFailureNotifyOneSameImpact() throws Exception {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);
@@ -588,6 +710,28 @@
assertThat(observer2.mMitigatedPackages).isEmpty();
}
+ @Test
+ public void testPackageFailureNotifyOneSameImpactRecoverabilityDetection() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver observer1 = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+ TestObserver observer2 = new TestObserver(OBSERVER_NAME_2,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+
+ // Start observing for observer1 and observer2 with failure handling
+ watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION);
+ watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
+
+ // Then fail APP_A above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only one observer is notifed
+ assertThat(observer1.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observer2.mMitigatedPackages).isEmpty();
+ }
+
/**
* Test package passing explicit health checks does not fail and vice versa.
*/
@@ -818,6 +962,7 @@
@Test
public void testNetworkStackFailure() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
final PackageWatchdog wd = createWatchdog();
// Start observing with failure handling
@@ -835,6 +980,25 @@
assertThat(observer.mMitigatedPackages).containsExactly(APP_A);
}
+ @Test
+ public void testNetworkStackFailureRecoverabilityDetection() {
+ final PackageWatchdog wd = createWatchdog();
+
+ // Start observing with failure handling
+ TestObserver observer = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);
+ wd.startObservingHealth(observer, Collections.singletonList(APP_A), SHORT_DURATION);
+
+ // Notify of NetworkStack failure
+ mConnectivityModuleCallbackCaptor.getValue().onNetworkStackFailure(APP_A);
+
+ // Run handler so package failures are dispatched to observers
+ mTestLooper.dispatchAll();
+
+ // Verify the NetworkStack observer is notified
+ assertThat(observer.mMitigatedPackages).isEmpty();
+ }
+
/** Test default values are used when device property is invalid. */
@Test
public void testInvalidConfig_watchdogTriggerFailureCount() {
@@ -1045,6 +1209,7 @@
/** Ensure that boot loop mitigation is done when the number of boots meets the threshold. */
@Test
public void testBootLoopDetection_meetsThreshold() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
watchdog.registerHealthObserver(bootObserver);
@@ -1054,6 +1219,16 @@
assertThat(bootObserver.mitigatedBootLoop()).isTrue();
}
+ @Test
+ public void testBootLoopDetection_meetsThresholdRecoverability() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver.mitigatedBootLoop()).isTrue();
+ }
/**
* Ensure that boot loop mitigation is not done when the number of boots does not meet the
@@ -1071,10 +1246,43 @@
}
/**
+ * Ensure that boot loop mitigation is not done when the number of boots does not meet the
+ * threshold.
+ */
+ @Test
+ public void testBootLoopDetection_doesNotMeetThresholdRecoverabilityLowImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver.mitigatedBootLoop()).isFalse();
+ }
+
+ /**
+ * Ensure that boot loop mitigation is not done when the number of boots does not meet the
+ * threshold.
+ */
+ @Test
+ public void testBootLoopDetection_doesNotMeetThresholdRecoverabilityHighImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver.mitigatedBootLoop()).isFalse();
+ }
+
+ /**
* Ensure that boot loop mitigation is done for the observer with the lowest user impact
*/
@Test
public void testBootLoopMitigationDoneForLowestUserImpact() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver bootObserver1 = new TestObserver(OBSERVER_NAME_1);
bootObserver1.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
@@ -1089,11 +1297,28 @@
assertThat(bootObserver2.mitigatedBootLoop()).isFalse();
}
+ @Test
+ public void testBootLoopMitigationDoneForLowestUserImpactRecoverability() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver1 = new TestObserver(OBSERVER_NAME_1);
+ bootObserver1.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+ TestObserver bootObserver2 = new TestObserver(OBSERVER_NAME_2);
+ bootObserver2.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ watchdog.registerHealthObserver(bootObserver1);
+ watchdog.registerHealthObserver(bootObserver2);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver1.mitigatedBootLoop()).isTrue();
+ assertThat(bootObserver2.mitigatedBootLoop()).isFalse();
+ }
+
/**
* Ensure that the correct mitigation counts are sent to the boot loop observer.
*/
@Test
public void testMultipleBootLoopMitigation() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
watchdog.registerHealthObserver(bootObserver);
@@ -1114,6 +1339,64 @@
assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));
}
+ @Test
+ public void testMultipleBootLoopMitigationRecoverabilityLowImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);
+
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));
+ }
+
+ @Test
+ public void testMultipleBootLoopMitigationRecoverabilityHighImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);
+
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));
+ }
+
/**
* Ensure that passing a null list of failed packages does not cause any mitigation logic to
* execute.
@@ -1304,6 +1587,78 @@
}
/**
+ * Ensure that a {@link ObserverInternal} may be correctly written and read in order to persist
+ * across reboots.
+ */
+ @Test
+ @SuppressWarnings("GuardedBy")
+ public void testWritingAndReadingObserverInternalRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+
+ LongArrayQueue mitigationCalls = new LongArrayQueue();
+ mitigationCalls.addLast(1000);
+ mitigationCalls.addLast(2000);
+ mitigationCalls.addLast(3000);
+ MonitoredPackage writePkg = watchdog.newMonitoredPackage(
+ "test.package", 1000, 2000, true, mitigationCalls);
+ final int bootMitigationCount = 4;
+ ObserverInternal writeObserver = new ObserverInternal("test", List.of(writePkg),
+ bootMitigationCount);
+
+ // Write the observer
+ File tmpFile = File.createTempFile("observer-watchdog-test", ".xml");
+ AtomicFile testFile = new AtomicFile(tmpFile);
+ FileOutputStream stream = testFile.startWrite();
+ TypedXmlSerializer outputSerializer = Xml.resolveSerializer(stream);
+ outputSerializer.startDocument(null, true);
+ writeObserver.writeLocked(outputSerializer);
+ outputSerializer.endDocument();
+ testFile.finishWrite(stream);
+
+ // Read the observer
+ TypedXmlPullParser parser = Xml.resolvePullParser(testFile.openRead());
+ XmlUtils.beginDocument(parser, "observer");
+ ObserverInternal readObserver = ObserverInternal.read(parser, watchdog);
+
+ assertThat(readObserver.name).isEqualTo(writeObserver.name);
+ assertThat(readObserver.getBootMitigationCount()).isEqualTo(bootMitigationCount);
+ }
+
+ /**
+ * Ensure that boot mitigation counts may be correctly written and read as metadata
+ * in order to persist across reboots.
+ */
+ @Test
+ @SuppressWarnings("GuardedBy")
+ public void testWritingAndReadingMetadataBootMitigationCountRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ String filePath = InstrumentationRegistry.getContext().getFilesDir().toString()
+ + "metadata_file.txt";
+
+ ObserverInternal observer1 = new ObserverInternal("test1", List.of(), 1);
+ ObserverInternal observer2 = new ObserverInternal("test2", List.of(), 2);
+ watchdog.registerObserverInternal(observer1);
+ watchdog.registerObserverInternal(observer2);
+
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+
+ watchdog.saveAllObserversBootMitigationCountToMetadata(filePath);
+
+ observer1.setBootMitigationCount(0);
+ observer2.setBootMitigationCount(0);
+ assertThat(observer1.getBootMitigationCount()).isEqualTo(0);
+ assertThat(observer2.getBootMitigationCount()).isEqualTo(0);
+
+ mSpyBootThreshold.readAllObserversBootMitigationCountIfNecessary(filePath);
+
+ assertThat(observer1.getBootMitigationCount()).isEqualTo(1);
+ assertThat(observer2.getBootMitigationCount()).isEqualTo(2);
+ }
+
+ /**
* Tests device config changes are propagated correctly.
*/
@Test
@@ -1440,11 +1795,19 @@
// Mock CrashRecoveryProperties as they cannot be accessed due to SEPolicy restrictions
private void mockCrashRecoveryProperties(PackageWatchdog watchdog) {
+ mCrashRecoveryPropertiesMap = new HashMap<>();
+
try {
- mSpyBootThreshold = spy(watchdog.new BootThreshold(
- PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
- PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
- mCrashRecoveryPropertiesMap = new HashMap<>();
+ if (Flags.recoverabilityDetection()) {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+ } else {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
+ }
doAnswer((Answer<Integer>) invocationOnMock -> {
String storedValue = mCrashRecoveryPropertiesMap