Add recoverability detection to RescueParty and PackageWatchdog
PackageWatchdog will apply high user impact actions only in case of
network failure.
RescueParty updated the list of rescue levels and their corresponding
impact.
Test: unit and manual tests
Bug: 310236690
Change-Id: Iff34f6051b801a0d45149c317a7c9f344a2bd984
diff --git a/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java b/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java
index 37b5d40..a8d8f9a 100644
--- a/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java
+++ b/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java
@@ -26,6 +26,7 @@
import android.content.pm.PackageInfo;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
+import android.crashrecovery.flags.Flags;
import android.net.ConnectivityModuleConnector;
import android.os.Environment;
import android.os.Handler;
@@ -57,16 +58,20 @@
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -130,8 +135,25 @@
@VisibleForTesting
static final int DEFAULT_BOOT_LOOP_TRIGGER_COUNT = 5;
- @VisibleForTesting
+
static final long DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS = TimeUnit.MINUTES.toMillis(10);
+ // Boot loop at which packageWatchdog starts first mitigation
+ private static final String BOOT_LOOP_THRESHOLD =
+ "persist.device_config.configuration.boot_loop_threshold";
+ @VisibleForTesting
+ static final int DEFAULT_BOOT_LOOP_THRESHOLD = 15;
+ // Once boot_loop_threshold is surpassed next mitigation would be triggered after
+ // specified number of reboots.
+ private static final String BOOT_LOOP_MITIGATION_INCREMENT =
+ "persist.device_config.configuration..boot_loop_mitigation_increment";
+ @VisibleForTesting
+ static final int DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT = 2;
+
+ // Threshold level at which or above user might experience significant disruption.
+ private static final String MAJOR_USER_IMPACT_LEVEL_THRESHOLD =
+ "persist.device_config.configuration.major_user_impact_level_threshold";
+ private static final int DEFAULT_MAJOR_USER_IMPACT_LEVEL_THRESHOLD =
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_71;
private long mNumberOfNativeCrashPollsRemaining;
@@ -145,6 +167,7 @@
private static final String ATTR_EXPLICIT_HEALTH_CHECK_DURATION = "health-check-duration";
private static final String ATTR_PASSED_HEALTH_CHECK = "passed-health-check";
private static final String ATTR_MITIGATION_CALLS = "mitigation-calls";
+ private static final String ATTR_MITIGATION_COUNT = "mitigation-count";
// A file containing information about the current mitigation count in the case of a boot loop.
// This allows boot loop information to persist in the case of an fs-checkpoint being
@@ -230,8 +253,16 @@
mConnectivityModuleConnector = connectivityModuleConnector;
mSystemClock = clock;
mNumberOfNativeCrashPollsRemaining = NUMBER_OF_NATIVE_CRASH_POLLS;
- mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
- DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS);
+ if (Flags.recoverabilityDetection()) {
+ mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ SystemProperties.getInt(BOOT_LOOP_MITIGATION_INCREMENT,
+ DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+ } else {
+ mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS);
+ }
+
loadFromFile();
sPackageWatchdog = this;
}
@@ -436,8 +467,13 @@
mitigationCount =
currentMonitoredPackage.getMitigationCountLocked();
}
- currentObserverToNotify.execute(versionedPackage,
- failureReason, mitigationCount);
+ if (Flags.recoverabilityDetection()) {
+ maybeExecute(currentObserverToNotify, versionedPackage,
+ failureReason, currentObserverImpact, mitigationCount);
+ } else {
+ currentObserverToNotify.execute(versionedPackage,
+ failureReason, mitigationCount);
+ }
}
}
}
@@ -467,37 +503,76 @@
}
}
if (currentObserverToNotify != null) {
- currentObserverToNotify.execute(failingPackage, failureReason, 1);
+ if (Flags.recoverabilityDetection()) {
+ maybeExecute(currentObserverToNotify, failingPackage, failureReason,
+ currentObserverImpact, /*mitigationCount=*/ 1);
+ } else {
+ currentObserverToNotify.execute(failingPackage, failureReason, 1);
+ }
}
}
+ private void maybeExecute(PackageHealthObserver currentObserverToNotify,
+ VersionedPackage versionedPackage,
+ @FailureReasons int failureReason,
+ int currentObserverImpact,
+ int mitigationCount) {
+ if (currentObserverImpact < getUserImpactLevelLimit()) {
+ currentObserverToNotify.execute(versionedPackage, failureReason, mitigationCount);
+ }
+ }
+
+
/**
* Called when the system server boots. If the system server is detected to be in a boot loop,
* query each observer and perform the mitigation action with the lowest user impact.
*/
+ @SuppressWarnings("GuardedBy")
public void noteBoot() {
synchronized (mLock) {
- if (mBootThreshold.incrementAndTest()) {
- mBootThreshold.reset();
+ boolean mitigate = mBootThreshold.incrementAndTest();
+ if (mitigate) {
+ if (!Flags.recoverabilityDetection()) {
+ mBootThreshold.reset();
+ }
int mitigationCount = mBootThreshold.getMitigationCount() + 1;
PackageHealthObserver currentObserverToNotify = null;
+ ObserverInternal currentObserverInternal = null;
int currentObserverImpact = Integer.MAX_VALUE;
for (int i = 0; i < mAllObservers.size(); i++) {
final ObserverInternal observer = mAllObservers.valueAt(i);
PackageHealthObserver registeredObserver = observer.registeredObserver;
if (registeredObserver != null) {
- int impact = registeredObserver.onBootLoop(mitigationCount);
+ int impact = Flags.recoverabilityDetection()
+ ? registeredObserver.onBootLoop(
+ observer.getBootMitigationCount() + 1)
+ : registeredObserver.onBootLoop(mitigationCount);
if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0
&& impact < currentObserverImpact) {
currentObserverToNotify = registeredObserver;
+ currentObserverInternal = observer;
currentObserverImpact = impact;
}
}
}
if (currentObserverToNotify != null) {
- mBootThreshold.setMitigationCount(mitigationCount);
- mBootThreshold.saveMitigationCountToMetadata();
- currentObserverToNotify.executeBootLoopMitigation(mitigationCount);
+ if (Flags.recoverabilityDetection()) {
+ if (currentObserverImpact < getUserImpactLevelLimit()
+ || (currentObserverImpact >= getUserImpactLevelLimit()
+ && mBootThreshold.getCount() >= getBootLoopThreshold())) {
+ int currentObserverMitigationCount =
+ currentObserverInternal.getBootMitigationCount() + 1;
+ currentObserverInternal.setBootMitigationCount(
+ currentObserverMitigationCount);
+ saveAllObserversBootMitigationCountToMetadata(METADATA_FILE);
+ currentObserverToNotify.executeBootLoopMitigation(
+ currentObserverMitigationCount);
+ }
+ } else {
+ mBootThreshold.setMitigationCount(mitigationCount);
+ mBootThreshold.saveMitigationCountToMetadata();
+ currentObserverToNotify.executeBootLoopMitigation(mitigationCount);
+ }
}
}
}
@@ -567,13 +642,27 @@
mShortTaskHandler.post(()->checkAndMitigateNativeCrashes());
}
+ private int getUserImpactLevelLimit() {
+ return SystemProperties.getInt(MAJOR_USER_IMPACT_LEVEL_THRESHOLD,
+ DEFAULT_MAJOR_USER_IMPACT_LEVEL_THRESHOLD);
+ }
+
+ private int getBootLoopThreshold() {
+ return SystemProperties.getInt(BOOT_LOOP_THRESHOLD,
+ DEFAULT_BOOT_LOOP_THRESHOLD);
+ }
+
/** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */
@Retention(SOURCE)
@IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_LEVEL_0,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_10,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_30,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_50,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_70,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_71,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_75,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_80,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_90,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_100})
public @interface PackageHealthObserverImpact {
@@ -582,11 +671,15 @@
/* Action has low user impact, user of a device will barely notice. */
int USER_IMPACT_LEVEL_10 = 10;
/* Actions having medium user impact, user of a device will likely notice. */
+ int USER_IMPACT_LEVEL_20 = 20;
int USER_IMPACT_LEVEL_30 = 30;
int USER_IMPACT_LEVEL_50 = 50;
int USER_IMPACT_LEVEL_70 = 70;
- int USER_IMPACT_LEVEL_90 = 90;
/* Action has high user impact, a last resort, user of a device will be very frustrated. */
+ int USER_IMPACT_LEVEL_71 = 71;
+ int USER_IMPACT_LEVEL_75 = 75;
+ int USER_IMPACT_LEVEL_80 = 80;
+ int USER_IMPACT_LEVEL_90 = 90;
int USER_IMPACT_LEVEL_100 = 100;
}
@@ -1144,6 +1237,12 @@
}
}
+ @VisibleForTesting
+ @GuardedBy("mLock")
+ void registerObserverInternal(ObserverInternal observerInternal) {
+ mAllObservers.put(observerInternal.name, observerInternal);
+ }
+
/**
* Represents an observer monitoring a set of packages along with the failure thresholds for
* each package.
@@ -1151,17 +1250,23 @@
* <p> Note, the PackageWatchdog#mLock must always be held when reading or writing
* instances of this class.
*/
- private static class ObserverInternal {
+ static class ObserverInternal {
public final String name;
@GuardedBy("mLock")
private final ArrayMap<String, MonitoredPackage> mPackages = new ArrayMap<>();
@Nullable
@GuardedBy("mLock")
public PackageHealthObserver registeredObserver;
+ private int mMitigationCount;
ObserverInternal(String name, List<MonitoredPackage> packages) {
+ this(name, packages, /*mitigationCount=*/ 0);
+ }
+
+ ObserverInternal(String name, List<MonitoredPackage> packages, int mitigationCount) {
this.name = name;
updatePackagesLocked(packages);
+ this.mMitigationCount = mitigationCount;
}
/**
@@ -1173,6 +1278,9 @@
try {
out.startTag(null, TAG_OBSERVER);
out.attribute(null, ATTR_NAME, name);
+ if (Flags.recoverabilityDetection()) {
+ out.attributeInt(null, ATTR_MITIGATION_COUNT, mMitigationCount);
+ }
for (int i = 0; i < mPackages.size(); i++) {
MonitoredPackage p = mPackages.valueAt(i);
p.writeLocked(out);
@@ -1185,6 +1293,14 @@
}
}
+ public int getBootMitigationCount() {
+ return mMitigationCount;
+ }
+
+ public void setBootMitigationCount(int mitigationCount) {
+ mMitigationCount = mitigationCount;
+ }
+
@GuardedBy("mLock")
public void updatePackagesLocked(List<MonitoredPackage> packages) {
for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
@@ -1289,6 +1405,7 @@
**/
public static ObserverInternal read(TypedXmlPullParser parser, PackageWatchdog watchdog) {
String observerName = null;
+ int observerMitigationCount = 0;
if (TAG_OBSERVER.equals(parser.getName())) {
observerName = parser.getAttributeValue(null, ATTR_NAME);
if (TextUtils.isEmpty(observerName)) {
@@ -1299,6 +1416,9 @@
List<MonitoredPackage> packages = new ArrayList<>();
int innerDepth = parser.getDepth();
try {
+ if (Flags.recoverabilityDetection()) {
+ observerMitigationCount = parser.getAttributeInt(null, ATTR_MITIGATION_COUNT);
+ }
while (XmlUtils.nextElementWithin(parser, innerDepth)) {
if (TAG_PACKAGE.equals(parser.getName())) {
try {
@@ -1319,7 +1439,7 @@
if (packages.isEmpty()) {
return null;
}
- return new ObserverInternal(observerName, packages);
+ return new ObserverInternal(observerName, packages, observerMitigationCount);
}
/** Dumps information about this observer and the packages it watches. */
@@ -1679,6 +1799,27 @@
}
}
+ @GuardedBy("mLock")
+ @SuppressWarnings("GuardedBy")
+ void saveAllObserversBootMitigationCountToMetadata(String filePath) {
+ HashMap<String, Integer> bootMitigationCounts = new HashMap<>();
+ for (int i = 0; i < mAllObservers.size(); i++) {
+ final ObserverInternal observer = mAllObservers.valueAt(i);
+ bootMitigationCounts.put(observer.name, observer.getBootMitigationCount());
+ }
+
+ try {
+ FileOutputStream fileStream = new FileOutputStream(new File(filePath));
+ ObjectOutputStream objectStream = new ObjectOutputStream(fileStream);
+ objectStream.writeObject(bootMitigationCounts);
+ objectStream.flush();
+ objectStream.close();
+ fileStream.close();
+ } catch (Exception e) {
+ Slog.i(TAG, "Could not save observers metadata to file: " + e);
+ }
+ }
+
/**
* Handles the thresholding logic for system server boots.
*/
@@ -1686,10 +1827,16 @@
private final int mBootTriggerCount;
private final long mTriggerWindow;
+ private final int mBootMitigationIncrement;
BootThreshold(int bootTriggerCount, long triggerWindow) {
+ this(bootTriggerCount, triggerWindow, /*bootMitigationIncrement=*/ 1);
+ }
+
+ BootThreshold(int bootTriggerCount, long triggerWindow, int bootMitigationIncrement) {
this.mBootTriggerCount = bootTriggerCount;
this.mTriggerWindow = triggerWindow;
+ this.mBootMitigationIncrement = bootMitigationIncrement;
}
public void reset() {
@@ -1761,8 +1908,13 @@
/** Increments the boot counter, and returns whether the device is bootlooping. */
+ @GuardedBy("mLock")
public boolean incrementAndTest() {
- readMitigationCountFromMetadataIfNecessary();
+ if (Flags.recoverabilityDetection()) {
+ readAllObserversBootMitigationCountIfNecessary(METADATA_FILE);
+ } else {
+ readMitigationCountFromMetadataIfNecessary();
+ }
final long now = mSystemClock.uptimeMillis();
if (now - getStart() < 0) {
Slog.e(TAG, "Window was less than zero. Resetting start to current time.");
@@ -1770,8 +1922,12 @@
setMitigationStart(now);
}
if (now - getMitigationStart() > DEFAULT_DEESCALATION_WINDOW_MS) {
- setMitigationCount(0);
setMitigationStart(now);
+ if (Flags.recoverabilityDetection()) {
+ resetAllObserversBootMitigationCount();
+ } else {
+ setMitigationCount(0);
+ }
}
final long window = now - getStart();
if (window >= mTriggerWindow) {
@@ -1782,9 +1938,48 @@
int count = getCount() + 1;
setCount(count);
EventLogTags.writeRescueNote(Process.ROOT_UID, count, window);
+ if (Flags.recoverabilityDetection()) {
+ boolean mitigate = (count >= mBootTriggerCount)
+ && (count - mBootTriggerCount) % mBootMitigationIncrement == 0;
+ return mitigate;
+ }
return count >= mBootTriggerCount;
}
}
+ @GuardedBy("mLock")
+ private void resetAllObserversBootMitigationCount() {
+ for (int i = 0; i < mAllObservers.size(); i++) {
+ final ObserverInternal observer = mAllObservers.valueAt(i);
+ observer.setBootMitigationCount(0);
+ }
+ }
+
+ @GuardedBy("mLock")
+ @SuppressWarnings("GuardedBy")
+ void readAllObserversBootMitigationCountIfNecessary(String filePath) {
+ File metadataFile = new File(filePath);
+ if (metadataFile.exists()) {
+ try {
+ FileInputStream fileStream = new FileInputStream(metadataFile);
+ ObjectInputStream objectStream = new ObjectInputStream(fileStream);
+ HashMap<String, Integer> bootMitigationCounts =
+ (HashMap<String, Integer>) objectStream.readObject();
+ objectStream.close();
+ fileStream.close();
+
+ for (int i = 0; i < mAllObservers.size(); i++) {
+ final ObserverInternal observer = mAllObservers.valueAt(i);
+ if (bootMitigationCounts.containsKey(observer.name)) {
+ observer.setBootMitigationCount(
+ bootMitigationCounts.get(observer.name));
+ }
+ }
+ } catch (Exception e) {
+ Slog.i(TAG, "Could not read observer metadata file: " + e);
+ }
+ }
+ }
+
}
}
diff --git a/packages/CrashRecovery/services/java/com/android/server/RescueParty.java b/packages/CrashRecovery/services/java/com/android/server/RescueParty.java
index 7bdc1a0..7093ba4 100644
--- a/packages/CrashRecovery/services/java/com/android/server/RescueParty.java
+++ b/packages/CrashRecovery/services/java/com/android/server/RescueParty.java
@@ -20,6 +20,7 @@
import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo;
+import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.ContentResolver;
@@ -27,6 +28,7 @@
import android.content.pm.ApplicationInfo;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
+import android.crashrecovery.flags.Flags;
import android.os.Build;
import android.os.Environment;
import android.os.PowerManager;
@@ -53,6 +55,8 @@
import com.android.server.crashrecovery.proto.CrashRecoveryStatsLog;
import java.io.File;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -89,6 +93,40 @@
@VisibleForTesting
static final int LEVEL_FACTORY_RESET = 5;
@VisibleForTesting
+ static final int RESCUE_LEVEL_NONE = 0;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET = 1;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET = 2;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_WARM_REBOOT = 3;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS = 4;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES = 5;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS = 6;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_FACTORY_RESET = 7;
+
+ @IntDef(prefix = { "RESCUE_LEVEL_" }, value = {
+ RESCUE_LEVEL_NONE,
+ RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET,
+ RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET,
+ RESCUE_LEVEL_WARM_REBOOT,
+ RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS,
+ RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES,
+ RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS,
+ RESCUE_LEVEL_FACTORY_RESET
+ })
+ @Retention(RetentionPolicy.SOURCE)
+ @interface RescueLevels {}
+
+ @VisibleForTesting
+ static final String RESCUE_NON_REBOOT_LEVEL_LIMIT = "persist.sys.rescue_non_reboot_level_limit";
+ @VisibleForTesting
+ static final int DEFAULT_RESCUE_NON_REBOOT_LEVEL_LIMIT = RESCUE_LEVEL_WARM_REBOOT - 1;
+ @VisibleForTesting
static final String TAG = "RescueParty";
@VisibleForTesting
static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2);
@@ -347,11 +385,20 @@
}
private static int getMaxRescueLevel(boolean mayPerformReboot) {
- if (!mayPerformReboot
- || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {
- return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;
+ if (Flags.recoverabilityDetection()) {
+ if (!mayPerformReboot
+ || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {
+ return SystemProperties.getInt(RESCUE_NON_REBOOT_LEVEL_LIMIT,
+ DEFAULT_RESCUE_NON_REBOOT_LEVEL_LIMIT);
+ }
+ return RESCUE_LEVEL_FACTORY_RESET;
+ } else {
+ if (!mayPerformReboot
+ || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {
+ return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;
+ }
+ return LEVEL_FACTORY_RESET;
}
- return LEVEL_FACTORY_RESET;
}
/**
@@ -379,6 +426,46 @@
}
}
+ /**
+ * Get the rescue level to perform if this is the n-th attempt at mitigating failure.
+ * When failedPackage is null then 1st and 2nd mitigation counts are redundant (scoped and
+ * all device config reset). Behaves as if one mitigation attempt was already done.
+ *
+ * @param mitigationCount the mitigation attempt number (1 = first attempt etc.).
+ * @param mayPerformReboot whether or not a reboot and factory reset may be performed
+ * for the given failure.
+ * @param failedPackage in case of bootloop this is null.
+ * @return the rescue level for the n-th mitigation attempt.
+ */
+ private static @RescueLevels int getRescueLevel(int mitigationCount, boolean mayPerformReboot,
+ @Nullable VersionedPackage failedPackage) {
+ // Skipping RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET since it's not defined without a failed
+ // package.
+ if (failedPackage == null && mitigationCount > 0) {
+ mitigationCount += 1;
+ }
+ if (mitigationCount == 1) {
+ return RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET;
+ } else if (mitigationCount == 2) {
+ return RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET;
+ } else if (mitigationCount == 3) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot), RESCUE_LEVEL_WARM_REBOOT);
+ } else if (mitigationCount == 4) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot),
+ RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS);
+ } else if (mitigationCount == 5) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot),
+ RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES);
+ } else if (mitigationCount == 6) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot),
+ RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS);
+ } else if (mitigationCount >= 7) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot), RESCUE_LEVEL_FACTORY_RESET);
+ } else {
+ return RESCUE_LEVEL_NONE;
+ }
+ }
+
private static void executeRescueLevel(Context context, @Nullable String failedPackage,
int level) {
Slog.w(TAG, "Attempting rescue level " + levelToString(level));
@@ -397,6 +484,15 @@
private static void executeRescueLevelInternal(Context context, int level, @Nullable
String failedPackage) throws Exception {
+ if (Flags.recoverabilityDetection()) {
+ executeRescueLevelInternalNew(context, level, failedPackage);
+ } else {
+ executeRescueLevelInternalOld(context, level, failedPackage);
+ }
+ }
+
+ private static void executeRescueLevelInternalOld(Context context, int level, @Nullable
+ String failedPackage) throws Exception {
if (level <= LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS) {
// Disabling flag resets on master branch for trunk stable launch.
@@ -410,8 +506,6 @@
// Try our best to reset all settings possible, and once finished
// rethrow any exception that we encountered
Exception res = null;
- Runnable runnable;
- Thread thread;
switch (level) {
case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
try {
@@ -453,21 +547,7 @@
}
break;
case LEVEL_WARM_REBOOT:
- // Request the reboot from a separate thread to avoid deadlock on PackageWatchdog
- // when device shutting down.
- setRebootProperty(true);
- runnable = () -> {
- try {
- PowerManager pm = context.getSystemService(PowerManager.class);
- if (pm != null) {
- pm.reboot(TAG);
- }
- } catch (Throwable t) {
- logRescueException(level, failedPackage, t);
- }
- };
- thread = new Thread(runnable);
- thread.start();
+ executeWarmReboot(context, level, failedPackage);
break;
case LEVEL_FACTORY_RESET:
// Before the completion of Reboot, if any crash happens then PackageWatchdog
@@ -475,23 +555,9 @@
// Adding a check to prevent factory reset to execute before above reboot completes.
// Note: this reboot property is not persistent resets after reboot is completed.
if (isRebootPropertySet()) {
- break;
+ return;
}
- setFactoryResetProperty(true);
- long now = System.currentTimeMillis();
- setLastFactoryResetTimeMs(now);
- runnable = new Runnable() {
- @Override
- public void run() {
- try {
- RecoverySystem.rebootPromptAndWipeUserData(context, TAG);
- } catch (Throwable t) {
- logRescueException(level, failedPackage, t);
- }
- }
- };
- thread = new Thread(runnable);
- thread.start();
+ executeFactoryReset(context, level, failedPackage);
break;
}
@@ -500,6 +566,83 @@
}
}
+ private static void executeRescueLevelInternalNew(Context context, @RescueLevels int level,
+ @Nullable String failedPackage) throws Exception {
+ CrashRecoveryStatsLog.write(CrashRecoveryStatsLog.RESCUE_PARTY_RESET_REPORTED,
+ level, levelToString(level));
+ switch (level) {
+ case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET:
+ // Temporary disable deviceConfig reset
+ // resetDeviceConfig(context, /*isScoped=*/true, failedPackage);
+ break;
+ case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET:
+ // Temporary disable deviceConfig reset
+ // resetDeviceConfig(context, /*isScoped=*/false, failedPackage);
+ break;
+ case RESCUE_LEVEL_WARM_REBOOT:
+ executeWarmReboot(context, level, failedPackage);
+ break;
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_DEFAULTS, level);
+ break;
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_CHANGES, level);
+ break;
+ case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ resetAllSettingsIfNecessary(context, Settings.RESET_MODE_TRUSTED_DEFAULTS, level);
+ break;
+ case RESCUE_LEVEL_FACTORY_RESET:
+ // Before the completion of Reboot, if any crash happens then PackageWatchdog
+ // escalates to next level i.e. factory reset, as they happen in separate threads.
+ // Adding a check to prevent factory reset to execute before above reboot completes.
+ // Note: this reboot property is not persistent resets after reboot is completed.
+ if (isRebootPropertySet()) {
+ return;
+ }
+ executeFactoryReset(context, level, failedPackage);
+ break;
+ }
+ }
+
+ private static void executeWarmReboot(Context context, int level,
+ @Nullable String failedPackage) {
+ // Request the reboot from a separate thread to avoid deadlock on PackageWatchdog
+ // when device shutting down.
+ setRebootProperty(true);
+ Runnable runnable = () -> {
+ try {
+ PowerManager pm = context.getSystemService(PowerManager.class);
+ if (pm != null) {
+ pm.reboot(TAG);
+ }
+ } catch (Throwable t) {
+ logRescueException(level, failedPackage, t);
+ }
+ };
+ Thread thread = new Thread(runnable);
+ thread.start();
+ }
+
+ private static void executeFactoryReset(Context context, int level,
+ @Nullable String failedPackage) {
+ setFactoryResetProperty(true);
+ long now = System.currentTimeMillis();
+ setLastFactoryResetTimeMs(now);
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ RecoverySystem.rebootPromptAndWipeUserData(context, TAG);
+ } catch (Throwable t) {
+ logRescueException(level, failedPackage, t);
+ }
+ }
+ };
+ Thread thread = new Thread(runnable);
+ thread.start();
+ }
+
+
private static String getCompleteMessage(Throwable t) {
final StringBuilder builder = new StringBuilder();
builder.append(t.getMessage());
@@ -521,17 +664,38 @@
}
private static int mapRescueLevelToUserImpact(int rescueLevel) {
- switch(rescueLevel) {
- case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
- case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
- return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10;
- case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
- case LEVEL_WARM_REBOOT:
- return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
- case LEVEL_FACTORY_RESET:
- return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100;
- default:
- return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ if (Flags.recoverabilityDetection()) {
+ switch (rescueLevel) {
+ case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10;
+ case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_20;
+ case RESCUE_LEVEL_WARM_REBOOT:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_71;
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_75;
+ case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_80;
+ case RESCUE_LEVEL_FACTORY_RESET:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100;
+ default:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ }
+ } else {
+ switch (rescueLevel) {
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10;
+ case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ case LEVEL_WARM_REBOOT:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
+ case LEVEL_FACTORY_RESET:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100;
+ default:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ }
}
}
@@ -548,7 +712,7 @@
final ContentResolver resolver = context.getContentResolver();
try {
Settings.Global.resetToDefaultsAsUser(resolver, null, mode,
- UserHandle.SYSTEM.getIdentifier());
+ UserHandle.SYSTEM.getIdentifier());
} catch (Exception e) {
res = new RuntimeException("Failed to reset global settings", e);
}
@@ -667,8 +831,13 @@
@FailureReasons int failureReason, int mitigationCount) {
if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
|| failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) {
- return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
+ if (Flags.recoverabilityDetection()) {
+ return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
+ mayPerformReboot(failedPackage), failedPackage));
+ } else {
+ return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
mayPerformReboot(failedPackage)));
+ }
} else {
return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
}
@@ -682,8 +851,10 @@
}
if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
|| failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {
- final int level = getRescueLevel(mitigationCount,
- mayPerformReboot(failedPackage));
+ final int level = Flags.recoverabilityDetection() ? getRescueLevel(mitigationCount,
+ mayPerformReboot(failedPackage), failedPackage)
+ : getRescueLevel(mitigationCount,
+ mayPerformReboot(failedPackage));
executeRescueLevel(mContext,
failedPackage == null ? null : failedPackage.getPackageName(), level);
return true;
@@ -716,7 +887,12 @@
if (isDisabled()) {
return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
}
- return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, true));
+ if (Flags.recoverabilityDetection()) {
+ return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
+ true, /*failedPackage=*/ null));
+ } else {
+ return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, true));
+ }
}
@Override
@@ -725,8 +901,10 @@
return false;
}
boolean mayPerformReboot = !shouldThrottleReboot();
- executeRescueLevel(mContext, /*failedPackage=*/ null,
- getRescueLevel(mitigationCount, mayPerformReboot));
+ final int level = Flags.recoverabilityDetection() ? getRescueLevel(mitigationCount,
+ mayPerformReboot, /*failedPackage=*/ null)
+ : getRescueLevel(mitigationCount, mayPerformReboot);
+ executeRescueLevel(mContext, /*failedPackage=*/ null, level);
return true;
}
@@ -843,14 +1021,44 @@
}
private static String levelToString(int level) {
- switch (level) {
- case LEVEL_NONE: return "NONE";
- case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS: return "RESET_SETTINGS_UNTRUSTED_DEFAULTS";
- case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES: return "RESET_SETTINGS_UNTRUSTED_CHANGES";
- case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS: return "RESET_SETTINGS_TRUSTED_DEFAULTS";
- case LEVEL_WARM_REBOOT: return "WARM_REBOOT";
- case LEVEL_FACTORY_RESET: return "FACTORY_RESET";
- default: return Integer.toString(level);
+ if (Flags.recoverabilityDetection()) {
+ switch (level) {
+ case RESCUE_LEVEL_NONE:
+ return "NONE";
+ case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET:
+ return "SCOPED_DEVICE_CONFIG_RESET";
+ case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET:
+ return "ALL_DEVICE_CONFIG_RESET";
+ case RESCUE_LEVEL_WARM_REBOOT:
+ return "WARM_REBOOT";
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ return "RESET_SETTINGS_UNTRUSTED_DEFAULTS";
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ return "RESET_SETTINGS_UNTRUSTED_CHANGES";
+ case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ return "RESET_SETTINGS_TRUSTED_DEFAULTS";
+ case RESCUE_LEVEL_FACTORY_RESET:
+ return "FACTORY_RESET";
+ default:
+ return Integer.toString(level);
+ }
+ } else {
+ switch (level) {
+ case LEVEL_NONE:
+ return "NONE";
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ return "RESET_SETTINGS_UNTRUSTED_DEFAULTS";
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ return "RESET_SETTINGS_UNTRUSTED_CHANGES";
+ case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ return "RESET_SETTINGS_TRUSTED_DEFAULTS";
+ case LEVEL_WARM_REBOOT:
+ return "WARM_REBOOT";
+ case LEVEL_FACTORY_RESET:
+ return "FACTORY_RESET";
+ default:
+ return Integer.toString(level);
+ }
}
}
}
diff --git a/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java
index 0fb9327..93f26ae 100644
--- a/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java
+++ b/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java
@@ -69,7 +69,7 @@
*
* @hide
*/
-final class RollbackPackageHealthObserver implements PackageHealthObserver {
+public final class RollbackPackageHealthObserver implements PackageHealthObserver {
private static final String TAG = "RollbackPackageHealthObserver";
private static final String NAME = "rollback-observer";
private static final int PERSISTENT_MASK = ApplicationInfo.FLAG_PERSISTENT
@@ -89,7 +89,7 @@
private boolean mTwoPhaseRollbackEnabled;
@VisibleForTesting
- RollbackPackageHealthObserver(Context context, ApexManager apexManager) {
+ public RollbackPackageHealthObserver(Context context, ApexManager apexManager) {
mContext = context;
HandlerThread handlerThread = new HandlerThread("RollbackPackageHealthObserver");
handlerThread.start();
diff --git a/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java b/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java
index c30ac2d..682569f 100644
--- a/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java
+++ b/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java
@@ -26,6 +26,7 @@
import static com.android.dx.mockito.inline.extended.ExtendedMockito.verify;
import static com.android.dx.mockito.inline.extended.ExtendedMockito.when;
import static com.android.server.RescueParty.LEVEL_FACTORY_RESET;
+import static com.android.server.RescueParty.RESCUE_LEVEL_FACTORY_RESET;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -41,9 +42,11 @@
import android.content.pm.ApplicationInfo;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
+import android.crashrecovery.flags.Flags;
import android.os.RecoverySystem;
import android.os.SystemProperties;
import android.os.UserHandle;
+import android.platform.test.flag.junit.SetFlagsRule;
import android.provider.DeviceConfig;
import android.provider.Settings;
import android.util.ArraySet;
@@ -55,6 +58,7 @@
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.mockito.Answers;
import org.mockito.ArgumentCaptor;
@@ -69,6 +73,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
@@ -100,6 +105,9 @@
private static final int THROTTLING_DURATION_MIN = 10;
+ @Rule
+ public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();
+
private MockitoSession mSession;
private HashMap<String, String> mSystemSettingsMap;
private HashMap<String, String> mCrashRecoveryPropertiesMap;
@@ -267,6 +275,42 @@
}
@Test
+ public void testBootLoopDetectionWithExecutionForAllRescueLevelsRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ RescueParty.onSettingsProviderPublished(mMockContext);
+ verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
+ any(Executor.class),
+ mMonitorCallbackCaptor.capture()));
+ HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
+
+ // Record DeviceConfig accesses
+ DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
+
+ final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
+
+ noteBoot(1);
+ verifyDeviceConfigReset(expectedAllResetNamespaces, verifiedTimesMap);
+
+ noteBoot(2);
+ assertTrue(RescueParty.isRebootPropertySet());
+
+ noteBoot(3);
+ verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
+
+ noteBoot(4);
+ verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_CHANGES);
+
+ noteBoot(5);
+ verifyOnlySettingsReset(Settings.RESET_MODE_TRUSTED_DEFAULTS);
+
+ setCrashRecoveryPropAttemptingReboot(false);
+ noteBoot(6);
+ assertTrue(RescueParty.isFactoryResetPropertySet());
+ }
+
+ @Test
public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevels() {
noteAppCrash(1, true);
@@ -292,6 +336,47 @@
}
@Test
+ public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevelsRecoverability() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ RescueParty.onSettingsProviderPublished(mMockContext);
+ verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
+ any(Executor.class),
+ mMonitorCallbackCaptor.capture()));
+ HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
+
+ // Record DeviceConfig accesses
+ DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
+ monitorCallback.onDeviceConfigAccess(PERSISTENT_PACKAGE, NAMESPACE1);
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
+
+ final String[] expectedResetNamespaces = new String[]{NAMESPACE1};
+ final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
+
+ noteAppCrash(1, true);
+ verifyDeviceConfigReset(expectedResetNamespaces, verifiedTimesMap);
+
+ noteAppCrash(2, true);
+ verifyDeviceConfigReset(expectedAllResetNamespaces, verifiedTimesMap);
+
+ noteAppCrash(3, true);
+ assertTrue(RescueParty.isRebootPropertySet());
+
+ noteAppCrash(4, true);
+ verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
+
+ noteAppCrash(5, true);
+ verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_CHANGES);
+
+ noteAppCrash(6, true);
+ verifyOnlySettingsReset(Settings.RESET_MODE_TRUSTED_DEFAULTS);
+
+ setCrashRecoveryPropAttemptingReboot(false);
+ noteAppCrash(7, true);
+ assertTrue(RescueParty.isFactoryResetPropertySet());
+ }
+
+ @Test
public void testNonPersistentAppOnlyPerformsFlagResets() {
noteAppCrash(1, false);
@@ -316,6 +401,45 @@
}
@Test
+ public void testNonPersistentAppOnlyPerformsFlagResetsRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ RescueParty.onSettingsProviderPublished(mMockContext);
+ verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
+ any(Executor.class),
+ mMonitorCallbackCaptor.capture()));
+ HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
+
+ // Record DeviceConfig accesses
+ DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
+ monitorCallback.onDeviceConfigAccess(NON_PERSISTENT_PACKAGE, NAMESPACE1);
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
+
+ final String[] expectedResetNamespaces = new String[]{NAMESPACE1};
+ final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
+
+ noteAppCrash(1, false);
+ verifyDeviceConfigReset(expectedResetNamespaces, verifiedTimesMap);
+
+ noteAppCrash(2, false);
+ verifyDeviceConfigReset(expectedAllResetNamespaces, verifiedTimesMap);
+
+ noteAppCrash(3, false);
+ assertFalse(RescueParty.isRebootPropertySet());
+
+ noteAppCrash(4, false);
+ verifyNoSettingsReset(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
+ noteAppCrash(5, false);
+ verifyNoSettingsReset(Settings.RESET_MODE_UNTRUSTED_CHANGES);
+ noteAppCrash(6, false);
+ verifyNoSettingsReset(Settings.RESET_MODE_TRUSTED_DEFAULTS);
+
+ setCrashRecoveryPropAttemptingReboot(false);
+ noteAppCrash(7, false);
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+ }
+
+ @Test
public void testNonPersistentAppCrashDetectionWithScopedResets() {
RescueParty.onSettingsProviderPublished(mMockContext);
verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
@@ -451,6 +575,19 @@
}
@Test
+ public void testIsRecoveryTriggeredRebootRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ for (int i = 0; i < RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteBoot(i + 1);
+ }
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+ setCrashRecoveryPropAttemptingReboot(false);
+ noteBoot(RESCUE_LEVEL_FACTORY_RESET + 1);
+ assertTrue(RescueParty.isRecoveryTriggeredReboot());
+ assertTrue(RescueParty.isFactoryResetPropertySet());
+ }
+
+ @Test
public void testIsRecoveryTriggeredRebootOnlyAfterRebootCompleted() {
for (int i = 0; i < LEVEL_FACTORY_RESET; i++) {
noteBoot(i + 1);
@@ -469,6 +606,25 @@
}
@Test
+ public void testIsRecoveryTriggeredRebootOnlyAfterRebootCompletedRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ for (int i = 0; i < RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteBoot(i + 1);
+ }
+ int mitigationCount = RESCUE_LEVEL_FACTORY_RESET + 1;
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+ noteBoot(mitigationCount++);
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+ noteBoot(mitigationCount++);
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+ noteBoot(mitigationCount++);
+ setCrashRecoveryPropAttemptingReboot(false);
+ noteBoot(mitigationCount + 1);
+ assertTrue(RescueParty.isRecoveryTriggeredReboot());
+ assertTrue(RescueParty.isFactoryResetPropertySet());
+ }
+
+ @Test
public void testThrottlingOnBootFailures() {
setCrashRecoveryPropAttemptingReboot(false);
long now = System.currentTimeMillis();
@@ -481,6 +637,19 @@
}
@Test
+ public void testThrottlingOnBootFailuresRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ setCrashRecoveryPropAttemptingReboot(false);
+ long now = System.currentTimeMillis();
+ long beforeTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN - 1);
+ setCrashRecoveryPropLastFactoryReset(beforeTimeout);
+ for (int i = 1; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteBoot(i);
+ }
+ assertFalse(RescueParty.isRecoveryTriggeredReboot());
+ }
+
+ @Test
public void testThrottlingOnAppCrash() {
setCrashRecoveryPropAttemptingReboot(false);
long now = System.currentTimeMillis();
@@ -493,6 +662,19 @@
}
@Test
+ public void testThrottlingOnAppCrashRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ setCrashRecoveryPropAttemptingReboot(false);
+ long now = System.currentTimeMillis();
+ long beforeTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN - 1);
+ setCrashRecoveryPropLastFactoryReset(beforeTimeout);
+ for (int i = 0; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteAppCrash(i + 1, true);
+ }
+ assertFalse(RescueParty.isRecoveryTriggeredReboot());
+ }
+
+ @Test
public void testNotThrottlingAfterTimeoutOnBootFailures() {
setCrashRecoveryPropAttemptingReboot(false);
long now = System.currentTimeMillis();
@@ -503,6 +685,20 @@
}
assertTrue(RescueParty.isRecoveryTriggeredReboot());
}
+
+ @Test
+ public void testNotThrottlingAfterTimeoutOnBootFailuresRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ setCrashRecoveryPropAttemptingReboot(false);
+ long now = System.currentTimeMillis();
+ long afterTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN + 1);
+ setCrashRecoveryPropLastFactoryReset(afterTimeout);
+ for (int i = 1; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteBoot(i);
+ }
+ assertTrue(RescueParty.isRecoveryTriggeredReboot());
+ }
+
@Test
public void testNotThrottlingAfterTimeoutOnAppCrash() {
setCrashRecoveryPropAttemptingReboot(false);
@@ -516,6 +712,19 @@
}
@Test
+ public void testNotThrottlingAfterTimeoutOnAppCrashRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ setCrashRecoveryPropAttemptingReboot(false);
+ long now = System.currentTimeMillis();
+ long afterTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN + 1);
+ setCrashRecoveryPropLastFactoryReset(afterTimeout);
+ for (int i = 0; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteAppCrash(i + 1, true);
+ }
+ assertTrue(RescueParty.isRecoveryTriggeredReboot());
+ }
+
+ @Test
public void testNativeRescuePartyResets() {
doReturn(true).when(() -> SettingsToPropertiesMapper.isNativeFlagsResetPerformed());
doReturn(FAKE_RESET_NATIVE_NAMESPACES).when(
@@ -531,6 +740,7 @@
@Test
public void testExplicitlyEnablingAndDisablingRescue() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(false));
SystemProperties.set(PROP_DISABLE_RESCUE, Boolean.toString(true));
assertEquals(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,
@@ -543,6 +753,7 @@
@Test
public void testDisablingRescueByDeviceConfigFlag() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(false));
SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(true));
@@ -568,6 +779,20 @@
}
@Test
+ public void testDisablingFactoryResetByDeviceConfigFlagRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ SystemProperties.set(PROP_DISABLE_FACTORY_RESET_FLAG, Boolean.toString(true));
+
+ for (int i = 0; i < RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteBoot(i + 1);
+ }
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+
+ // Restore the property value initialized in SetUp()
+ SystemProperties.set(PROP_DISABLE_FACTORY_RESET_FLAG, "");
+ }
+
+ @Test
public void testHealthCheckLevels() {
RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
@@ -594,6 +819,46 @@
}
@Test
+ public void testHealthCheckLevelsRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
+
+ // Ensure that no action is taken for cases where the failure reason is unknown
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_UNKNOWN, 1),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
+
+ // Ensure the correct user impact is returned for each mitigation count.
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 5),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 6),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 7),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+ }
+
+ @Test
public void testBootLoopLevels() {
RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
@@ -606,6 +871,19 @@
}
@Test
+ public void testBootLoopLevelsRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
+
+ assertEquals(observer.onBootLoop(1), PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+ assertEquals(observer.onBootLoop(2), PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+ assertEquals(observer.onBootLoop(3), PackageHealthObserverImpact.USER_IMPACT_LEVEL_71);
+ assertEquals(observer.onBootLoop(4), PackageHealthObserverImpact.USER_IMPACT_LEVEL_75);
+ assertEquals(observer.onBootLoop(5), PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);
+ assertEquals(observer.onBootLoop(6), PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);
+ }
+
+ @Test
public void testResetDeviceConfigForPackagesOnlyRuntimeMap() {
RescueParty.onSettingsProviderPublished(mMockContext);
verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
@@ -727,11 +1005,26 @@
private void verifySettingsResets(int resetMode, String[] resetNamespaces,
HashMap<String, Integer> configResetVerifiedTimesMap) {
+ verifyOnlySettingsReset(resetMode);
+ verifyDeviceConfigReset(resetNamespaces, configResetVerifiedTimesMap);
+ }
+
+ private void verifyOnlySettingsReset(int resetMode) {
verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null,
resetMode, UserHandle.USER_SYSTEM));
verify(() -> Settings.Secure.resetToDefaultsAsUser(eq(mMockContentResolver), isNull(),
eq(resetMode), anyInt()));
- // Verify DeviceConfig resets
+ }
+
+ private void verifyNoSettingsReset(int resetMode) {
+ verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null,
+ resetMode, UserHandle.USER_SYSTEM), never());
+ verify(() -> Settings.Secure.resetToDefaultsAsUser(eq(mMockContentResolver), isNull(),
+ eq(resetMode), anyInt()), never());
+ }
+
+ private void verifyDeviceConfigReset(String[] resetNamespaces,
+ Map<String, Integer> configResetVerifiedTimesMap) {
if (resetNamespaces == null) {
verify(() -> DeviceConfig.resetToDefaults(anyInt(), anyString()), never());
} else {
@@ -818,9 +1111,16 @@
// mock properties in BootThreshold
try {
- mSpyBootThreshold = spy(watchdog.new BootThreshold(
- PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
- PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
+ if (Flags.recoverabilityDetection()) {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+ } else {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
+ }
mCrashRecoveryPropertiesMap = new HashMap<>();
doAnswer((Answer<Integer>) invocationOnMock -> {
diff --git a/tests/PackageWatchdog/Android.bp b/tests/PackageWatchdog/Android.bp
index e0e6c4c..2c5fdd3 100644
--- a/tests/PackageWatchdog/Android.bp
+++ b/tests/PackageWatchdog/Android.bp
@@ -28,8 +28,10 @@
static_libs: [
"junit",
"mockito-target-extended-minus-junit4",
+ "flag-junit",
"frameworks-base-testutils",
"androidx.test.rules",
+ "PlatformProperties",
"services.core",
"services.net",
"truth",
diff --git a/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java b/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java
new file mode 100644
index 0000000..081da11
--- /dev/null
+++ b/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java
@@ -0,0 +1,644 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server;
+
+import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;
+
+import static com.android.dx.mockito.inline.extended.ExtendedMockito.doAnswer;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import static org.junit.Assert.assertFalse;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import android.Manifest;
+import android.content.Context;
+import android.content.pm.PackageInfo;
+import android.content.pm.PackageManager;
+import android.content.pm.VersionedPackage;
+import android.content.rollback.PackageRollbackInfo;
+import android.content.rollback.RollbackInfo;
+import android.content.rollback.RollbackManager;
+import android.crashrecovery.flags.Flags;
+import android.net.ConnectivityModuleConnector;
+import android.net.ConnectivityModuleConnector.ConnectivityModuleHealthListener;
+import android.os.Handler;
+import android.os.SystemProperties;
+import android.os.test.TestLooper;
+import android.platform.test.flag.junit.SetFlagsRule;
+import android.provider.DeviceConfig;
+import android.util.AtomicFile;
+
+import androidx.test.InstrumentationRegistry;
+
+import com.android.dx.mockito.inline.extended.ExtendedMockito;
+import com.android.server.RescueParty.RescuePartyObserver;
+import com.android.server.pm.ApexManager;
+import com.android.server.rollback.RollbackPackageHealthObserver;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Answers;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.mockito.MockitoSession;
+import org.mockito.quality.Strictness;
+import org.mockito.stubbing.Answer;
+
+import java.io.File;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+/**
+ * Test CrashRecovery, integration tests that include PackageWatchdog, RescueParty and
+ * RollbackPackageHealthObserver
+ */
+public class CrashRecoveryTest {
+ private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG =
+ "persist.device_config.configuration.disable_rescue_party";
+
+ private static final String APP_A = "com.package.a";
+ private static final String APP_B = "com.package.b";
+ private static final String APP_C = "com.package.c";
+ private static final long VERSION_CODE = 1L;
+ private static final long SHORT_DURATION = TimeUnit.SECONDS.toMillis(1);
+
+ private static final RollbackInfo ROLLBACK_INFO_LOW = getRollbackInfo(APP_A, VERSION_CODE, 1,
+ PackageManager.ROLLBACK_USER_IMPACT_LOW);
+ private static final RollbackInfo ROLLBACK_INFO_HIGH = getRollbackInfo(APP_B, VERSION_CODE, 2,
+ PackageManager.ROLLBACK_USER_IMPACT_HIGH);
+ private static final RollbackInfo ROLLBACK_INFO_MANUAL = getRollbackInfo(APP_C, VERSION_CODE, 3,
+ PackageManager.ROLLBACK_USER_IMPACT_ONLY_MANUAL);
+
+ @Rule
+ public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();
+
+ private final TestClock mTestClock = new TestClock();
+ private TestLooper mTestLooper;
+ private Context mSpyContext;
+ // Keep track of all created watchdogs to apply device config changes
+ private List<PackageWatchdog> mAllocatedWatchdogs;
+ @Mock
+ private ConnectivityModuleConnector mConnectivityModuleConnector;
+ @Mock
+ private PackageManager mMockPackageManager;
+ @Mock(answer = Answers.RETURNS_DEEP_STUBS)
+ private ApexManager mApexManager;
+ @Mock
+ RollbackManager mRollbackManager;
+ // Mock only sysprop apis
+ private PackageWatchdog.BootThreshold mSpyBootThreshold;
+ @Captor
+ private ArgumentCaptor<ConnectivityModuleHealthListener> mConnectivityModuleCallbackCaptor;
+ private MockitoSession mSession;
+ private HashMap<String, String> mSystemSettingsMap;
+ private HashMap<String, String> mCrashRecoveryPropertiesMap;
+
+ @Before
+ public void setUp() throws Exception {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ MockitoAnnotations.initMocks(this);
+ new File(InstrumentationRegistry.getContext().getFilesDir(),
+ "package-watchdog.xml").delete();
+ adoptShellPermissions(Manifest.permission.READ_DEVICE_CONFIG,
+ Manifest.permission.WRITE_DEVICE_CONFIG);
+ mTestLooper = new TestLooper();
+ mSpyContext = spy(InstrumentationRegistry.getContext());
+ when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager);
+ when(mMockPackageManager.getPackageInfo(anyString(), anyInt())).then(inv -> {
+ final PackageInfo res = new PackageInfo();
+ res.packageName = inv.getArgument(0);
+ res.setLongVersionCode(VERSION_CODE);
+ return res;
+ });
+ mSession = ExtendedMockito.mockitoSession()
+ .initMocks(this)
+ .strictness(Strictness.LENIENT)
+ .spyStatic(SystemProperties.class)
+ .spyStatic(RescueParty.class)
+ .startMocking();
+ mSystemSettingsMap = new HashMap<>();
+
+ // Mock SystemProperties setter and various getters
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ String value = invocationOnMock.getArgument(1);
+
+ mSystemSettingsMap.put(key, value);
+ return null;
+ }
+ ).when(() -> SystemProperties.set(anyString(), anyString()));
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ int defaultValue = invocationOnMock.getArgument(1);
+
+ String storedValue = mSystemSettingsMap.get(key);
+ return storedValue == null ? defaultValue : Integer.parseInt(storedValue);
+ }
+ ).when(() -> SystemProperties.getInt(anyString(), anyInt()));
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ long defaultValue = invocationOnMock.getArgument(1);
+
+ String storedValue = mSystemSettingsMap.get(key);
+ return storedValue == null ? defaultValue : Long.parseLong(storedValue);
+ }
+ ).when(() -> SystemProperties.getLong(anyString(), anyLong()));
+
+ doAnswer((Answer<Boolean>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ boolean defaultValue = invocationOnMock.getArgument(1);
+
+ String storedValue = mSystemSettingsMap.get(key);
+ return storedValue == null ? defaultValue : Boolean.parseBoolean(storedValue);
+ }
+ ).when(() -> SystemProperties.getBoolean(anyString(), anyBoolean()));
+
+ SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));
+ SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(false));
+
+ DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
+ PackageWatchdog.PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED,
+ Boolean.toString(true), false);
+
+ DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
+ PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
+ Integer.toString(PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT), false);
+
+ mAllocatedWatchdogs = new ArrayList<>();
+ RescuePartyObserver.reset();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ dropShellPermissions();
+ mSession.finishMocking();
+ // Clean up listeners since too many listeners will delay notifications significantly
+ for (PackageWatchdog watchdog : mAllocatedWatchdogs) {
+ watchdog.removePropertyChangedListener();
+ }
+ mAllocatedWatchdogs.clear();
+ }
+
+ @Test
+ public void testBootLoopWithRescueParty() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ RescuePartyObserver rescuePartyObserver = setUpRescuePartyObserver(watchdog);
+
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);
+ int bootCounter = 0;
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(1);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(2);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
+
+ int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;
+ for (int i = 0; i < bootLoopThreshold; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(3);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(4);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(5);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(6);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);
+ }
+
+ @Test
+ public void testBootLoopWithRollbackPackageHealthObserver() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ RollbackPackageHealthObserver rollbackObserver =
+ setUpRollbackPackageHealthObserver(watchdog);
+
+ verify(rollbackObserver, never()).executeBootLoopMitigation(1);
+ int bootCounter = 0;
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rollbackObserver).executeBootLoopMitigation(1);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ // Update the list of available rollbacks after executing bootloop mitigation once
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,
+ ROLLBACK_INFO_MANUAL));
+
+ int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;
+ for (int i = 0; i < bootLoopThreshold; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rollbackObserver).executeBootLoopMitigation(2);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+
+ // Update the list of available rollbacks after executing bootloop mitigation once
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+ }
+
+ @Test
+ public void testBootLoopWithRescuePartyAndRollbackPackageHealthObserver() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ RescuePartyObserver rescuePartyObserver = setUpRescuePartyObserver(watchdog);
+ RollbackPackageHealthObserver rollbackObserver =
+ setUpRollbackPackageHealthObserver(watchdog);
+
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(1);
+ int bootCounter = 0;
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(1);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(1);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(2);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
+ verify(rollbackObserver).executeBootLoopMitigation(1);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+ // Update the list of available rollbacks after executing bootloop mitigation once
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,
+ ROLLBACK_INFO_MANUAL));
+
+ int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;
+ for (int i = 0; i < bootLoopThreshold; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(3);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(4);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(5);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
+ verify(rollbackObserver).executeBootLoopMitigation(2);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+ // Update the list of available rollbacks after executing bootloop mitigation
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(6);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+ }
+
+ RollbackPackageHealthObserver setUpRollbackPackageHealthObserver(PackageWatchdog watchdog) {
+ RollbackPackageHealthObserver rollbackObserver =
+ spy(new RollbackPackageHealthObserver(mSpyContext, mApexManager));
+ when(mSpyContext.getSystemService(RollbackManager.class)).thenReturn(mRollbackManager);
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_LOW,
+ ROLLBACK_INFO_HIGH, ROLLBACK_INFO_MANUAL));
+ when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager);
+
+ watchdog.registerHealthObserver(rollbackObserver);
+ return rollbackObserver;
+ }
+
+ RescuePartyObserver setUpRescuePartyObserver(PackageWatchdog watchdog) {
+ setCrashRecoveryPropRescueBootCount(0);
+ RescuePartyObserver rescuePartyObserver = spy(RescuePartyObserver.getInstance(mSpyContext));
+ assertFalse(RescueParty.isRebootPropertySet());
+ watchdog.registerHealthObserver(rescuePartyObserver);
+ return rescuePartyObserver;
+ }
+
+ private static RollbackInfo getRollbackInfo(String packageName, long versionCode,
+ int rollbackId, int rollbackUserImpact) {
+ VersionedPackage appFrom = new VersionedPackage(packageName, versionCode + 1);
+ VersionedPackage appTo = new VersionedPackage(packageName, versionCode);
+ PackageRollbackInfo packageRollbackInfo = new PackageRollbackInfo(appFrom, appTo, null,
+ null, false, false, null);
+ RollbackInfo rollbackInfo = new RollbackInfo(rollbackId, List.of(packageRollbackInfo),
+ false, null, 111, rollbackUserImpact);
+ return rollbackInfo;
+ }
+
+ private void adoptShellPermissions(String... permissions) {
+ androidx.test.platform.app.InstrumentationRegistry
+ .getInstrumentation()
+ .getUiAutomation()
+ .adoptShellPermissionIdentity(permissions);
+ }
+
+ private void dropShellPermissions() {
+ androidx.test.platform.app.InstrumentationRegistry
+ .getInstrumentation()
+ .getUiAutomation()
+ .dropShellPermissionIdentity();
+ }
+
+
+ private PackageWatchdog createWatchdog() {
+ return createWatchdog(new TestController(), true /* withPackagesReady */);
+ }
+
+ private PackageWatchdog createWatchdog(TestController controller, boolean withPackagesReady) {
+ AtomicFile policyFile =
+ new AtomicFile(new File(mSpyContext.getFilesDir(), "package-watchdog.xml"));
+ Handler handler = new Handler(mTestLooper.getLooper());
+ PackageWatchdog watchdog =
+ new PackageWatchdog(mSpyContext, policyFile, handler, handler, controller,
+ mConnectivityModuleConnector, mTestClock);
+ mockCrashRecoveryProperties(watchdog);
+
+ // Verify controller is not automatically started
+ assertThat(controller.mIsEnabled).isFalse();
+ if (withPackagesReady) {
+ // Only capture the NetworkStack callback for the latest registered watchdog
+ reset(mConnectivityModuleConnector);
+ watchdog.onPackagesReady();
+ // Verify controller by default is started when packages are ready
+ assertThat(controller.mIsEnabled).isTrue();
+
+ verify(mConnectivityModuleConnector).registerHealthListener(
+ mConnectivityModuleCallbackCaptor.capture());
+ }
+ mAllocatedWatchdogs.add(watchdog);
+ return watchdog;
+ }
+
+ // Mock CrashRecoveryProperties as they cannot be accessed due to SEPolicy restrictions
+ private void mockCrashRecoveryProperties(PackageWatchdog watchdog) {
+ mCrashRecoveryPropertiesMap = new HashMap<>();
+
+ // mock properties in RescueParty
+ try {
+
+ doAnswer((Answer<Boolean>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.attempting_factory_reset", "false");
+ return Boolean.parseBoolean(storedValue);
+ }).when(() -> RescueParty.isFactoryResetPropertySet());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ boolean value = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.attempting_factory_reset",
+ Boolean.toString(value));
+ return null;
+ }).when(() -> RescueParty.setFactoryResetProperty(anyBoolean()));
+
+ doAnswer((Answer<Boolean>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.attempting_reboot", "false");
+ return Boolean.parseBoolean(storedValue);
+ }).when(() -> RescueParty.isRebootPropertySet());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ boolean value = invocationOnMock.getArgument(0);
+ setCrashRecoveryPropAttemptingReboot(value);
+ return null;
+ }).when(() -> RescueParty.setRebootProperty(anyBoolean()));
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("persist.crashrecovery.last_factory_reset", "0");
+ return Long.parseLong(storedValue);
+ }).when(() -> RescueParty.getLastFactoryResetTimeMs());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ long value = invocationOnMock.getArgument(0);
+ setCrashRecoveryPropLastFactoryReset(value);
+ return null;
+ }).when(() -> RescueParty.setLastFactoryResetTimeMs(anyLong()));
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.max_rescue_level_attempted", "0");
+ return Integer.parseInt(storedValue);
+ }).when(() -> RescueParty.getMaxRescueLevelAttempted());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ int value = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.max_rescue_level_attempted",
+ Integer.toString(value));
+ return null;
+ }).when(() -> RescueParty.setMaxRescueLevelAttempted(anyInt()));
+
+ } catch (Exception e) {
+ // tests will fail, just printing the error
+ System.out.println("Error while mocking crashrecovery properties " + e.getMessage());
+ }
+
+ try {
+ if (Flags.recoverabilityDetection()) {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+ } else {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
+ }
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.rescue_boot_count", "0");
+ return Integer.parseInt(storedValue);
+ }).when(mSpyBootThreshold).getCount();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ int count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_count",
+ Integer.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setCount(anyInt());
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.boot_mitigation_count", "0");
+ return Integer.parseInt(storedValue);
+ }).when(mSpyBootThreshold).getMitigationCount();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ int count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.boot_mitigation_count",
+ Integer.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setMitigationCount(anyInt());
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.rescue_boot_start", "0");
+ return Long.parseLong(storedValue);
+ }).when(mSpyBootThreshold).getStart();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ long count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_start",
+ Long.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setStart(anyLong());
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.boot_mitigation_start", "0");
+ return Long.parseLong(storedValue);
+ }).when(mSpyBootThreshold).getMitigationStart();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ long count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.boot_mitigation_start",
+ Long.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setMitigationStart(anyLong());
+
+ Field mBootThresholdField = watchdog.getClass().getDeclaredField("mBootThreshold");
+ mBootThresholdField.setAccessible(true);
+ mBootThresholdField.set(watchdog, mSpyBootThreshold);
+ } catch (Exception e) {
+ // tests will fail, just printing the error
+ System.out.println("Error detected while spying BootThreshold" + e.getMessage());
+ }
+ }
+
+ private void setCrashRecoveryPropRescueBootCount(int count) {
+ mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_count",
+ Integer.toString(count));
+ }
+
+ private void setCrashRecoveryPropAttemptingReboot(boolean value) {
+ mCrashRecoveryPropertiesMap.put("crashrecovery.attempting_reboot",
+ Boolean.toString(value));
+ }
+
+ private void setCrashRecoveryPropLastFactoryReset(long value) {
+ mCrashRecoveryPropertiesMap.put("persist.crashrecovery.last_factory_reset",
+ Long.toString(value));
+ }
+
+ private static class TestController extends ExplicitHealthCheckController {
+ TestController() {
+ super(null /* controller */);
+ }
+
+ private boolean mIsEnabled;
+ private List<String> mSupportedPackages = new ArrayList<>();
+ private List<String> mRequestedPackages = new ArrayList<>();
+ private Consumer<List<PackageConfig>> mSupportedConsumer;
+ private List<Set> mSyncRequests = new ArrayList<>();
+
+ @Override
+ public void setEnabled(boolean enabled) {
+ mIsEnabled = enabled;
+ if (!mIsEnabled) {
+ mSupportedPackages.clear();
+ }
+ }
+
+ @Override
+ public void setCallbacks(Consumer<String> passedConsumer,
+ Consumer<List<PackageConfig>> supportedConsumer, Runnable notifySyncRunnable) {
+ mSupportedConsumer = supportedConsumer;
+ }
+
+ @Override
+ public void syncRequests(Set<String> packages) {
+ mSyncRequests.add(packages);
+ mRequestedPackages.clear();
+ if (mIsEnabled) {
+ packages.retainAll(mSupportedPackages);
+ mRequestedPackages.addAll(packages);
+ List<PackageConfig> packageConfigs = new ArrayList<>();
+ for (String packageName: packages) {
+ packageConfigs.add(new PackageConfig(packageName, SHORT_DURATION));
+ }
+ mSupportedConsumer.accept(packageConfigs);
+ } else {
+ mSupportedConsumer.accept(Collections.emptyList());
+ }
+ }
+ }
+
+ private static class TestClock implements PackageWatchdog.SystemClock {
+ // Note 0 is special to the internal clock of PackageWatchdog. We need to start from
+ // a non-zero value in order not to disrupt the logic of PackageWatchdog.
+ private long mUpTimeMillis = 1;
+ @Override
+ public long uptimeMillis() {
+ return mUpTimeMillis;
+ }
+ }
+}
diff --git a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
index 75284c7..4f27e06 100644
--- a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
+++ b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
@@ -36,11 +36,13 @@
import android.content.pm.PackageInfo;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
+import android.crashrecovery.flags.Flags;
import android.net.ConnectivityModuleConnector;
import android.net.ConnectivityModuleConnector.ConnectivityModuleHealthListener;
import android.os.Handler;
import android.os.SystemProperties;
import android.os.test.TestLooper;
+import android.platform.test.flag.junit.SetFlagsRule;
import android.provider.DeviceConfig;
import android.util.AtomicFile;
import android.util.Xml;
@@ -54,11 +56,13 @@
import com.android.modules.utils.TypedXmlSerializer;
import com.android.server.PackageWatchdog.HealthCheckState;
import com.android.server.PackageWatchdog.MonitoredPackage;
+import com.android.server.PackageWatchdog.ObserverInternal;
import com.android.server.PackageWatchdog.PackageHealthObserver;
import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
@@ -99,6 +103,10 @@
private static final String OBSERVER_NAME_4 = "observer4";
private static final long SHORT_DURATION = TimeUnit.SECONDS.toMillis(1);
private static final long LONG_DURATION = TimeUnit.SECONDS.toMillis(5);
+
+ @Rule
+ public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();
+
private final TestClock mTestClock = new TestClock();
private TestLooper mTestLooper;
private Context mSpyContext;
@@ -128,6 +136,7 @@
@Before
public void setUp() throws Exception {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
MockitoAnnotations.initMocks(this);
new File(InstrumentationRegistry.getContext().getFilesDir(),
"package-watchdog.xml").delete();
@@ -444,6 +453,7 @@
*/
@Test
public void testPackageFailureNotifyAllDifferentImpacts() throws Exception {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver observerNone = new TestObserver(OBSERVER_NAME_1,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
@@ -488,6 +498,52 @@
assertThat(observerLowPackages).containsExactly(APP_A);
}
+ @Test
+ public void testPackageFailureNotifyAllDifferentImpactsRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver observerNone = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
+ TestObserver observerHigh = new TestObserver(OBSERVER_NAME_2,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+ TestObserver observerMid = new TestObserver(OBSERVER_NAME_3,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ TestObserver observerLow = new TestObserver(OBSERVER_NAME_4,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+
+ // Start observing for all impact observers
+ watchdog.startObservingHealth(observerNone, Arrays.asList(APP_A, APP_B, APP_C, APP_D),
+ SHORT_DURATION);
+ watchdog.startObservingHealth(observerHigh, Arrays.asList(APP_A, APP_B, APP_C),
+ SHORT_DURATION);
+ watchdog.startObservingHealth(observerMid, Arrays.asList(APP_A, APP_B),
+ SHORT_DURATION);
+ watchdog.startObservingHealth(observerLow, Arrays.asList(APP_A),
+ SHORT_DURATION);
+
+ // Then fail all apps above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE),
+ new VersionedPackage(APP_B, VERSION_CODE),
+ new VersionedPackage(APP_C, VERSION_CODE),
+ new VersionedPackage(APP_D, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify least impact observers are notifed of package failures
+ List<String> observerNonePackages = observerNone.mMitigatedPackages;
+ List<String> observerHighPackages = observerHigh.mMitigatedPackages;
+ List<String> observerMidPackages = observerMid.mMitigatedPackages;
+ List<String> observerLowPackages = observerLow.mMitigatedPackages;
+
+ // APP_D failure observed by only observerNone is not caught cos its impact is none
+ assertThat(observerNonePackages).isEmpty();
+ // APP_C failure is caught by observerHigh cos it's the lowest impact observer
+ assertThat(observerHighPackages).containsExactly(APP_C);
+ // APP_B failure is caught by observerMid cos it's the lowest impact observer
+ assertThat(observerMidPackages).containsExactly(APP_B);
+ // APP_A failure is caught by observerLow cos it's the lowest impact observer
+ assertThat(observerLowPackages).containsExactly(APP_A);
+ }
+
/**
* Test package failure and least impact observers are notified successively.
* State transistions:
@@ -501,6 +557,7 @@
*/
@Test
public void testPackageFailureNotifyLeastImpactSuccessively() throws Exception {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver observerFirst = new TestObserver(OBSERVER_NAME_1,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
@@ -563,11 +620,76 @@
assertThat(observerSecond.mMitigatedPackages).isEmpty();
}
+ @Test
+ public void testPackageFailureNotifyLeastImpactSuccessivelyRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver observerFirst = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+ TestObserver observerSecond = new TestObserver(OBSERVER_NAME_2,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+
+ // Start observing for observerFirst and observerSecond with failure handling
+ watchdog.startObservingHealth(observerFirst, Arrays.asList(APP_A), LONG_DURATION);
+ watchdog.startObservingHealth(observerSecond, Arrays.asList(APP_A), LONG_DURATION);
+
+ // Then fail APP_A above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only observerFirst is notifed
+ assertThat(observerFirst.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observerSecond.mMitigatedPackages).isEmpty();
+
+ // After observerFirst handles failure, next action it has is high impact
+ observerFirst.mImpact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
+ observerFirst.mMitigatedPackages.clear();
+ observerSecond.mMitigatedPackages.clear();
+
+ // Then fail APP_A again above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only observerSecond is notifed cos it has least impact
+ assertThat(observerSecond.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observerFirst.mMitigatedPackages).isEmpty();
+
+ // After observerSecond handles failure, it has no further actions
+ observerSecond.mImpact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ observerFirst.mMitigatedPackages.clear();
+ observerSecond.mMitigatedPackages.clear();
+
+ // Then fail APP_A again above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only observerFirst is notifed cos it has the only action
+ assertThat(observerFirst.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observerSecond.mMitigatedPackages).isEmpty();
+
+ // After observerFirst handles failure, it too has no further actions
+ observerFirst.mImpact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ observerFirst.mMitigatedPackages.clear();
+ observerSecond.mMitigatedPackages.clear();
+
+ // Then fail APP_A again above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify no observer is notified cos no actions left
+ assertThat(observerFirst.mMitigatedPackages).isEmpty();
+ assertThat(observerSecond.mMitigatedPackages).isEmpty();
+ }
+
/**
* Test package failure and notifies only one observer even with observer impact tie.
*/
@Test
public void testPackageFailureNotifyOneSameImpact() throws Exception {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);
@@ -588,6 +710,28 @@
assertThat(observer2.mMitigatedPackages).isEmpty();
}
+ @Test
+ public void testPackageFailureNotifyOneSameImpactRecoverabilityDetection() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver observer1 = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+ TestObserver observer2 = new TestObserver(OBSERVER_NAME_2,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+
+ // Start observing for observer1 and observer2 with failure handling
+ watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION);
+ watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
+
+ // Then fail APP_A above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only one observer is notifed
+ assertThat(observer1.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observer2.mMitigatedPackages).isEmpty();
+ }
+
/**
* Test package passing explicit health checks does not fail and vice versa.
*/
@@ -818,6 +962,7 @@
@Test
public void testNetworkStackFailure() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
final PackageWatchdog wd = createWatchdog();
// Start observing with failure handling
@@ -835,6 +980,25 @@
assertThat(observer.mMitigatedPackages).containsExactly(APP_A);
}
+ @Test
+ public void testNetworkStackFailureRecoverabilityDetection() {
+ final PackageWatchdog wd = createWatchdog();
+
+ // Start observing with failure handling
+ TestObserver observer = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);
+ wd.startObservingHealth(observer, Collections.singletonList(APP_A), SHORT_DURATION);
+
+ // Notify of NetworkStack failure
+ mConnectivityModuleCallbackCaptor.getValue().onNetworkStackFailure(APP_A);
+
+ // Run handler so package failures are dispatched to observers
+ mTestLooper.dispatchAll();
+
+ // Verify the NetworkStack observer is notified
+ assertThat(observer.mMitigatedPackages).isEmpty();
+ }
+
/** Test default values are used when device property is invalid. */
@Test
public void testInvalidConfig_watchdogTriggerFailureCount() {
@@ -1045,6 +1209,7 @@
/** Ensure that boot loop mitigation is done when the number of boots meets the threshold. */
@Test
public void testBootLoopDetection_meetsThreshold() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
watchdog.registerHealthObserver(bootObserver);
@@ -1054,6 +1219,16 @@
assertThat(bootObserver.mitigatedBootLoop()).isTrue();
}
+ @Test
+ public void testBootLoopDetection_meetsThresholdRecoverability() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver.mitigatedBootLoop()).isTrue();
+ }
/**
* Ensure that boot loop mitigation is not done when the number of boots does not meet the
@@ -1071,10 +1246,43 @@
}
/**
+ * Ensure that boot loop mitigation is not done when the number of boots does not meet the
+ * threshold.
+ */
+ @Test
+ public void testBootLoopDetection_doesNotMeetThresholdRecoverabilityLowImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver.mitigatedBootLoop()).isFalse();
+ }
+
+ /**
+ * Ensure that boot loop mitigation is not done when the number of boots does not meet the
+ * threshold.
+ */
+ @Test
+ public void testBootLoopDetection_doesNotMeetThresholdRecoverabilityHighImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver.mitigatedBootLoop()).isFalse();
+ }
+
+ /**
* Ensure that boot loop mitigation is done for the observer with the lowest user impact
*/
@Test
public void testBootLoopMitigationDoneForLowestUserImpact() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver bootObserver1 = new TestObserver(OBSERVER_NAME_1);
bootObserver1.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
@@ -1089,11 +1297,28 @@
assertThat(bootObserver2.mitigatedBootLoop()).isFalse();
}
+ @Test
+ public void testBootLoopMitigationDoneForLowestUserImpactRecoverability() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver1 = new TestObserver(OBSERVER_NAME_1);
+ bootObserver1.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+ TestObserver bootObserver2 = new TestObserver(OBSERVER_NAME_2);
+ bootObserver2.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ watchdog.registerHealthObserver(bootObserver1);
+ watchdog.registerHealthObserver(bootObserver2);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver1.mitigatedBootLoop()).isTrue();
+ assertThat(bootObserver2.mitigatedBootLoop()).isFalse();
+ }
+
/**
* Ensure that the correct mitigation counts are sent to the boot loop observer.
*/
@Test
public void testMultipleBootLoopMitigation() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
watchdog.registerHealthObserver(bootObserver);
@@ -1114,6 +1339,64 @@
assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));
}
+ @Test
+ public void testMultipleBootLoopMitigationRecoverabilityLowImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);
+
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));
+ }
+
+ @Test
+ public void testMultipleBootLoopMitigationRecoverabilityHighImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);
+
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));
+ }
+
/**
* Ensure that passing a null list of failed packages does not cause any mitigation logic to
* execute.
@@ -1304,6 +1587,78 @@
}
/**
+ * Ensure that a {@link ObserverInternal} may be correctly written and read in order to persist
+ * across reboots.
+ */
+ @Test
+ @SuppressWarnings("GuardedBy")
+ public void testWritingAndReadingObserverInternalRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+
+ LongArrayQueue mitigationCalls = new LongArrayQueue();
+ mitigationCalls.addLast(1000);
+ mitigationCalls.addLast(2000);
+ mitigationCalls.addLast(3000);
+ MonitoredPackage writePkg = watchdog.newMonitoredPackage(
+ "test.package", 1000, 2000, true, mitigationCalls);
+ final int bootMitigationCount = 4;
+ ObserverInternal writeObserver = new ObserverInternal("test", List.of(writePkg),
+ bootMitigationCount);
+
+ // Write the observer
+ File tmpFile = File.createTempFile("observer-watchdog-test", ".xml");
+ AtomicFile testFile = new AtomicFile(tmpFile);
+ FileOutputStream stream = testFile.startWrite();
+ TypedXmlSerializer outputSerializer = Xml.resolveSerializer(stream);
+ outputSerializer.startDocument(null, true);
+ writeObserver.writeLocked(outputSerializer);
+ outputSerializer.endDocument();
+ testFile.finishWrite(stream);
+
+ // Read the observer
+ TypedXmlPullParser parser = Xml.resolvePullParser(testFile.openRead());
+ XmlUtils.beginDocument(parser, "observer");
+ ObserverInternal readObserver = ObserverInternal.read(parser, watchdog);
+
+ assertThat(readObserver.name).isEqualTo(writeObserver.name);
+ assertThat(readObserver.getBootMitigationCount()).isEqualTo(bootMitigationCount);
+ }
+
+ /**
+ * Ensure that boot mitigation counts may be correctly written and read as metadata
+ * in order to persist across reboots.
+ */
+ @Test
+ @SuppressWarnings("GuardedBy")
+ public void testWritingAndReadingMetadataBootMitigationCountRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ String filePath = InstrumentationRegistry.getContext().getFilesDir().toString()
+ + "metadata_file.txt";
+
+ ObserverInternal observer1 = new ObserverInternal("test1", List.of(), 1);
+ ObserverInternal observer2 = new ObserverInternal("test2", List.of(), 2);
+ watchdog.registerObserverInternal(observer1);
+ watchdog.registerObserverInternal(observer2);
+
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+
+ watchdog.saveAllObserversBootMitigationCountToMetadata(filePath);
+
+ observer1.setBootMitigationCount(0);
+ observer2.setBootMitigationCount(0);
+ assertThat(observer1.getBootMitigationCount()).isEqualTo(0);
+ assertThat(observer2.getBootMitigationCount()).isEqualTo(0);
+
+ mSpyBootThreshold.readAllObserversBootMitigationCountIfNecessary(filePath);
+
+ assertThat(observer1.getBootMitigationCount()).isEqualTo(1);
+ assertThat(observer2.getBootMitigationCount()).isEqualTo(2);
+ }
+
+ /**
* Tests device config changes are propagated correctly.
*/
@Test
@@ -1440,11 +1795,19 @@
// Mock CrashRecoveryProperties as they cannot be accessed due to SEPolicy restrictions
private void mockCrashRecoveryProperties(PackageWatchdog watchdog) {
+ mCrashRecoveryPropertiesMap = new HashMap<>();
+
try {
- mSpyBootThreshold = spy(watchdog.new BootThreshold(
- PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
- PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
- mCrashRecoveryPropertiesMap = new HashMap<>();
+ if (Flags.recoverabilityDetection()) {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+ } else {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
+ }
doAnswer((Answer<Integer>) invocationOnMock -> {
String storedValue = mCrashRecoveryPropertiesMap