Merge "Add recoverability detection to RescueParty and PackageWatchdog" into main am: dfe6ab6101
Original change: https://android-review.googlesource.com/c/platform/frameworks/base/+/2925157
Change-Id: I4fe5036c1a17ff4fc613f39265631534e6a58b0f
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
diff --git a/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java b/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java
index 37b5d40..a8d8f9a 100644
--- a/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java
+++ b/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java
@@ -26,6 +26,7 @@
import android.content.pm.PackageInfo;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
+import android.crashrecovery.flags.Flags;
import android.net.ConnectivityModuleConnector;
import android.os.Environment;
import android.os.Handler;
@@ -57,16 +58,20 @@
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -130,8 +135,25 @@
@VisibleForTesting
static final int DEFAULT_BOOT_LOOP_TRIGGER_COUNT = 5;
- @VisibleForTesting
+
static final long DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS = TimeUnit.MINUTES.toMillis(10);
+ // Boot loop at which packageWatchdog starts first mitigation
+ private static final String BOOT_LOOP_THRESHOLD =
+ "persist.device_config.configuration.boot_loop_threshold";
+ @VisibleForTesting
+ static final int DEFAULT_BOOT_LOOP_THRESHOLD = 15;
+ // Once boot_loop_threshold is surpassed next mitigation would be triggered after
+ // specified number of reboots.
+ private static final String BOOT_LOOP_MITIGATION_INCREMENT =
+ "persist.device_config.configuration..boot_loop_mitigation_increment";
+ @VisibleForTesting
+ static final int DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT = 2;
+
+ // Threshold level at which or above user might experience significant disruption.
+ private static final String MAJOR_USER_IMPACT_LEVEL_THRESHOLD =
+ "persist.device_config.configuration.major_user_impact_level_threshold";
+ private static final int DEFAULT_MAJOR_USER_IMPACT_LEVEL_THRESHOLD =
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_71;
private long mNumberOfNativeCrashPollsRemaining;
@@ -145,6 +167,7 @@
private static final String ATTR_EXPLICIT_HEALTH_CHECK_DURATION = "health-check-duration";
private static final String ATTR_PASSED_HEALTH_CHECK = "passed-health-check";
private static final String ATTR_MITIGATION_CALLS = "mitigation-calls";
+ private static final String ATTR_MITIGATION_COUNT = "mitigation-count";
// A file containing information about the current mitigation count in the case of a boot loop.
// This allows boot loop information to persist in the case of an fs-checkpoint being
@@ -230,8 +253,16 @@
mConnectivityModuleConnector = connectivityModuleConnector;
mSystemClock = clock;
mNumberOfNativeCrashPollsRemaining = NUMBER_OF_NATIVE_CRASH_POLLS;
- mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
- DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS);
+ if (Flags.recoverabilityDetection()) {
+ mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ SystemProperties.getInt(BOOT_LOOP_MITIGATION_INCREMENT,
+ DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+ } else {
+ mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS);
+ }
+
loadFromFile();
sPackageWatchdog = this;
}
@@ -436,8 +467,13 @@
mitigationCount =
currentMonitoredPackage.getMitigationCountLocked();
}
- currentObserverToNotify.execute(versionedPackage,
- failureReason, mitigationCount);
+ if (Flags.recoverabilityDetection()) {
+ maybeExecute(currentObserverToNotify, versionedPackage,
+ failureReason, currentObserverImpact, mitigationCount);
+ } else {
+ currentObserverToNotify.execute(versionedPackage,
+ failureReason, mitigationCount);
+ }
}
}
}
@@ -467,37 +503,76 @@
}
}
if (currentObserverToNotify != null) {
- currentObserverToNotify.execute(failingPackage, failureReason, 1);
+ if (Flags.recoverabilityDetection()) {
+ maybeExecute(currentObserverToNotify, failingPackage, failureReason,
+ currentObserverImpact, /*mitigationCount=*/ 1);
+ } else {
+ currentObserverToNotify.execute(failingPackage, failureReason, 1);
+ }
}
}
+ private void maybeExecute(PackageHealthObserver currentObserverToNotify,
+ VersionedPackage versionedPackage,
+ @FailureReasons int failureReason,
+ int currentObserverImpact,
+ int mitigationCount) {
+ if (currentObserverImpact < getUserImpactLevelLimit()) {
+ currentObserverToNotify.execute(versionedPackage, failureReason, mitigationCount);
+ }
+ }
+
+
/**
* Called when the system server boots. If the system server is detected to be in a boot loop,
* query each observer and perform the mitigation action with the lowest user impact.
*/
+ @SuppressWarnings("GuardedBy")
public void noteBoot() {
synchronized (mLock) {
- if (mBootThreshold.incrementAndTest()) {
- mBootThreshold.reset();
+ boolean mitigate = mBootThreshold.incrementAndTest();
+ if (mitigate) {
+ if (!Flags.recoverabilityDetection()) {
+ mBootThreshold.reset();
+ }
int mitigationCount = mBootThreshold.getMitigationCount() + 1;
PackageHealthObserver currentObserverToNotify = null;
+ ObserverInternal currentObserverInternal = null;
int currentObserverImpact = Integer.MAX_VALUE;
for (int i = 0; i < mAllObservers.size(); i++) {
final ObserverInternal observer = mAllObservers.valueAt(i);
PackageHealthObserver registeredObserver = observer.registeredObserver;
if (registeredObserver != null) {
- int impact = registeredObserver.onBootLoop(mitigationCount);
+ int impact = Flags.recoverabilityDetection()
+ ? registeredObserver.onBootLoop(
+ observer.getBootMitigationCount() + 1)
+ : registeredObserver.onBootLoop(mitigationCount);
if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0
&& impact < currentObserverImpact) {
currentObserverToNotify = registeredObserver;
+ currentObserverInternal = observer;
currentObserverImpact = impact;
}
}
}
if (currentObserverToNotify != null) {
- mBootThreshold.setMitigationCount(mitigationCount);
- mBootThreshold.saveMitigationCountToMetadata();
- currentObserverToNotify.executeBootLoopMitigation(mitigationCount);
+ if (Flags.recoverabilityDetection()) {
+ if (currentObserverImpact < getUserImpactLevelLimit()
+ || (currentObserverImpact >= getUserImpactLevelLimit()
+ && mBootThreshold.getCount() >= getBootLoopThreshold())) {
+ int currentObserverMitigationCount =
+ currentObserverInternal.getBootMitigationCount() + 1;
+ currentObserverInternal.setBootMitigationCount(
+ currentObserverMitigationCount);
+ saveAllObserversBootMitigationCountToMetadata(METADATA_FILE);
+ currentObserverToNotify.executeBootLoopMitigation(
+ currentObserverMitigationCount);
+ }
+ } else {
+ mBootThreshold.setMitigationCount(mitigationCount);
+ mBootThreshold.saveMitigationCountToMetadata();
+ currentObserverToNotify.executeBootLoopMitigation(mitigationCount);
+ }
}
}
}
@@ -567,13 +642,27 @@
mShortTaskHandler.post(()->checkAndMitigateNativeCrashes());
}
+ private int getUserImpactLevelLimit() {
+ return SystemProperties.getInt(MAJOR_USER_IMPACT_LEVEL_THRESHOLD,
+ DEFAULT_MAJOR_USER_IMPACT_LEVEL_THRESHOLD);
+ }
+
+ private int getBootLoopThreshold() {
+ return SystemProperties.getInt(BOOT_LOOP_THRESHOLD,
+ DEFAULT_BOOT_LOOP_THRESHOLD);
+ }
+
/** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */
@Retention(SOURCE)
@IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_LEVEL_0,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_10,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_30,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_50,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_70,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_71,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_75,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_80,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_90,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_100})
public @interface PackageHealthObserverImpact {
@@ -582,11 +671,15 @@
/* Action has low user impact, user of a device will barely notice. */
int USER_IMPACT_LEVEL_10 = 10;
/* Actions having medium user impact, user of a device will likely notice. */
+ int USER_IMPACT_LEVEL_20 = 20;
int USER_IMPACT_LEVEL_30 = 30;
int USER_IMPACT_LEVEL_50 = 50;
int USER_IMPACT_LEVEL_70 = 70;
- int USER_IMPACT_LEVEL_90 = 90;
/* Action has high user impact, a last resort, user of a device will be very frustrated. */
+ int USER_IMPACT_LEVEL_71 = 71;
+ int USER_IMPACT_LEVEL_75 = 75;
+ int USER_IMPACT_LEVEL_80 = 80;
+ int USER_IMPACT_LEVEL_90 = 90;
int USER_IMPACT_LEVEL_100 = 100;
}
@@ -1144,6 +1237,12 @@
}
}
+ @VisibleForTesting
+ @GuardedBy("mLock")
+ void registerObserverInternal(ObserverInternal observerInternal) {
+ mAllObservers.put(observerInternal.name, observerInternal);
+ }
+
/**
* Represents an observer monitoring a set of packages along with the failure thresholds for
* each package.
@@ -1151,17 +1250,23 @@
* <p> Note, the PackageWatchdog#mLock must always be held when reading or writing
* instances of this class.
*/
- private static class ObserverInternal {
+ static class ObserverInternal {
public final String name;
@GuardedBy("mLock")
private final ArrayMap<String, MonitoredPackage> mPackages = new ArrayMap<>();
@Nullable
@GuardedBy("mLock")
public PackageHealthObserver registeredObserver;
+ private int mMitigationCount;
ObserverInternal(String name, List<MonitoredPackage> packages) {
+ this(name, packages, /*mitigationCount=*/ 0);
+ }
+
+ ObserverInternal(String name, List<MonitoredPackage> packages, int mitigationCount) {
this.name = name;
updatePackagesLocked(packages);
+ this.mMitigationCount = mitigationCount;
}
/**
@@ -1173,6 +1278,9 @@
try {
out.startTag(null, TAG_OBSERVER);
out.attribute(null, ATTR_NAME, name);
+ if (Flags.recoverabilityDetection()) {
+ out.attributeInt(null, ATTR_MITIGATION_COUNT, mMitigationCount);
+ }
for (int i = 0; i < mPackages.size(); i++) {
MonitoredPackage p = mPackages.valueAt(i);
p.writeLocked(out);
@@ -1185,6 +1293,14 @@
}
}
+ public int getBootMitigationCount() {
+ return mMitigationCount;
+ }
+
+ public void setBootMitigationCount(int mitigationCount) {
+ mMitigationCount = mitigationCount;
+ }
+
@GuardedBy("mLock")
public void updatePackagesLocked(List<MonitoredPackage> packages) {
for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
@@ -1289,6 +1405,7 @@
**/
public static ObserverInternal read(TypedXmlPullParser parser, PackageWatchdog watchdog) {
String observerName = null;
+ int observerMitigationCount = 0;
if (TAG_OBSERVER.equals(parser.getName())) {
observerName = parser.getAttributeValue(null, ATTR_NAME);
if (TextUtils.isEmpty(observerName)) {
@@ -1299,6 +1416,9 @@
List<MonitoredPackage> packages = new ArrayList<>();
int innerDepth = parser.getDepth();
try {
+ if (Flags.recoverabilityDetection()) {
+ observerMitigationCount = parser.getAttributeInt(null, ATTR_MITIGATION_COUNT);
+ }
while (XmlUtils.nextElementWithin(parser, innerDepth)) {
if (TAG_PACKAGE.equals(parser.getName())) {
try {
@@ -1319,7 +1439,7 @@
if (packages.isEmpty()) {
return null;
}
- return new ObserverInternal(observerName, packages);
+ return new ObserverInternal(observerName, packages, observerMitigationCount);
}
/** Dumps information about this observer and the packages it watches. */
@@ -1679,6 +1799,27 @@
}
}
+ @GuardedBy("mLock")
+ @SuppressWarnings("GuardedBy")
+ void saveAllObserversBootMitigationCountToMetadata(String filePath) {
+ HashMap<String, Integer> bootMitigationCounts = new HashMap<>();
+ for (int i = 0; i < mAllObservers.size(); i++) {
+ final ObserverInternal observer = mAllObservers.valueAt(i);
+ bootMitigationCounts.put(observer.name, observer.getBootMitigationCount());
+ }
+
+ try {
+ FileOutputStream fileStream = new FileOutputStream(new File(filePath));
+ ObjectOutputStream objectStream = new ObjectOutputStream(fileStream);
+ objectStream.writeObject(bootMitigationCounts);
+ objectStream.flush();
+ objectStream.close();
+ fileStream.close();
+ } catch (Exception e) {
+ Slog.i(TAG, "Could not save observers metadata to file: " + e);
+ }
+ }
+
/**
* Handles the thresholding logic for system server boots.
*/
@@ -1686,10 +1827,16 @@
private final int mBootTriggerCount;
private final long mTriggerWindow;
+ private final int mBootMitigationIncrement;
BootThreshold(int bootTriggerCount, long triggerWindow) {
+ this(bootTriggerCount, triggerWindow, /*bootMitigationIncrement=*/ 1);
+ }
+
+ BootThreshold(int bootTriggerCount, long triggerWindow, int bootMitigationIncrement) {
this.mBootTriggerCount = bootTriggerCount;
this.mTriggerWindow = triggerWindow;
+ this.mBootMitigationIncrement = bootMitigationIncrement;
}
public void reset() {
@@ -1761,8 +1908,13 @@
/** Increments the boot counter, and returns whether the device is bootlooping. */
+ @GuardedBy("mLock")
public boolean incrementAndTest() {
- readMitigationCountFromMetadataIfNecessary();
+ if (Flags.recoverabilityDetection()) {
+ readAllObserversBootMitigationCountIfNecessary(METADATA_FILE);
+ } else {
+ readMitigationCountFromMetadataIfNecessary();
+ }
final long now = mSystemClock.uptimeMillis();
if (now - getStart() < 0) {
Slog.e(TAG, "Window was less than zero. Resetting start to current time.");
@@ -1770,8 +1922,12 @@
setMitigationStart(now);
}
if (now - getMitigationStart() > DEFAULT_DEESCALATION_WINDOW_MS) {
- setMitigationCount(0);
setMitigationStart(now);
+ if (Flags.recoverabilityDetection()) {
+ resetAllObserversBootMitigationCount();
+ } else {
+ setMitigationCount(0);
+ }
}
final long window = now - getStart();
if (window >= mTriggerWindow) {
@@ -1782,9 +1938,48 @@
int count = getCount() + 1;
setCount(count);
EventLogTags.writeRescueNote(Process.ROOT_UID, count, window);
+ if (Flags.recoverabilityDetection()) {
+ boolean mitigate = (count >= mBootTriggerCount)
+ && (count - mBootTriggerCount) % mBootMitigationIncrement == 0;
+ return mitigate;
+ }
return count >= mBootTriggerCount;
}
}
+ @GuardedBy("mLock")
+ private void resetAllObserversBootMitigationCount() {
+ for (int i = 0; i < mAllObservers.size(); i++) {
+ final ObserverInternal observer = mAllObservers.valueAt(i);
+ observer.setBootMitigationCount(0);
+ }
+ }
+
+ @GuardedBy("mLock")
+ @SuppressWarnings("GuardedBy")
+ void readAllObserversBootMitigationCountIfNecessary(String filePath) {
+ File metadataFile = new File(filePath);
+ if (metadataFile.exists()) {
+ try {
+ FileInputStream fileStream = new FileInputStream(metadataFile);
+ ObjectInputStream objectStream = new ObjectInputStream(fileStream);
+ HashMap<String, Integer> bootMitigationCounts =
+ (HashMap<String, Integer>) objectStream.readObject();
+ objectStream.close();
+ fileStream.close();
+
+ for (int i = 0; i < mAllObservers.size(); i++) {
+ final ObserverInternal observer = mAllObservers.valueAt(i);
+ if (bootMitigationCounts.containsKey(observer.name)) {
+ observer.setBootMitigationCount(
+ bootMitigationCounts.get(observer.name));
+ }
+ }
+ } catch (Exception e) {
+ Slog.i(TAG, "Could not read observer metadata file: " + e);
+ }
+ }
+ }
+
}
}
diff --git a/packages/CrashRecovery/services/java/com/android/server/RescueParty.java b/packages/CrashRecovery/services/java/com/android/server/RescueParty.java
index 7bdc1a0..7093ba4 100644
--- a/packages/CrashRecovery/services/java/com/android/server/RescueParty.java
+++ b/packages/CrashRecovery/services/java/com/android/server/RescueParty.java
@@ -20,6 +20,7 @@
import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo;
+import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.ContentResolver;
@@ -27,6 +28,7 @@
import android.content.pm.ApplicationInfo;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
+import android.crashrecovery.flags.Flags;
import android.os.Build;
import android.os.Environment;
import android.os.PowerManager;
@@ -53,6 +55,8 @@
import com.android.server.crashrecovery.proto.CrashRecoveryStatsLog;
import java.io.File;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -89,6 +93,40 @@
@VisibleForTesting
static final int LEVEL_FACTORY_RESET = 5;
@VisibleForTesting
+ static final int RESCUE_LEVEL_NONE = 0;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET = 1;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET = 2;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_WARM_REBOOT = 3;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS = 4;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES = 5;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS = 6;
+ @VisibleForTesting
+ static final int RESCUE_LEVEL_FACTORY_RESET = 7;
+
+ @IntDef(prefix = { "RESCUE_LEVEL_" }, value = {
+ RESCUE_LEVEL_NONE,
+ RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET,
+ RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET,
+ RESCUE_LEVEL_WARM_REBOOT,
+ RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS,
+ RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES,
+ RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS,
+ RESCUE_LEVEL_FACTORY_RESET
+ })
+ @Retention(RetentionPolicy.SOURCE)
+ @interface RescueLevels {}
+
+ @VisibleForTesting
+ static final String RESCUE_NON_REBOOT_LEVEL_LIMIT = "persist.sys.rescue_non_reboot_level_limit";
+ @VisibleForTesting
+ static final int DEFAULT_RESCUE_NON_REBOOT_LEVEL_LIMIT = RESCUE_LEVEL_WARM_REBOOT - 1;
+ @VisibleForTesting
static final String TAG = "RescueParty";
@VisibleForTesting
static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2);
@@ -347,11 +385,20 @@
}
private static int getMaxRescueLevel(boolean mayPerformReboot) {
- if (!mayPerformReboot
- || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {
- return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;
+ if (Flags.recoverabilityDetection()) {
+ if (!mayPerformReboot
+ || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {
+ return SystemProperties.getInt(RESCUE_NON_REBOOT_LEVEL_LIMIT,
+ DEFAULT_RESCUE_NON_REBOOT_LEVEL_LIMIT);
+ }
+ return RESCUE_LEVEL_FACTORY_RESET;
+ } else {
+ if (!mayPerformReboot
+ || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {
+ return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;
+ }
+ return LEVEL_FACTORY_RESET;
}
- return LEVEL_FACTORY_RESET;
}
/**
@@ -379,6 +426,46 @@
}
}
+ /**
+ * Get the rescue level to perform if this is the n-th attempt at mitigating failure.
+ * When failedPackage is null then 1st and 2nd mitigation counts are redundant (scoped and
+ * all device config reset). Behaves as if one mitigation attempt was already done.
+ *
+ * @param mitigationCount the mitigation attempt number (1 = first attempt etc.).
+ * @param mayPerformReboot whether or not a reboot and factory reset may be performed
+ * for the given failure.
+ * @param failedPackage in case of bootloop this is null.
+ * @return the rescue level for the n-th mitigation attempt.
+ */
+ private static @RescueLevels int getRescueLevel(int mitigationCount, boolean mayPerformReboot,
+ @Nullable VersionedPackage failedPackage) {
+ // Skipping RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET since it's not defined without a failed
+ // package.
+ if (failedPackage == null && mitigationCount > 0) {
+ mitigationCount += 1;
+ }
+ if (mitigationCount == 1) {
+ return RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET;
+ } else if (mitigationCount == 2) {
+ return RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET;
+ } else if (mitigationCount == 3) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot), RESCUE_LEVEL_WARM_REBOOT);
+ } else if (mitigationCount == 4) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot),
+ RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS);
+ } else if (mitigationCount == 5) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot),
+ RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES);
+ } else if (mitigationCount == 6) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot),
+ RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS);
+ } else if (mitigationCount >= 7) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot), RESCUE_LEVEL_FACTORY_RESET);
+ } else {
+ return RESCUE_LEVEL_NONE;
+ }
+ }
+
private static void executeRescueLevel(Context context, @Nullable String failedPackage,
int level) {
Slog.w(TAG, "Attempting rescue level " + levelToString(level));
@@ -397,6 +484,15 @@
private static void executeRescueLevelInternal(Context context, int level, @Nullable
String failedPackage) throws Exception {
+ if (Flags.recoverabilityDetection()) {
+ executeRescueLevelInternalNew(context, level, failedPackage);
+ } else {
+ executeRescueLevelInternalOld(context, level, failedPackage);
+ }
+ }
+
+ private static void executeRescueLevelInternalOld(Context context, int level, @Nullable
+ String failedPackage) throws Exception {
if (level <= LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS) {
// Disabling flag resets on master branch for trunk stable launch.
@@ -410,8 +506,6 @@
// Try our best to reset all settings possible, and once finished
// rethrow any exception that we encountered
Exception res = null;
- Runnable runnable;
- Thread thread;
switch (level) {
case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
try {
@@ -453,21 +547,7 @@
}
break;
case LEVEL_WARM_REBOOT:
- // Request the reboot from a separate thread to avoid deadlock on PackageWatchdog
- // when device shutting down.
- setRebootProperty(true);
- runnable = () -> {
- try {
- PowerManager pm = context.getSystemService(PowerManager.class);
- if (pm != null) {
- pm.reboot(TAG);
- }
- } catch (Throwable t) {
- logRescueException(level, failedPackage, t);
- }
- };
- thread = new Thread(runnable);
- thread.start();
+ executeWarmReboot(context, level, failedPackage);
break;
case LEVEL_FACTORY_RESET:
// Before the completion of Reboot, if any crash happens then PackageWatchdog
@@ -475,23 +555,9 @@
// Adding a check to prevent factory reset to execute before above reboot completes.
// Note: this reboot property is not persistent resets after reboot is completed.
if (isRebootPropertySet()) {
- break;
+ return;
}
- setFactoryResetProperty(true);
- long now = System.currentTimeMillis();
- setLastFactoryResetTimeMs(now);
- runnable = new Runnable() {
- @Override
- public void run() {
- try {
- RecoverySystem.rebootPromptAndWipeUserData(context, TAG);
- } catch (Throwable t) {
- logRescueException(level, failedPackage, t);
- }
- }
- };
- thread = new Thread(runnable);
- thread.start();
+ executeFactoryReset(context, level, failedPackage);
break;
}
@@ -500,6 +566,83 @@
}
}
+ private static void executeRescueLevelInternalNew(Context context, @RescueLevels int level,
+ @Nullable String failedPackage) throws Exception {
+ CrashRecoveryStatsLog.write(CrashRecoveryStatsLog.RESCUE_PARTY_RESET_REPORTED,
+ level, levelToString(level));
+ switch (level) {
+ case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET:
+ // Temporary disable deviceConfig reset
+ // resetDeviceConfig(context, /*isScoped=*/true, failedPackage);
+ break;
+ case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET:
+ // Temporary disable deviceConfig reset
+ // resetDeviceConfig(context, /*isScoped=*/false, failedPackage);
+ break;
+ case RESCUE_LEVEL_WARM_REBOOT:
+ executeWarmReboot(context, level, failedPackage);
+ break;
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_DEFAULTS, level);
+ break;
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_CHANGES, level);
+ break;
+ case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ resetAllSettingsIfNecessary(context, Settings.RESET_MODE_TRUSTED_DEFAULTS, level);
+ break;
+ case RESCUE_LEVEL_FACTORY_RESET:
+ // Before the completion of Reboot, if any crash happens then PackageWatchdog
+ // escalates to next level i.e. factory reset, as they happen in separate threads.
+ // Adding a check to prevent factory reset to execute before above reboot completes.
+ // Note: this reboot property is not persistent resets after reboot is completed.
+ if (isRebootPropertySet()) {
+ return;
+ }
+ executeFactoryReset(context, level, failedPackage);
+ break;
+ }
+ }
+
+ private static void executeWarmReboot(Context context, int level,
+ @Nullable String failedPackage) {
+ // Request the reboot from a separate thread to avoid deadlock on PackageWatchdog
+ // when device shutting down.
+ setRebootProperty(true);
+ Runnable runnable = () -> {
+ try {
+ PowerManager pm = context.getSystemService(PowerManager.class);
+ if (pm != null) {
+ pm.reboot(TAG);
+ }
+ } catch (Throwable t) {
+ logRescueException(level, failedPackage, t);
+ }
+ };
+ Thread thread = new Thread(runnable);
+ thread.start();
+ }
+
+ private static void executeFactoryReset(Context context, int level,
+ @Nullable String failedPackage) {
+ setFactoryResetProperty(true);
+ long now = System.currentTimeMillis();
+ setLastFactoryResetTimeMs(now);
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ RecoverySystem.rebootPromptAndWipeUserData(context, TAG);
+ } catch (Throwable t) {
+ logRescueException(level, failedPackage, t);
+ }
+ }
+ };
+ Thread thread = new Thread(runnable);
+ thread.start();
+ }
+
+
private static String getCompleteMessage(Throwable t) {
final StringBuilder builder = new StringBuilder();
builder.append(t.getMessage());
@@ -521,17 +664,38 @@
}
private static int mapRescueLevelToUserImpact(int rescueLevel) {
- switch(rescueLevel) {
- case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
- case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
- return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10;
- case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
- case LEVEL_WARM_REBOOT:
- return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
- case LEVEL_FACTORY_RESET:
- return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100;
- default:
- return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ if (Flags.recoverabilityDetection()) {
+ switch (rescueLevel) {
+ case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10;
+ case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_20;
+ case RESCUE_LEVEL_WARM_REBOOT:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_71;
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_75;
+ case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_80;
+ case RESCUE_LEVEL_FACTORY_RESET:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100;
+ default:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ }
+ } else {
+ switch (rescueLevel) {
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10;
+ case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ case LEVEL_WARM_REBOOT:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
+ case LEVEL_FACTORY_RESET:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100;
+ default:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ }
}
}
@@ -548,7 +712,7 @@
final ContentResolver resolver = context.getContentResolver();
try {
Settings.Global.resetToDefaultsAsUser(resolver, null, mode,
- UserHandle.SYSTEM.getIdentifier());
+ UserHandle.SYSTEM.getIdentifier());
} catch (Exception e) {
res = new RuntimeException("Failed to reset global settings", e);
}
@@ -667,8 +831,13 @@
@FailureReasons int failureReason, int mitigationCount) {
if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
|| failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) {
- return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
+ if (Flags.recoverabilityDetection()) {
+ return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
+ mayPerformReboot(failedPackage), failedPackage));
+ } else {
+ return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
mayPerformReboot(failedPackage)));
+ }
} else {
return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
}
@@ -682,8 +851,10 @@
}
if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
|| failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {
- final int level = getRescueLevel(mitigationCount,
- mayPerformReboot(failedPackage));
+ final int level = Flags.recoverabilityDetection() ? getRescueLevel(mitigationCount,
+ mayPerformReboot(failedPackage), failedPackage)
+ : getRescueLevel(mitigationCount,
+ mayPerformReboot(failedPackage));
executeRescueLevel(mContext,
failedPackage == null ? null : failedPackage.getPackageName(), level);
return true;
@@ -716,7 +887,12 @@
if (isDisabled()) {
return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
}
- return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, true));
+ if (Flags.recoverabilityDetection()) {
+ return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
+ true, /*failedPackage=*/ null));
+ } else {
+ return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, true));
+ }
}
@Override
@@ -725,8 +901,10 @@
return false;
}
boolean mayPerformReboot = !shouldThrottleReboot();
- executeRescueLevel(mContext, /*failedPackage=*/ null,
- getRescueLevel(mitigationCount, mayPerformReboot));
+ final int level = Flags.recoverabilityDetection() ? getRescueLevel(mitigationCount,
+ mayPerformReboot, /*failedPackage=*/ null)
+ : getRescueLevel(mitigationCount, mayPerformReboot);
+ executeRescueLevel(mContext, /*failedPackage=*/ null, level);
return true;
}
@@ -843,14 +1021,44 @@
}
private static String levelToString(int level) {
- switch (level) {
- case LEVEL_NONE: return "NONE";
- case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS: return "RESET_SETTINGS_UNTRUSTED_DEFAULTS";
- case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES: return "RESET_SETTINGS_UNTRUSTED_CHANGES";
- case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS: return "RESET_SETTINGS_TRUSTED_DEFAULTS";
- case LEVEL_WARM_REBOOT: return "WARM_REBOOT";
- case LEVEL_FACTORY_RESET: return "FACTORY_RESET";
- default: return Integer.toString(level);
+ if (Flags.recoverabilityDetection()) {
+ switch (level) {
+ case RESCUE_LEVEL_NONE:
+ return "NONE";
+ case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET:
+ return "SCOPED_DEVICE_CONFIG_RESET";
+ case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET:
+ return "ALL_DEVICE_CONFIG_RESET";
+ case RESCUE_LEVEL_WARM_REBOOT:
+ return "WARM_REBOOT";
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ return "RESET_SETTINGS_UNTRUSTED_DEFAULTS";
+ case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ return "RESET_SETTINGS_UNTRUSTED_CHANGES";
+ case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ return "RESET_SETTINGS_TRUSTED_DEFAULTS";
+ case RESCUE_LEVEL_FACTORY_RESET:
+ return "FACTORY_RESET";
+ default:
+ return Integer.toString(level);
+ }
+ } else {
+ switch (level) {
+ case LEVEL_NONE:
+ return "NONE";
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ return "RESET_SETTINGS_UNTRUSTED_DEFAULTS";
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ return "RESET_SETTINGS_UNTRUSTED_CHANGES";
+ case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ return "RESET_SETTINGS_TRUSTED_DEFAULTS";
+ case LEVEL_WARM_REBOOT:
+ return "WARM_REBOOT";
+ case LEVEL_FACTORY_RESET:
+ return "FACTORY_RESET";
+ default:
+ return Integer.toString(level);
+ }
}
}
}
diff --git a/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java
index 0fb9327..93f26ae 100644
--- a/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java
+++ b/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java
@@ -69,7 +69,7 @@
*
* @hide
*/
-final class RollbackPackageHealthObserver implements PackageHealthObserver {
+public final class RollbackPackageHealthObserver implements PackageHealthObserver {
private static final String TAG = "RollbackPackageHealthObserver";
private static final String NAME = "rollback-observer";
private static final int PERSISTENT_MASK = ApplicationInfo.FLAG_PERSISTENT
@@ -89,7 +89,7 @@
private boolean mTwoPhaseRollbackEnabled;
@VisibleForTesting
- RollbackPackageHealthObserver(Context context, ApexManager apexManager) {
+ public RollbackPackageHealthObserver(Context context, ApexManager apexManager) {
mContext = context;
HandlerThread handlerThread = new HandlerThread("RollbackPackageHealthObserver");
handlerThread.start();
diff --git a/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java b/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java
index c30ac2d..682569f 100644
--- a/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java
+++ b/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java
@@ -26,6 +26,7 @@
import static com.android.dx.mockito.inline.extended.ExtendedMockito.verify;
import static com.android.dx.mockito.inline.extended.ExtendedMockito.when;
import static com.android.server.RescueParty.LEVEL_FACTORY_RESET;
+import static com.android.server.RescueParty.RESCUE_LEVEL_FACTORY_RESET;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -41,9 +42,11 @@
import android.content.pm.ApplicationInfo;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
+import android.crashrecovery.flags.Flags;
import android.os.RecoverySystem;
import android.os.SystemProperties;
import android.os.UserHandle;
+import android.platform.test.flag.junit.SetFlagsRule;
import android.provider.DeviceConfig;
import android.provider.Settings;
import android.util.ArraySet;
@@ -55,6 +58,7 @@
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.mockito.Answers;
import org.mockito.ArgumentCaptor;
@@ -69,6 +73,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
@@ -100,6 +105,9 @@
private static final int THROTTLING_DURATION_MIN = 10;
+ @Rule
+ public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();
+
private MockitoSession mSession;
private HashMap<String, String> mSystemSettingsMap;
private HashMap<String, String> mCrashRecoveryPropertiesMap;
@@ -267,6 +275,42 @@
}
@Test
+ public void testBootLoopDetectionWithExecutionForAllRescueLevelsRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ RescueParty.onSettingsProviderPublished(mMockContext);
+ verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
+ any(Executor.class),
+ mMonitorCallbackCaptor.capture()));
+ HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
+
+ // Record DeviceConfig accesses
+ DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
+
+ final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
+
+ noteBoot(1);
+ verifyDeviceConfigReset(expectedAllResetNamespaces, verifiedTimesMap);
+
+ noteBoot(2);
+ assertTrue(RescueParty.isRebootPropertySet());
+
+ noteBoot(3);
+ verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
+
+ noteBoot(4);
+ verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_CHANGES);
+
+ noteBoot(5);
+ verifyOnlySettingsReset(Settings.RESET_MODE_TRUSTED_DEFAULTS);
+
+ setCrashRecoveryPropAttemptingReboot(false);
+ noteBoot(6);
+ assertTrue(RescueParty.isFactoryResetPropertySet());
+ }
+
+ @Test
public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevels() {
noteAppCrash(1, true);
@@ -292,6 +336,47 @@
}
@Test
+ public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevelsRecoverability() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ RescueParty.onSettingsProviderPublished(mMockContext);
+ verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
+ any(Executor.class),
+ mMonitorCallbackCaptor.capture()));
+ HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
+
+ // Record DeviceConfig accesses
+ DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
+ monitorCallback.onDeviceConfigAccess(PERSISTENT_PACKAGE, NAMESPACE1);
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
+
+ final String[] expectedResetNamespaces = new String[]{NAMESPACE1};
+ final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
+
+ noteAppCrash(1, true);
+ verifyDeviceConfigReset(expectedResetNamespaces, verifiedTimesMap);
+
+ noteAppCrash(2, true);
+ verifyDeviceConfigReset(expectedAllResetNamespaces, verifiedTimesMap);
+
+ noteAppCrash(3, true);
+ assertTrue(RescueParty.isRebootPropertySet());
+
+ noteAppCrash(4, true);
+ verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
+
+ noteAppCrash(5, true);
+ verifyOnlySettingsReset(Settings.RESET_MODE_UNTRUSTED_CHANGES);
+
+ noteAppCrash(6, true);
+ verifyOnlySettingsReset(Settings.RESET_MODE_TRUSTED_DEFAULTS);
+
+ setCrashRecoveryPropAttemptingReboot(false);
+ noteAppCrash(7, true);
+ assertTrue(RescueParty.isFactoryResetPropertySet());
+ }
+
+ @Test
public void testNonPersistentAppOnlyPerformsFlagResets() {
noteAppCrash(1, false);
@@ -316,6 +401,45 @@
}
@Test
+ public void testNonPersistentAppOnlyPerformsFlagResetsRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ RescueParty.onSettingsProviderPublished(mMockContext);
+ verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
+ any(Executor.class),
+ mMonitorCallbackCaptor.capture()));
+ HashMap<String, Integer> verifiedTimesMap = new HashMap<String, Integer>();
+
+ // Record DeviceConfig accesses
+ DeviceConfig.MonitorCallback monitorCallback = mMonitorCallbackCaptor.getValue();
+ monitorCallback.onDeviceConfigAccess(NON_PERSISTENT_PACKAGE, NAMESPACE1);
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE1);
+ monitorCallback.onDeviceConfigAccess(CALLING_PACKAGE1, NAMESPACE2);
+
+ final String[] expectedResetNamespaces = new String[]{NAMESPACE1};
+ final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2};
+
+ noteAppCrash(1, false);
+ verifyDeviceConfigReset(expectedResetNamespaces, verifiedTimesMap);
+
+ noteAppCrash(2, false);
+ verifyDeviceConfigReset(expectedAllResetNamespaces, verifiedTimesMap);
+
+ noteAppCrash(3, false);
+ assertFalse(RescueParty.isRebootPropertySet());
+
+ noteAppCrash(4, false);
+ verifyNoSettingsReset(Settings.RESET_MODE_UNTRUSTED_DEFAULTS);
+ noteAppCrash(5, false);
+ verifyNoSettingsReset(Settings.RESET_MODE_UNTRUSTED_CHANGES);
+ noteAppCrash(6, false);
+ verifyNoSettingsReset(Settings.RESET_MODE_TRUSTED_DEFAULTS);
+
+ setCrashRecoveryPropAttemptingReboot(false);
+ noteAppCrash(7, false);
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+ }
+
+ @Test
public void testNonPersistentAppCrashDetectionWithScopedResets() {
RescueParty.onSettingsProviderPublished(mMockContext);
verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
@@ -451,6 +575,19 @@
}
@Test
+ public void testIsRecoveryTriggeredRebootRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ for (int i = 0; i < RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteBoot(i + 1);
+ }
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+ setCrashRecoveryPropAttemptingReboot(false);
+ noteBoot(RESCUE_LEVEL_FACTORY_RESET + 1);
+ assertTrue(RescueParty.isRecoveryTriggeredReboot());
+ assertTrue(RescueParty.isFactoryResetPropertySet());
+ }
+
+ @Test
public void testIsRecoveryTriggeredRebootOnlyAfterRebootCompleted() {
for (int i = 0; i < LEVEL_FACTORY_RESET; i++) {
noteBoot(i + 1);
@@ -469,6 +606,25 @@
}
@Test
+ public void testIsRecoveryTriggeredRebootOnlyAfterRebootCompletedRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ for (int i = 0; i < RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteBoot(i + 1);
+ }
+ int mitigationCount = RESCUE_LEVEL_FACTORY_RESET + 1;
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+ noteBoot(mitigationCount++);
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+ noteBoot(mitigationCount++);
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+ noteBoot(mitigationCount++);
+ setCrashRecoveryPropAttemptingReboot(false);
+ noteBoot(mitigationCount + 1);
+ assertTrue(RescueParty.isRecoveryTriggeredReboot());
+ assertTrue(RescueParty.isFactoryResetPropertySet());
+ }
+
+ @Test
public void testThrottlingOnBootFailures() {
setCrashRecoveryPropAttemptingReboot(false);
long now = System.currentTimeMillis();
@@ -481,6 +637,19 @@
}
@Test
+ public void testThrottlingOnBootFailuresRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ setCrashRecoveryPropAttemptingReboot(false);
+ long now = System.currentTimeMillis();
+ long beforeTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN - 1);
+ setCrashRecoveryPropLastFactoryReset(beforeTimeout);
+ for (int i = 1; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteBoot(i);
+ }
+ assertFalse(RescueParty.isRecoveryTriggeredReboot());
+ }
+
+ @Test
public void testThrottlingOnAppCrash() {
setCrashRecoveryPropAttemptingReboot(false);
long now = System.currentTimeMillis();
@@ -493,6 +662,19 @@
}
@Test
+ public void testThrottlingOnAppCrashRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ setCrashRecoveryPropAttemptingReboot(false);
+ long now = System.currentTimeMillis();
+ long beforeTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN - 1);
+ setCrashRecoveryPropLastFactoryReset(beforeTimeout);
+ for (int i = 0; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteAppCrash(i + 1, true);
+ }
+ assertFalse(RescueParty.isRecoveryTriggeredReboot());
+ }
+
+ @Test
public void testNotThrottlingAfterTimeoutOnBootFailures() {
setCrashRecoveryPropAttemptingReboot(false);
long now = System.currentTimeMillis();
@@ -503,6 +685,20 @@
}
assertTrue(RescueParty.isRecoveryTriggeredReboot());
}
+
+ @Test
+ public void testNotThrottlingAfterTimeoutOnBootFailuresRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ setCrashRecoveryPropAttemptingReboot(false);
+ long now = System.currentTimeMillis();
+ long afterTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN + 1);
+ setCrashRecoveryPropLastFactoryReset(afterTimeout);
+ for (int i = 1; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteBoot(i);
+ }
+ assertTrue(RescueParty.isRecoveryTriggeredReboot());
+ }
+
@Test
public void testNotThrottlingAfterTimeoutOnAppCrash() {
setCrashRecoveryPropAttemptingReboot(false);
@@ -516,6 +712,19 @@
}
@Test
+ public void testNotThrottlingAfterTimeoutOnAppCrashRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ setCrashRecoveryPropAttemptingReboot(false);
+ long now = System.currentTimeMillis();
+ long afterTimeout = now - TimeUnit.MINUTES.toMillis(THROTTLING_DURATION_MIN + 1);
+ setCrashRecoveryPropLastFactoryReset(afterTimeout);
+ for (int i = 0; i <= RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteAppCrash(i + 1, true);
+ }
+ assertTrue(RescueParty.isRecoveryTriggeredReboot());
+ }
+
+ @Test
public void testNativeRescuePartyResets() {
doReturn(true).when(() -> SettingsToPropertiesMapper.isNativeFlagsResetPerformed());
doReturn(FAKE_RESET_NATIVE_NAMESPACES).when(
@@ -531,6 +740,7 @@
@Test
public void testExplicitlyEnablingAndDisablingRescue() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(false));
SystemProperties.set(PROP_DISABLE_RESCUE, Boolean.toString(true));
assertEquals(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage,
@@ -543,6 +753,7 @@
@Test
public void testDisablingRescueByDeviceConfigFlag() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(false));
SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(true));
@@ -568,6 +779,20 @@
}
@Test
+ public void testDisablingFactoryResetByDeviceConfigFlagRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ SystemProperties.set(PROP_DISABLE_FACTORY_RESET_FLAG, Boolean.toString(true));
+
+ for (int i = 0; i < RESCUE_LEVEL_FACTORY_RESET; i++) {
+ noteBoot(i + 1);
+ }
+ assertFalse(RescueParty.isFactoryResetPropertySet());
+
+ // Restore the property value initialized in SetUp()
+ SystemProperties.set(PROP_DISABLE_FACTORY_RESET_FLAG, "");
+ }
+
+ @Test
public void testHealthCheckLevels() {
RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
@@ -594,6 +819,46 @@
}
@Test
+ public void testHealthCheckLevelsRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
+
+ // Ensure that no action is taken for cases where the failure reason is unknown
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_UNKNOWN, 1),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
+
+ // Ensure the correct user impact is returned for each mitigation count.
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 5),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 6),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+
+ assertEquals(observer.onHealthCheckFailed(sFailingPackage,
+ PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 7),
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+ }
+
+ @Test
public void testBootLoopLevels() {
RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
@@ -606,6 +871,19 @@
}
@Test
+ public void testBootLoopLevelsRecoverabilityDetection() {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext);
+
+ assertEquals(observer.onBootLoop(1), PackageHealthObserverImpact.USER_IMPACT_LEVEL_20);
+ assertEquals(observer.onBootLoop(2), PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+ assertEquals(observer.onBootLoop(3), PackageHealthObserverImpact.USER_IMPACT_LEVEL_71);
+ assertEquals(observer.onBootLoop(4), PackageHealthObserverImpact.USER_IMPACT_LEVEL_75);
+ assertEquals(observer.onBootLoop(5), PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);
+ assertEquals(observer.onBootLoop(6), PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);
+ }
+
+ @Test
public void testResetDeviceConfigForPackagesOnlyRuntimeMap() {
RescueParty.onSettingsProviderPublished(mMockContext);
verify(() -> DeviceConfig.setMonitorCallback(eq(mMockContentResolver),
@@ -727,11 +1005,26 @@
private void verifySettingsResets(int resetMode, String[] resetNamespaces,
HashMap<String, Integer> configResetVerifiedTimesMap) {
+ verifyOnlySettingsReset(resetMode);
+ verifyDeviceConfigReset(resetNamespaces, configResetVerifiedTimesMap);
+ }
+
+ private void verifyOnlySettingsReset(int resetMode) {
verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null,
resetMode, UserHandle.USER_SYSTEM));
verify(() -> Settings.Secure.resetToDefaultsAsUser(eq(mMockContentResolver), isNull(),
eq(resetMode), anyInt()));
- // Verify DeviceConfig resets
+ }
+
+ private void verifyNoSettingsReset(int resetMode) {
+ verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null,
+ resetMode, UserHandle.USER_SYSTEM), never());
+ verify(() -> Settings.Secure.resetToDefaultsAsUser(eq(mMockContentResolver), isNull(),
+ eq(resetMode), anyInt()), never());
+ }
+
+ private void verifyDeviceConfigReset(String[] resetNamespaces,
+ Map<String, Integer> configResetVerifiedTimesMap) {
if (resetNamespaces == null) {
verify(() -> DeviceConfig.resetToDefaults(anyInt(), anyString()), never());
} else {
@@ -818,9 +1111,16 @@
// mock properties in BootThreshold
try {
- mSpyBootThreshold = spy(watchdog.new BootThreshold(
- PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
- PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
+ if (Flags.recoverabilityDetection()) {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+ } else {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
+ }
mCrashRecoveryPropertiesMap = new HashMap<>();
doAnswer((Answer<Integer>) invocationOnMock -> {
diff --git a/tests/PackageWatchdog/Android.bp b/tests/PackageWatchdog/Android.bp
index e0e6c4c..2c5fdd3 100644
--- a/tests/PackageWatchdog/Android.bp
+++ b/tests/PackageWatchdog/Android.bp
@@ -28,8 +28,10 @@
static_libs: [
"junit",
"mockito-target-extended-minus-junit4",
+ "flag-junit",
"frameworks-base-testutils",
"androidx.test.rules",
+ "PlatformProperties",
"services.core",
"services.net",
"truth",
diff --git a/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java b/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java
new file mode 100644
index 0000000..081da11
--- /dev/null
+++ b/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java
@@ -0,0 +1,644 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server;
+
+import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;
+
+import static com.android.dx.mockito.inline.extended.ExtendedMockito.doAnswer;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import static org.junit.Assert.assertFalse;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import android.Manifest;
+import android.content.Context;
+import android.content.pm.PackageInfo;
+import android.content.pm.PackageManager;
+import android.content.pm.VersionedPackage;
+import android.content.rollback.PackageRollbackInfo;
+import android.content.rollback.RollbackInfo;
+import android.content.rollback.RollbackManager;
+import android.crashrecovery.flags.Flags;
+import android.net.ConnectivityModuleConnector;
+import android.net.ConnectivityModuleConnector.ConnectivityModuleHealthListener;
+import android.os.Handler;
+import android.os.SystemProperties;
+import android.os.test.TestLooper;
+import android.platform.test.flag.junit.SetFlagsRule;
+import android.provider.DeviceConfig;
+import android.util.AtomicFile;
+
+import androidx.test.InstrumentationRegistry;
+
+import com.android.dx.mockito.inline.extended.ExtendedMockito;
+import com.android.server.RescueParty.RescuePartyObserver;
+import com.android.server.pm.ApexManager;
+import com.android.server.rollback.RollbackPackageHealthObserver;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Answers;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.mockito.MockitoSession;
+import org.mockito.quality.Strictness;
+import org.mockito.stubbing.Answer;
+
+import java.io.File;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+/**
+ * Test CrashRecovery, integration tests that include PackageWatchdog, RescueParty and
+ * RollbackPackageHealthObserver
+ */
+public class CrashRecoveryTest {
+ private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG =
+ "persist.device_config.configuration.disable_rescue_party";
+
+ private static final String APP_A = "com.package.a";
+ private static final String APP_B = "com.package.b";
+ private static final String APP_C = "com.package.c";
+ private static final long VERSION_CODE = 1L;
+ private static final long SHORT_DURATION = TimeUnit.SECONDS.toMillis(1);
+
+ private static final RollbackInfo ROLLBACK_INFO_LOW = getRollbackInfo(APP_A, VERSION_CODE, 1,
+ PackageManager.ROLLBACK_USER_IMPACT_LOW);
+ private static final RollbackInfo ROLLBACK_INFO_HIGH = getRollbackInfo(APP_B, VERSION_CODE, 2,
+ PackageManager.ROLLBACK_USER_IMPACT_HIGH);
+ private static final RollbackInfo ROLLBACK_INFO_MANUAL = getRollbackInfo(APP_C, VERSION_CODE, 3,
+ PackageManager.ROLLBACK_USER_IMPACT_ONLY_MANUAL);
+
+ @Rule
+ public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();
+
+ private final TestClock mTestClock = new TestClock();
+ private TestLooper mTestLooper;
+ private Context mSpyContext;
+ // Keep track of all created watchdogs to apply device config changes
+ private List<PackageWatchdog> mAllocatedWatchdogs;
+ @Mock
+ private ConnectivityModuleConnector mConnectivityModuleConnector;
+ @Mock
+ private PackageManager mMockPackageManager;
+ @Mock(answer = Answers.RETURNS_DEEP_STUBS)
+ private ApexManager mApexManager;
+ @Mock
+ RollbackManager mRollbackManager;
+ // Mock only sysprop apis
+ private PackageWatchdog.BootThreshold mSpyBootThreshold;
+ @Captor
+ private ArgumentCaptor<ConnectivityModuleHealthListener> mConnectivityModuleCallbackCaptor;
+ private MockitoSession mSession;
+ private HashMap<String, String> mSystemSettingsMap;
+ private HashMap<String, String> mCrashRecoveryPropertiesMap;
+
+ @Before
+ public void setUp() throws Exception {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
+ MockitoAnnotations.initMocks(this);
+ new File(InstrumentationRegistry.getContext().getFilesDir(),
+ "package-watchdog.xml").delete();
+ adoptShellPermissions(Manifest.permission.READ_DEVICE_CONFIG,
+ Manifest.permission.WRITE_DEVICE_CONFIG);
+ mTestLooper = new TestLooper();
+ mSpyContext = spy(InstrumentationRegistry.getContext());
+ when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager);
+ when(mMockPackageManager.getPackageInfo(anyString(), anyInt())).then(inv -> {
+ final PackageInfo res = new PackageInfo();
+ res.packageName = inv.getArgument(0);
+ res.setLongVersionCode(VERSION_CODE);
+ return res;
+ });
+ mSession = ExtendedMockito.mockitoSession()
+ .initMocks(this)
+ .strictness(Strictness.LENIENT)
+ .spyStatic(SystemProperties.class)
+ .spyStatic(RescueParty.class)
+ .startMocking();
+ mSystemSettingsMap = new HashMap<>();
+
+ // Mock SystemProperties setter and various getters
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ String value = invocationOnMock.getArgument(1);
+
+ mSystemSettingsMap.put(key, value);
+ return null;
+ }
+ ).when(() -> SystemProperties.set(anyString(), anyString()));
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ int defaultValue = invocationOnMock.getArgument(1);
+
+ String storedValue = mSystemSettingsMap.get(key);
+ return storedValue == null ? defaultValue : Integer.parseInt(storedValue);
+ }
+ ).when(() -> SystemProperties.getInt(anyString(), anyInt()));
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ long defaultValue = invocationOnMock.getArgument(1);
+
+ String storedValue = mSystemSettingsMap.get(key);
+ return storedValue == null ? defaultValue : Long.parseLong(storedValue);
+ }
+ ).when(() -> SystemProperties.getLong(anyString(), anyLong()));
+
+ doAnswer((Answer<Boolean>) invocationOnMock -> {
+ String key = invocationOnMock.getArgument(0);
+ boolean defaultValue = invocationOnMock.getArgument(1);
+
+ String storedValue = mSystemSettingsMap.get(key);
+ return storedValue == null ? defaultValue : Boolean.parseBoolean(storedValue);
+ }
+ ).when(() -> SystemProperties.getBoolean(anyString(), anyBoolean()));
+
+ SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true));
+ SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(false));
+
+ DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
+ PackageWatchdog.PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED,
+ Boolean.toString(true), false);
+
+ DeviceConfig.setProperty(DeviceConfig.NAMESPACE_ROLLBACK,
+ PackageWatchdog.PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
+ Integer.toString(PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT), false);
+
+ mAllocatedWatchdogs = new ArrayList<>();
+ RescuePartyObserver.reset();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ dropShellPermissions();
+ mSession.finishMocking();
+ // Clean up listeners since too many listeners will delay notifications significantly
+ for (PackageWatchdog watchdog : mAllocatedWatchdogs) {
+ watchdog.removePropertyChangedListener();
+ }
+ mAllocatedWatchdogs.clear();
+ }
+
+ @Test
+ public void testBootLoopWithRescueParty() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ RescuePartyObserver rescuePartyObserver = setUpRescuePartyObserver(watchdog);
+
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);
+ int bootCounter = 0;
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(1);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(2);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
+
+ int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;
+ for (int i = 0; i < bootLoopThreshold; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(3);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(4);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(5);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(6);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);
+ }
+
+ @Test
+ public void testBootLoopWithRollbackPackageHealthObserver() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ RollbackPackageHealthObserver rollbackObserver =
+ setUpRollbackPackageHealthObserver(watchdog);
+
+ verify(rollbackObserver, never()).executeBootLoopMitigation(1);
+ int bootCounter = 0;
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rollbackObserver).executeBootLoopMitigation(1);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ // Update the list of available rollbacks after executing bootloop mitigation once
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,
+ ROLLBACK_INFO_MANUAL));
+
+ int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;
+ for (int i = 0; i < bootLoopThreshold; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rollbackObserver).executeBootLoopMitigation(2);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+
+ // Update the list of available rollbacks after executing bootloop mitigation once
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+ }
+
+ @Test
+ public void testBootLoopWithRescuePartyAndRollbackPackageHealthObserver() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ RescuePartyObserver rescuePartyObserver = setUpRescuePartyObserver(watchdog);
+ RollbackPackageHealthObserver rollbackObserver =
+ setUpRollbackPackageHealthObserver(watchdog);
+
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(1);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(1);
+ int bootCounter = 0;
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(1);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(2);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(1);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(2);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ bootCounter += 1;
+ }
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(3);
+ verify(rollbackObserver).executeBootLoopMitigation(1);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+ // Update the list of available rollbacks after executing bootloop mitigation once
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_HIGH,
+ ROLLBACK_INFO_MANUAL));
+
+ int bootLoopThreshold = PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - bootCounter;
+ for (int i = 0; i < bootLoopThreshold; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(3);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(4);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(4);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(5);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(5);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(2);
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(6);
+ verify(rollbackObserver).executeBootLoopMitigation(2);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+ // Update the list of available rollbacks after executing bootloop mitigation
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_MANUAL));
+
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; i++) {
+ watchdog.noteBoot();
+ }
+ verify(rescuePartyObserver).executeBootLoopMitigation(6);
+ verify(rescuePartyObserver, never()).executeBootLoopMitigation(7);
+ verify(rollbackObserver, never()).executeBootLoopMitigation(3);
+ }
+
+ RollbackPackageHealthObserver setUpRollbackPackageHealthObserver(PackageWatchdog watchdog) {
+ RollbackPackageHealthObserver rollbackObserver =
+ spy(new RollbackPackageHealthObserver(mSpyContext, mApexManager));
+ when(mSpyContext.getSystemService(RollbackManager.class)).thenReturn(mRollbackManager);
+ when(mRollbackManager.getAvailableRollbacks()).thenReturn(List.of(ROLLBACK_INFO_LOW,
+ ROLLBACK_INFO_HIGH, ROLLBACK_INFO_MANUAL));
+ when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager);
+
+ watchdog.registerHealthObserver(rollbackObserver);
+ return rollbackObserver;
+ }
+
+ RescuePartyObserver setUpRescuePartyObserver(PackageWatchdog watchdog) {
+ setCrashRecoveryPropRescueBootCount(0);
+ RescuePartyObserver rescuePartyObserver = spy(RescuePartyObserver.getInstance(mSpyContext));
+ assertFalse(RescueParty.isRebootPropertySet());
+ watchdog.registerHealthObserver(rescuePartyObserver);
+ return rescuePartyObserver;
+ }
+
+ private static RollbackInfo getRollbackInfo(String packageName, long versionCode,
+ int rollbackId, int rollbackUserImpact) {
+ VersionedPackage appFrom = new VersionedPackage(packageName, versionCode + 1);
+ VersionedPackage appTo = new VersionedPackage(packageName, versionCode);
+ PackageRollbackInfo packageRollbackInfo = new PackageRollbackInfo(appFrom, appTo, null,
+ null, false, false, null);
+ RollbackInfo rollbackInfo = new RollbackInfo(rollbackId, List.of(packageRollbackInfo),
+ false, null, 111, rollbackUserImpact);
+ return rollbackInfo;
+ }
+
+ private void adoptShellPermissions(String... permissions) {
+ androidx.test.platform.app.InstrumentationRegistry
+ .getInstrumentation()
+ .getUiAutomation()
+ .adoptShellPermissionIdentity(permissions);
+ }
+
+ private void dropShellPermissions() {
+ androidx.test.platform.app.InstrumentationRegistry
+ .getInstrumentation()
+ .getUiAutomation()
+ .dropShellPermissionIdentity();
+ }
+
+
+ private PackageWatchdog createWatchdog() {
+ return createWatchdog(new TestController(), true /* withPackagesReady */);
+ }
+
+ private PackageWatchdog createWatchdog(TestController controller, boolean withPackagesReady) {
+ AtomicFile policyFile =
+ new AtomicFile(new File(mSpyContext.getFilesDir(), "package-watchdog.xml"));
+ Handler handler = new Handler(mTestLooper.getLooper());
+ PackageWatchdog watchdog =
+ new PackageWatchdog(mSpyContext, policyFile, handler, handler, controller,
+ mConnectivityModuleConnector, mTestClock);
+ mockCrashRecoveryProperties(watchdog);
+
+ // Verify controller is not automatically started
+ assertThat(controller.mIsEnabled).isFalse();
+ if (withPackagesReady) {
+ // Only capture the NetworkStack callback for the latest registered watchdog
+ reset(mConnectivityModuleConnector);
+ watchdog.onPackagesReady();
+ // Verify controller by default is started when packages are ready
+ assertThat(controller.mIsEnabled).isTrue();
+
+ verify(mConnectivityModuleConnector).registerHealthListener(
+ mConnectivityModuleCallbackCaptor.capture());
+ }
+ mAllocatedWatchdogs.add(watchdog);
+ return watchdog;
+ }
+
+ // Mock CrashRecoveryProperties as they cannot be accessed due to SEPolicy restrictions
+ private void mockCrashRecoveryProperties(PackageWatchdog watchdog) {
+ mCrashRecoveryPropertiesMap = new HashMap<>();
+
+ // mock properties in RescueParty
+ try {
+
+ doAnswer((Answer<Boolean>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.attempting_factory_reset", "false");
+ return Boolean.parseBoolean(storedValue);
+ }).when(() -> RescueParty.isFactoryResetPropertySet());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ boolean value = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.attempting_factory_reset",
+ Boolean.toString(value));
+ return null;
+ }).when(() -> RescueParty.setFactoryResetProperty(anyBoolean()));
+
+ doAnswer((Answer<Boolean>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.attempting_reboot", "false");
+ return Boolean.parseBoolean(storedValue);
+ }).when(() -> RescueParty.isRebootPropertySet());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ boolean value = invocationOnMock.getArgument(0);
+ setCrashRecoveryPropAttemptingReboot(value);
+ return null;
+ }).when(() -> RescueParty.setRebootProperty(anyBoolean()));
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("persist.crashrecovery.last_factory_reset", "0");
+ return Long.parseLong(storedValue);
+ }).when(() -> RescueParty.getLastFactoryResetTimeMs());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ long value = invocationOnMock.getArgument(0);
+ setCrashRecoveryPropLastFactoryReset(value);
+ return null;
+ }).when(() -> RescueParty.setLastFactoryResetTimeMs(anyLong()));
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.max_rescue_level_attempted", "0");
+ return Integer.parseInt(storedValue);
+ }).when(() -> RescueParty.getMaxRescueLevelAttempted());
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ int value = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.max_rescue_level_attempted",
+ Integer.toString(value));
+ return null;
+ }).when(() -> RescueParty.setMaxRescueLevelAttempted(anyInt()));
+
+ } catch (Exception e) {
+ // tests will fail, just printing the error
+ System.out.println("Error while mocking crashrecovery properties " + e.getMessage());
+ }
+
+ try {
+ if (Flags.recoverabilityDetection()) {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+ } else {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
+ }
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.rescue_boot_count", "0");
+ return Integer.parseInt(storedValue);
+ }).when(mSpyBootThreshold).getCount();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ int count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_count",
+ Integer.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setCount(anyInt());
+
+ doAnswer((Answer<Integer>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.boot_mitigation_count", "0");
+ return Integer.parseInt(storedValue);
+ }).when(mSpyBootThreshold).getMitigationCount();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ int count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.boot_mitigation_count",
+ Integer.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setMitigationCount(anyInt());
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.rescue_boot_start", "0");
+ return Long.parseLong(storedValue);
+ }).when(mSpyBootThreshold).getStart();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ long count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_start",
+ Long.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setStart(anyLong());
+
+ doAnswer((Answer<Long>) invocationOnMock -> {
+ String storedValue = mCrashRecoveryPropertiesMap
+ .getOrDefault("crashrecovery.boot_mitigation_start", "0");
+ return Long.parseLong(storedValue);
+ }).when(mSpyBootThreshold).getMitigationStart();
+ doAnswer((Answer<Void>) invocationOnMock -> {
+ long count = invocationOnMock.getArgument(0);
+ mCrashRecoveryPropertiesMap.put("crashrecovery.boot_mitigation_start",
+ Long.toString(count));
+ return null;
+ }).when(mSpyBootThreshold).setMitigationStart(anyLong());
+
+ Field mBootThresholdField = watchdog.getClass().getDeclaredField("mBootThreshold");
+ mBootThresholdField.setAccessible(true);
+ mBootThresholdField.set(watchdog, mSpyBootThreshold);
+ } catch (Exception e) {
+ // tests will fail, just printing the error
+ System.out.println("Error detected while spying BootThreshold" + e.getMessage());
+ }
+ }
+
+ private void setCrashRecoveryPropRescueBootCount(int count) {
+ mCrashRecoveryPropertiesMap.put("crashrecovery.rescue_boot_count",
+ Integer.toString(count));
+ }
+
+ private void setCrashRecoveryPropAttemptingReboot(boolean value) {
+ mCrashRecoveryPropertiesMap.put("crashrecovery.attempting_reboot",
+ Boolean.toString(value));
+ }
+
+ private void setCrashRecoveryPropLastFactoryReset(long value) {
+ mCrashRecoveryPropertiesMap.put("persist.crashrecovery.last_factory_reset",
+ Long.toString(value));
+ }
+
+ private static class TestController extends ExplicitHealthCheckController {
+ TestController() {
+ super(null /* controller */);
+ }
+
+ private boolean mIsEnabled;
+ private List<String> mSupportedPackages = new ArrayList<>();
+ private List<String> mRequestedPackages = new ArrayList<>();
+ private Consumer<List<PackageConfig>> mSupportedConsumer;
+ private List<Set> mSyncRequests = new ArrayList<>();
+
+ @Override
+ public void setEnabled(boolean enabled) {
+ mIsEnabled = enabled;
+ if (!mIsEnabled) {
+ mSupportedPackages.clear();
+ }
+ }
+
+ @Override
+ public void setCallbacks(Consumer<String> passedConsumer,
+ Consumer<List<PackageConfig>> supportedConsumer, Runnable notifySyncRunnable) {
+ mSupportedConsumer = supportedConsumer;
+ }
+
+ @Override
+ public void syncRequests(Set<String> packages) {
+ mSyncRequests.add(packages);
+ mRequestedPackages.clear();
+ if (mIsEnabled) {
+ packages.retainAll(mSupportedPackages);
+ mRequestedPackages.addAll(packages);
+ List<PackageConfig> packageConfigs = new ArrayList<>();
+ for (String packageName: packages) {
+ packageConfigs.add(new PackageConfig(packageName, SHORT_DURATION));
+ }
+ mSupportedConsumer.accept(packageConfigs);
+ } else {
+ mSupportedConsumer.accept(Collections.emptyList());
+ }
+ }
+ }
+
+ private static class TestClock implements PackageWatchdog.SystemClock {
+ // Note 0 is special to the internal clock of PackageWatchdog. We need to start from
+ // a non-zero value in order not to disrupt the logic of PackageWatchdog.
+ private long mUpTimeMillis = 1;
+ @Override
+ public long uptimeMillis() {
+ return mUpTimeMillis;
+ }
+ }
+}
diff --git a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
index 75284c7..4f27e06 100644
--- a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
+++ b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java
@@ -36,11 +36,13 @@
import android.content.pm.PackageInfo;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
+import android.crashrecovery.flags.Flags;
import android.net.ConnectivityModuleConnector;
import android.net.ConnectivityModuleConnector.ConnectivityModuleHealthListener;
import android.os.Handler;
import android.os.SystemProperties;
import android.os.test.TestLooper;
+import android.platform.test.flag.junit.SetFlagsRule;
import android.provider.DeviceConfig;
import android.util.AtomicFile;
import android.util.Xml;
@@ -54,11 +56,13 @@
import com.android.modules.utils.TypedXmlSerializer;
import com.android.server.PackageWatchdog.HealthCheckState;
import com.android.server.PackageWatchdog.MonitoredPackage;
+import com.android.server.PackageWatchdog.ObserverInternal;
import com.android.server.PackageWatchdog.PackageHealthObserver;
import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
@@ -99,6 +103,10 @@
private static final String OBSERVER_NAME_4 = "observer4";
private static final long SHORT_DURATION = TimeUnit.SECONDS.toMillis(1);
private static final long LONG_DURATION = TimeUnit.SECONDS.toMillis(5);
+
+ @Rule
+ public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();
+
private final TestClock mTestClock = new TestClock();
private TestLooper mTestLooper;
private Context mSpyContext;
@@ -128,6 +136,7 @@
@Before
public void setUp() throws Exception {
+ mSetFlagsRule.enableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
MockitoAnnotations.initMocks(this);
new File(InstrumentationRegistry.getContext().getFilesDir(),
"package-watchdog.xml").delete();
@@ -444,6 +453,7 @@
*/
@Test
public void testPackageFailureNotifyAllDifferentImpacts() throws Exception {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver observerNone = new TestObserver(OBSERVER_NAME_1,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
@@ -488,6 +498,52 @@
assertThat(observerLowPackages).containsExactly(APP_A);
}
+ @Test
+ public void testPackageFailureNotifyAllDifferentImpactsRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver observerNone = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_0);
+ TestObserver observerHigh = new TestObserver(OBSERVER_NAME_2,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+ TestObserver observerMid = new TestObserver(OBSERVER_NAME_3,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ TestObserver observerLow = new TestObserver(OBSERVER_NAME_4,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+
+ // Start observing for all impact observers
+ watchdog.startObservingHealth(observerNone, Arrays.asList(APP_A, APP_B, APP_C, APP_D),
+ SHORT_DURATION);
+ watchdog.startObservingHealth(observerHigh, Arrays.asList(APP_A, APP_B, APP_C),
+ SHORT_DURATION);
+ watchdog.startObservingHealth(observerMid, Arrays.asList(APP_A, APP_B),
+ SHORT_DURATION);
+ watchdog.startObservingHealth(observerLow, Arrays.asList(APP_A),
+ SHORT_DURATION);
+
+ // Then fail all apps above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE),
+ new VersionedPackage(APP_B, VERSION_CODE),
+ new VersionedPackage(APP_C, VERSION_CODE),
+ new VersionedPackage(APP_D, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify least impact observers are notifed of package failures
+ List<String> observerNonePackages = observerNone.mMitigatedPackages;
+ List<String> observerHighPackages = observerHigh.mMitigatedPackages;
+ List<String> observerMidPackages = observerMid.mMitigatedPackages;
+ List<String> observerLowPackages = observerLow.mMitigatedPackages;
+
+ // APP_D failure observed by only observerNone is not caught cos its impact is none
+ assertThat(observerNonePackages).isEmpty();
+ // APP_C failure is caught by observerHigh cos it's the lowest impact observer
+ assertThat(observerHighPackages).containsExactly(APP_C);
+ // APP_B failure is caught by observerMid cos it's the lowest impact observer
+ assertThat(observerMidPackages).containsExactly(APP_B);
+ // APP_A failure is caught by observerLow cos it's the lowest impact observer
+ assertThat(observerLowPackages).containsExactly(APP_A);
+ }
+
/**
* Test package failure and least impact observers are notified successively.
* State transistions:
@@ -501,6 +557,7 @@
*/
@Test
public void testPackageFailureNotifyLeastImpactSuccessively() throws Exception {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver observerFirst = new TestObserver(OBSERVER_NAME_1,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
@@ -563,11 +620,76 @@
assertThat(observerSecond.mMitigatedPackages).isEmpty();
}
+ @Test
+ public void testPackageFailureNotifyLeastImpactSuccessivelyRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver observerFirst = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+ TestObserver observerSecond = new TestObserver(OBSERVER_NAME_2,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+
+ // Start observing for observerFirst and observerSecond with failure handling
+ watchdog.startObservingHealth(observerFirst, Arrays.asList(APP_A), LONG_DURATION);
+ watchdog.startObservingHealth(observerSecond, Arrays.asList(APP_A), LONG_DURATION);
+
+ // Then fail APP_A above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only observerFirst is notifed
+ assertThat(observerFirst.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observerSecond.mMitigatedPackages).isEmpty();
+
+ // After observerFirst handles failure, next action it has is high impact
+ observerFirst.mImpact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
+ observerFirst.mMitigatedPackages.clear();
+ observerSecond.mMitigatedPackages.clear();
+
+ // Then fail APP_A again above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only observerSecond is notifed cos it has least impact
+ assertThat(observerSecond.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observerFirst.mMitigatedPackages).isEmpty();
+
+ // After observerSecond handles failure, it has no further actions
+ observerSecond.mImpact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ observerFirst.mMitigatedPackages.clear();
+ observerSecond.mMitigatedPackages.clear();
+
+ // Then fail APP_A again above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only observerFirst is notifed cos it has the only action
+ assertThat(observerFirst.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observerSecond.mMitigatedPackages).isEmpty();
+
+ // After observerFirst handles failure, it too has no further actions
+ observerFirst.mImpact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ observerFirst.mMitigatedPackages.clear();
+ observerSecond.mMitigatedPackages.clear();
+
+ // Then fail APP_A again above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify no observer is notified cos no actions left
+ assertThat(observerFirst.mMitigatedPackages).isEmpty();
+ assertThat(observerSecond.mMitigatedPackages).isEmpty();
+ }
+
/**
* Test package failure and notifies only one observer even with observer impact tie.
*/
@Test
public void testPackageFailureNotifyOneSameImpact() throws Exception {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver observer1 = new TestObserver(OBSERVER_NAME_1,
PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);
@@ -588,6 +710,28 @@
assertThat(observer2.mMitigatedPackages).isEmpty();
}
+ @Test
+ public void testPackageFailureNotifyOneSameImpactRecoverabilityDetection() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver observer1 = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+ TestObserver observer2 = new TestObserver(OBSERVER_NAME_2,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_50);
+
+ // Start observing for observer1 and observer2 with failure handling
+ watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION);
+ watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION);
+
+ // Then fail APP_A above the threshold
+ raiseFatalFailureAndDispatch(watchdog,
+ Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)),
+ PackageWatchdog.FAILURE_REASON_UNKNOWN);
+
+ // Verify only one observer is notifed
+ assertThat(observer1.mMitigatedPackages).containsExactly(APP_A);
+ assertThat(observer2.mMitigatedPackages).isEmpty();
+ }
+
/**
* Test package passing explicit health checks does not fail and vice versa.
*/
@@ -818,6 +962,7 @@
@Test
public void testNetworkStackFailure() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
final PackageWatchdog wd = createWatchdog();
// Start observing with failure handling
@@ -835,6 +980,25 @@
assertThat(observer.mMitigatedPackages).containsExactly(APP_A);
}
+ @Test
+ public void testNetworkStackFailureRecoverabilityDetection() {
+ final PackageWatchdog wd = createWatchdog();
+
+ // Start observing with failure handling
+ TestObserver observer = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_100);
+ wd.startObservingHealth(observer, Collections.singletonList(APP_A), SHORT_DURATION);
+
+ // Notify of NetworkStack failure
+ mConnectivityModuleCallbackCaptor.getValue().onNetworkStackFailure(APP_A);
+
+ // Run handler so package failures are dispatched to observers
+ mTestLooper.dispatchAll();
+
+ // Verify the NetworkStack observer is notified
+ assertThat(observer.mMitigatedPackages).isEmpty();
+ }
+
/** Test default values are used when device property is invalid. */
@Test
public void testInvalidConfig_watchdogTriggerFailureCount() {
@@ -1045,6 +1209,7 @@
/** Ensure that boot loop mitigation is done when the number of boots meets the threshold. */
@Test
public void testBootLoopDetection_meetsThreshold() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
watchdog.registerHealthObserver(bootObserver);
@@ -1054,6 +1219,16 @@
assertThat(bootObserver.mitigatedBootLoop()).isTrue();
}
+ @Test
+ public void testBootLoopDetection_meetsThresholdRecoverability() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver.mitigatedBootLoop()).isTrue();
+ }
/**
* Ensure that boot loop mitigation is not done when the number of boots does not meet the
@@ -1071,10 +1246,43 @@
}
/**
+ * Ensure that boot loop mitigation is not done when the number of boots does not meet the
+ * threshold.
+ */
+ @Test
+ public void testBootLoopDetection_doesNotMeetThresholdRecoverabilityLowImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver.mitigatedBootLoop()).isFalse();
+ }
+
+ /**
+ * Ensure that boot loop mitigation is not done when the number of boots does not meet the
+ * threshold.
+ */
+ @Test
+ public void testBootLoopDetection_doesNotMeetThresholdRecoverabilityHighImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver.mitigatedBootLoop()).isFalse();
+ }
+
+ /**
* Ensure that boot loop mitigation is done for the observer with the lowest user impact
*/
@Test
public void testBootLoopMitigationDoneForLowestUserImpact() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver bootObserver1 = new TestObserver(OBSERVER_NAME_1);
bootObserver1.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
@@ -1089,11 +1297,28 @@
assertThat(bootObserver2.mitigatedBootLoop()).isFalse();
}
+ @Test
+ public void testBootLoopMitigationDoneForLowestUserImpactRecoverability() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver1 = new TestObserver(OBSERVER_NAME_1);
+ bootObserver1.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_10);
+ TestObserver bootObserver2 = new TestObserver(OBSERVER_NAME_2);
+ bootObserver2.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ watchdog.registerHealthObserver(bootObserver1);
+ watchdog.registerHealthObserver(bootObserver2);
+ for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD; i++) {
+ watchdog.noteBoot();
+ }
+ assertThat(bootObserver1.mitigatedBootLoop()).isTrue();
+ assertThat(bootObserver2.mitigatedBootLoop()).isFalse();
+ }
+
/**
* Ensure that the correct mitigation counts are sent to the boot loop observer.
*/
@Test
public void testMultipleBootLoopMitigation() {
+ mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION);
PackageWatchdog watchdog = createWatchdog();
TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1);
watchdog.registerHealthObserver(bootObserver);
@@ -1114,6 +1339,64 @@
assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));
}
+ @Test
+ public void testMultipleBootLoopMitigationRecoverabilityLowImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);
+
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));
+ }
+
+ @Test
+ public void testMultipleBootLoopMitigationRecoverabilityHighImpact() {
+ PackageWatchdog watchdog = createWatchdog();
+ TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_80);
+ watchdog.registerHealthObserver(bootObserver);
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS + 1);
+
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_THRESHOLD - 1; j++) {
+ watchdog.noteBoot();
+ }
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT; j++) {
+ watchdog.noteBoot();
+ }
+ }
+
+ assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4));
+ }
+
/**
* Ensure that passing a null list of failed packages does not cause any mitigation logic to
* execute.
@@ -1304,6 +1587,78 @@
}
/**
+ * Ensure that a {@link ObserverInternal} may be correctly written and read in order to persist
+ * across reboots.
+ */
+ @Test
+ @SuppressWarnings("GuardedBy")
+ public void testWritingAndReadingObserverInternalRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+
+ LongArrayQueue mitigationCalls = new LongArrayQueue();
+ mitigationCalls.addLast(1000);
+ mitigationCalls.addLast(2000);
+ mitigationCalls.addLast(3000);
+ MonitoredPackage writePkg = watchdog.newMonitoredPackage(
+ "test.package", 1000, 2000, true, mitigationCalls);
+ final int bootMitigationCount = 4;
+ ObserverInternal writeObserver = new ObserverInternal("test", List.of(writePkg),
+ bootMitigationCount);
+
+ // Write the observer
+ File tmpFile = File.createTempFile("observer-watchdog-test", ".xml");
+ AtomicFile testFile = new AtomicFile(tmpFile);
+ FileOutputStream stream = testFile.startWrite();
+ TypedXmlSerializer outputSerializer = Xml.resolveSerializer(stream);
+ outputSerializer.startDocument(null, true);
+ writeObserver.writeLocked(outputSerializer);
+ outputSerializer.endDocument();
+ testFile.finishWrite(stream);
+
+ // Read the observer
+ TypedXmlPullParser parser = Xml.resolvePullParser(testFile.openRead());
+ XmlUtils.beginDocument(parser, "observer");
+ ObserverInternal readObserver = ObserverInternal.read(parser, watchdog);
+
+ assertThat(readObserver.name).isEqualTo(writeObserver.name);
+ assertThat(readObserver.getBootMitigationCount()).isEqualTo(bootMitigationCount);
+ }
+
+ /**
+ * Ensure that boot mitigation counts may be correctly written and read as metadata
+ * in order to persist across reboots.
+ */
+ @Test
+ @SuppressWarnings("GuardedBy")
+ public void testWritingAndReadingMetadataBootMitigationCountRecoverability() throws Exception {
+ PackageWatchdog watchdog = createWatchdog();
+ String filePath = InstrumentationRegistry.getContext().getFilesDir().toString()
+ + "metadata_file.txt";
+
+ ObserverInternal observer1 = new ObserverInternal("test1", List.of(), 1);
+ ObserverInternal observer2 = new ObserverInternal("test2", List.of(), 2);
+ watchdog.registerObserverInternal(observer1);
+ watchdog.registerObserverInternal(observer2);
+
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+
+ watchdog.saveAllObserversBootMitigationCountToMetadata(filePath);
+
+ observer1.setBootMitigationCount(0);
+ observer2.setBootMitigationCount(0);
+ assertThat(observer1.getBootMitigationCount()).isEqualTo(0);
+ assertThat(observer2.getBootMitigationCount()).isEqualTo(0);
+
+ mSpyBootThreshold.readAllObserversBootMitigationCountIfNecessary(filePath);
+
+ assertThat(observer1.getBootMitigationCount()).isEqualTo(1);
+ assertThat(observer2.getBootMitigationCount()).isEqualTo(2);
+ }
+
+ /**
* Tests device config changes are propagated correctly.
*/
@Test
@@ -1440,11 +1795,19 @@
// Mock CrashRecoveryProperties as they cannot be accessed due to SEPolicy restrictions
private void mockCrashRecoveryProperties(PackageWatchdog watchdog) {
+ mCrashRecoveryPropertiesMap = new HashMap<>();
+
try {
- mSpyBootThreshold = spy(watchdog.new BootThreshold(
- PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
- PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
- mCrashRecoveryPropertiesMap = new HashMap<>();
+ if (Flags.recoverabilityDetection()) {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_MITIGATION_INCREMENT));
+ } else {
+ mSpyBootThreshold = spy(watchdog.new BootThreshold(
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS));
+ }
doAnswer((Answer<Integer>) invocationOnMock -> {
String storedValue = mCrashRecoveryPropertiesMap