[CrashRecovery] Separating out required files
We would be moving these files to a new module.
Separating them to a new filegroup, which would be later moved to the
new module behind guarded by build flag when ready.
Bug: b/289203818
Test: m
Change-Id: I5275cf6d416fb74384eb2f0a66d3bcba50dd3fc4
diff --git a/packages/CrashRecovery/OWNERS b/packages/CrashRecovery/OWNERS
new file mode 100644
index 0000000..daa0211
--- /dev/null
+++ b/packages/CrashRecovery/OWNERS
@@ -0,0 +1,3 @@
+ancr@google.com
+harshitmahajan@google.com
+robertogil@google.com
diff --git a/packages/CrashRecovery/framework/Android.bp b/packages/CrashRecovery/framework/Android.bp
new file mode 100644
index 0000000..b2af315
--- /dev/null
+++ b/packages/CrashRecovery/framework/Android.bp
@@ -0,0 +1,9 @@
+filegroup {
+ name: "framework-crashrecovery-sources",
+ srcs: [
+ "java/**/*.java",
+ "java/**/*.aidl",
+ ],
+ path: "java",
+ visibility: ["//frameworks/base:__subpackages__"],
+}
diff --git a/packages/CrashRecovery/framework/java/android/service/watchdog/ExplicitHealthCheckService.java b/packages/CrashRecovery/framework/java/android/service/watchdog/ExplicitHealthCheckService.java
new file mode 100644
index 0000000..7befbfb
--- /dev/null
+++ b/packages/CrashRecovery/framework/java/android/service/watchdog/ExplicitHealthCheckService.java
@@ -0,0 +1,338 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.service.watchdog;
+
+import static android.os.Parcelable.Creator;
+
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.annotation.SdkConstant;
+import android.annotation.SuppressLint;
+import android.annotation.SystemApi;
+import android.annotation.TestApi;
+import android.app.Service;
+import android.content.Intent;
+import android.content.pm.PackageManager;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.IBinder;
+import android.os.Looper;
+import android.os.Parcel;
+import android.os.Parcelable;
+import android.os.RemoteCallback;
+import android.os.RemoteException;
+import android.util.Log;
+
+import com.android.internal.util.Preconditions;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A service to provide packages supporting explicit health checks and route checks to these
+ * packages on behalf of the package watchdog.
+ *
+ * <p>To extend this class, you must declare the service in your manifest file with the
+ * {@link android.Manifest.permission#BIND_EXPLICIT_HEALTH_CHECK_SERVICE} permission,
+ * and include an intent filter with the {@link #SERVICE_INTERFACE} action. In adddition,
+ * your implementation must live in
+ * {@link PackageManager#getServicesSystemSharedLibraryPackageName()}.
+ * For example:</p>
+ * <pre>
+ * <service android:name=".FooExplicitHealthCheckService"
+ * android:exported="true"
+ * android:priority="100"
+ * android:permission="android.permission.BIND_EXPLICIT_HEALTH_CHECK_SERVICE">
+ * <intent-filter>
+ * <action android:name="android.service.watchdog.ExplicitHealthCheckService" />
+ * </intent-filter>
+ * </service>
+ * </pre>
+ * @hide
+ */
+@SystemApi
+public abstract class ExplicitHealthCheckService extends Service {
+
+ private static final String TAG = "ExplicitHealthCheckService";
+
+ /**
+ * {@link Bundle} key for a {@link List} of {@link PackageConfig} value.
+ *
+ * {@hide}
+ */
+ public static final String EXTRA_SUPPORTED_PACKAGES =
+ "android.service.watchdog.extra.supported_packages";
+
+ /**
+ * {@link Bundle} key for a {@link List} of {@link String} value.
+ *
+ * {@hide}
+ */
+ public static final String EXTRA_REQUESTED_PACKAGES =
+ "android.service.watchdog.extra.requested_packages";
+
+ /**
+ * {@link Bundle} key for a {@link String} value.
+ *
+ * {@hide}
+ */
+ public static final String EXTRA_HEALTH_CHECK_PASSED_PACKAGE =
+ "android.service.watchdog.extra.health_check_passed_package";
+
+ /**
+ * The Intent action that a service must respond to. Add it to the intent filter of the service
+ * in its manifest.
+ */
+ @SdkConstant(SdkConstant.SdkConstantType.SERVICE_ACTION)
+ public static final String SERVICE_INTERFACE =
+ "android.service.watchdog.ExplicitHealthCheckService";
+
+ /**
+ * The permission that a service must require to ensure that only Android system can bind to it.
+ * If this permission is not enforced in the AndroidManifest of the service, the system will
+ * skip that service.
+ */
+ public static final String BIND_PERMISSION =
+ "android.permission.BIND_EXPLICIT_HEALTH_CHECK_SERVICE";
+
+ private final ExplicitHealthCheckServiceWrapper mWrapper =
+ new ExplicitHealthCheckServiceWrapper();
+
+ /**
+ * Called when the system requests an explicit health check for {@code packageName}.
+ *
+ * <p> When {@code packageName} passes the check, implementors should call
+ * {@link #notifyHealthCheckPassed} to inform the system.
+ *
+ * <p> It could take many hours before a {@code packageName} passes a check and implementors
+ * should never drop requests unless {@link onCancel} is called or the service dies.
+ *
+ * <p> Requests should not be queued and additional calls while expecting a result for
+ * {@code packageName} should have no effect.
+ */
+ public abstract void onRequestHealthCheck(@NonNull String packageName);
+
+ /**
+ * Called when the system cancels the explicit health check request for {@code packageName}.
+ * Should do nothing if there are is no active request for {@code packageName}.
+ */
+ public abstract void onCancelHealthCheck(@NonNull String packageName);
+
+ /**
+ * Called when the system requests for all the packages supporting explicit health checks. The
+ * system may request an explicit health check for any of these packages with
+ * {@link #onRequestHealthCheck}.
+ *
+ * @return all packages supporting explicit health checks
+ */
+ @NonNull public abstract List<PackageConfig> onGetSupportedPackages();
+
+ /**
+ * Called when the system requests for all the packages that it has currently requested
+ * an explicit health check for.
+ *
+ * @return all packages expecting an explicit health check result
+ */
+ @NonNull public abstract List<String> onGetRequestedPackages();
+
+ private final Handler mHandler = Handler.createAsync(Looper.getMainLooper());
+ @Nullable private RemoteCallback mCallback;
+
+ @Override
+ @NonNull
+ public final IBinder onBind(@NonNull Intent intent) {
+ return mWrapper;
+ }
+
+ /**
+ * Sets {@link RemoteCallback}, for testing purpose.
+ *
+ * @hide
+ */
+ @TestApi
+ public void setCallback(@Nullable RemoteCallback callback) {
+ mCallback = callback;
+ }
+ /**
+ * Implementors should call this to notify the system when explicit health check passes
+ * for {@code packageName};
+ */
+ public final void notifyHealthCheckPassed(@NonNull String packageName) {
+ mHandler.post(() -> {
+ if (mCallback != null) {
+ Objects.requireNonNull(packageName,
+ "Package passing explicit health check must be non-null");
+ Bundle bundle = new Bundle();
+ bundle.putString(EXTRA_HEALTH_CHECK_PASSED_PACKAGE, packageName);
+ mCallback.sendResult(bundle);
+ } else {
+ Log.wtf(TAG, "System missed explicit health check result for " + packageName);
+ }
+ });
+ }
+
+ /**
+ * A PackageConfig contains a package supporting explicit health checks and the
+ * timeout in {@link System#uptimeMillis} across reboots after which health
+ * check requests from clients are failed.
+ *
+ * @hide
+ */
+ @SystemApi
+ public static final class PackageConfig implements Parcelable {
+ private static final long DEFAULT_HEALTH_CHECK_TIMEOUT_MILLIS = TimeUnit.HOURS.toMillis(1);
+
+ private final String mPackageName;
+ private final long mHealthCheckTimeoutMillis;
+
+ /**
+ * Creates a new instance.
+ *
+ * @param packageName the package name
+ * @param durationMillis the duration in milliseconds, must be greater than or
+ * equal to 0. If it is 0, it will use a system default value.
+ */
+ public PackageConfig(@NonNull String packageName, long healthCheckTimeoutMillis) {
+ mPackageName = Preconditions.checkNotNull(packageName);
+ if (healthCheckTimeoutMillis == 0) {
+ mHealthCheckTimeoutMillis = DEFAULT_HEALTH_CHECK_TIMEOUT_MILLIS;
+ } else {
+ mHealthCheckTimeoutMillis = Preconditions.checkArgumentNonnegative(
+ healthCheckTimeoutMillis);
+ }
+ }
+
+ private PackageConfig(Parcel parcel) {
+ mPackageName = parcel.readString();
+ mHealthCheckTimeoutMillis = parcel.readLong();
+ }
+
+ /**
+ * Gets the package name.
+ *
+ * @return the package name
+ */
+ public @NonNull String getPackageName() {
+ return mPackageName;
+ }
+
+ /**
+ * Gets the timeout in milliseconds to evaluate an explicit health check result after a
+ * request.
+ *
+ * @return the duration in {@link System#uptimeMillis} across reboots
+ */
+ public long getHealthCheckTimeoutMillis() {
+ return mHealthCheckTimeoutMillis;
+ }
+
+ @NonNull
+ @Override
+ public String toString() {
+ return "PackageConfig{" + mPackageName + ", " + mHealthCheckTimeoutMillis + "}";
+ }
+
+ @Override
+ public boolean equals(@Nullable Object other) {
+ if (other == this) {
+ return true;
+ }
+ if (!(other instanceof PackageConfig)) {
+ return false;
+ }
+
+ PackageConfig otherInfo = (PackageConfig) other;
+ return Objects.equals(otherInfo.getHealthCheckTimeoutMillis(),
+ mHealthCheckTimeoutMillis)
+ && Objects.equals(otherInfo.getPackageName(), mPackageName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(mPackageName, mHealthCheckTimeoutMillis);
+ }
+
+ @Override
+ public int describeContents() {
+ return 0;
+ }
+
+ @Override
+ public void writeToParcel(@SuppressLint({"MissingNullability"}) Parcel parcel, int flags) {
+ parcel.writeString(mPackageName);
+ parcel.writeLong(mHealthCheckTimeoutMillis);
+ }
+
+ public static final @NonNull Creator<PackageConfig> CREATOR = new Creator<PackageConfig>() {
+ @Override
+ public PackageConfig createFromParcel(Parcel source) {
+ return new PackageConfig(source);
+ }
+
+ @Override
+ public PackageConfig[] newArray(int size) {
+ return new PackageConfig[size];
+ }
+ };
+ }
+
+
+ private class ExplicitHealthCheckServiceWrapper extends IExplicitHealthCheckService.Stub {
+ @Override
+ public void setCallback(RemoteCallback callback) throws RemoteException {
+ mHandler.post(() -> {
+ mCallback = callback;
+ });
+ }
+
+ @Override
+ public void request(String packageName) throws RemoteException {
+ mHandler.post(() -> ExplicitHealthCheckService.this.onRequestHealthCheck(packageName));
+ }
+
+ @Override
+ public void cancel(String packageName) throws RemoteException {
+ mHandler.post(() -> ExplicitHealthCheckService.this.onCancelHealthCheck(packageName));
+ }
+
+ @Override
+ public void getSupportedPackages(RemoteCallback callback) throws RemoteException {
+ mHandler.post(() -> {
+ List<PackageConfig> packages =
+ ExplicitHealthCheckService.this.onGetSupportedPackages();
+ Objects.requireNonNull(packages, "Supported package list must be non-null");
+ Bundle bundle = new Bundle();
+ bundle.putParcelableArrayList(EXTRA_SUPPORTED_PACKAGES, new ArrayList<>(packages));
+ callback.sendResult(bundle);
+ });
+ }
+
+ @Override
+ public void getRequestedPackages(RemoteCallback callback) throws RemoteException {
+ mHandler.post(() -> {
+ List<String> packages =
+ ExplicitHealthCheckService.this.onGetRequestedPackages();
+ Objects.requireNonNull(packages, "Requested package list must be non-null");
+ Bundle bundle = new Bundle();
+ bundle.putStringArrayList(EXTRA_REQUESTED_PACKAGES, new ArrayList<>(packages));
+ callback.sendResult(bundle);
+ });
+ }
+ }
+}
diff --git a/packages/CrashRecovery/framework/java/android/service/watchdog/IExplicitHealthCheckService.aidl b/packages/CrashRecovery/framework/java/android/service/watchdog/IExplicitHealthCheckService.aidl
new file mode 100644
index 0000000..9096509
--- /dev/null
+++ b/packages/CrashRecovery/framework/java/android/service/watchdog/IExplicitHealthCheckService.aidl
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.service.watchdog;
+
+import android.os.RemoteCallback;
+
+/**
+ * @hide
+ */
+@PermissionManuallyEnforced
+oneway interface IExplicitHealthCheckService
+{
+ void setCallback(in @nullable RemoteCallback callback);
+ void request(String packageName);
+ void cancel(String packageName);
+ void getSupportedPackages(in RemoteCallback callback);
+ void getRequestedPackages(in RemoteCallback callback);
+}
diff --git a/packages/CrashRecovery/framework/java/android/service/watchdog/OWNERS b/packages/CrashRecovery/framework/java/android/service/watchdog/OWNERS
new file mode 100644
index 0000000..1c045e1
--- /dev/null
+++ b/packages/CrashRecovery/framework/java/android/service/watchdog/OWNERS
@@ -0,0 +1,3 @@
+narayan@google.com
+nandana@google.com
+olilan@google.com
diff --git a/packages/CrashRecovery/framework/java/android/service/watchdog/PackageConfig.aidl b/packages/CrashRecovery/framework/java/android/service/watchdog/PackageConfig.aidl
new file mode 100644
index 0000000..0131586
--- /dev/null
+++ b/packages/CrashRecovery/framework/java/android/service/watchdog/PackageConfig.aidl
@@ -0,0 +1,22 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.service.watchdog;
+
+/**
+ * @hide
+ */
+parcelable PackageConfig;
diff --git a/packages/CrashRecovery/services/Android.bp b/packages/CrashRecovery/services/Android.bp
new file mode 100644
index 0000000..27ddff9
--- /dev/null
+++ b/packages/CrashRecovery/services/Android.bp
@@ -0,0 +1,9 @@
+filegroup {
+ name: "services-crashrecovery-sources",
+ srcs: [
+ "java/**/*.java",
+ "java/**/*.aidl",
+ ],
+ path: "java",
+ visibility: ["//frameworks/base:__subpackages__"],
+}
diff --git a/packages/CrashRecovery/services/java/com/android/server/ExplicitHealthCheckController.java b/packages/CrashRecovery/services/java/com/android/server/ExplicitHealthCheckController.java
new file mode 100644
index 0000000..3d610d3
--- /dev/null
+++ b/packages/CrashRecovery/services/java/com/android/server/ExplicitHealthCheckController.java
@@ -0,0 +1,454 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server;
+import static android.service.watchdog.ExplicitHealthCheckService.EXTRA_HEALTH_CHECK_PASSED_PACKAGE;
+import static android.service.watchdog.ExplicitHealthCheckService.EXTRA_REQUESTED_PACKAGES;
+import static android.service.watchdog.ExplicitHealthCheckService.EXTRA_SUPPORTED_PACKAGES;
+import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;
+
+import android.Manifest;
+import android.annotation.MainThread;
+import android.annotation.Nullable;
+import android.content.ComponentName;
+import android.content.Context;
+import android.content.Intent;
+import android.content.ServiceConnection;
+import android.content.pm.PackageManager;
+import android.content.pm.ResolveInfo;
+import android.content.pm.ServiceInfo;
+import android.os.IBinder;
+import android.os.RemoteCallback;
+import android.os.RemoteException;
+import android.os.UserHandle;
+import android.service.watchdog.ExplicitHealthCheckService;
+import android.service.watchdog.IExplicitHealthCheckService;
+import android.text.TextUtils;
+import android.util.ArraySet;
+import android.util.Slog;
+
+import com.android.internal.annotations.GuardedBy;
+import com.android.internal.util.Preconditions;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Consumer;
+
+// TODO(b/120598832): Add tests
+/**
+ * Controls the connections with {@link ExplicitHealthCheckService}.
+ */
+class ExplicitHealthCheckController {
+ private static final String TAG = "ExplicitHealthCheckController";
+ private final Object mLock = new Object();
+ private final Context mContext;
+
+ // Called everytime a package passes the health check, so the watchdog is notified of the
+ // passing check. In practice, should never be null after it has been #setEnabled.
+ // To prevent deadlocks between the controller and watchdog threads, we have
+ // a lock invariant to ALWAYS acquire the PackageWatchdog#mLock before #mLock in this class.
+ // It's easier to just NOT hold #mLock when calling into watchdog code on this consumer.
+ @GuardedBy("mLock") @Nullable private Consumer<String> mPassedConsumer;
+ // Called everytime after a successful #syncRequest call, so the watchdog can receive packages
+ // supporting health checks and update its internal state. In practice, should never be null
+ // after it has been #setEnabled.
+ // To prevent deadlocks between the controller and watchdog threads, we have
+ // a lock invariant to ALWAYS acquire the PackageWatchdog#mLock before #mLock in this class.
+ // It's easier to just NOT hold #mLock when calling into watchdog code on this consumer.
+ @GuardedBy("mLock") @Nullable private Consumer<List<PackageConfig>> mSupportedConsumer;
+ // Called everytime we need to notify the watchdog to sync requests between itself and the
+ // health check service. In practice, should never be null after it has been #setEnabled.
+ // To prevent deadlocks between the controller and watchdog threads, we have
+ // a lock invariant to ALWAYS acquire the PackageWatchdog#mLock before #mLock in this class.
+ // It's easier to just NOT hold #mLock when calling into watchdog code on this runnable.
+ @GuardedBy("mLock") @Nullable private Runnable mNotifySyncRunnable;
+ // Actual binder object to the explicit health check service.
+ @GuardedBy("mLock") @Nullable private IExplicitHealthCheckService mRemoteService;
+ // Connection to the explicit health check service, necessary to unbind.
+ // We should only try to bind if mConnection is null, non-null indicates we
+ // are connected or at least connecting.
+ @GuardedBy("mLock") @Nullable private ServiceConnection mConnection;
+ // Bind state of the explicit health check service.
+ @GuardedBy("mLock") private boolean mEnabled;
+
+ ExplicitHealthCheckController(Context context) {
+ mContext = context;
+ }
+
+ /** Enables or disables explicit health checks. */
+ public void setEnabled(boolean enabled) {
+ synchronized (mLock) {
+ Slog.i(TAG, "Explicit health checks " + (enabled ? "enabled." : "disabled."));
+ mEnabled = enabled;
+ }
+ }
+
+ /**
+ * Sets callbacks to listen to important events from the controller.
+ *
+ * <p> Should be called once at initialization before any other calls to the controller to
+ * ensure a happens-before relationship of the set parameters and visibility on other threads.
+ */
+ public void setCallbacks(Consumer<String> passedConsumer,
+ Consumer<List<PackageConfig>> supportedConsumer, Runnable notifySyncRunnable) {
+ synchronized (mLock) {
+ if (mPassedConsumer != null || mSupportedConsumer != null
+ || mNotifySyncRunnable != null) {
+ Slog.wtf(TAG, "Resetting health check controller callbacks");
+ }
+
+ mPassedConsumer = Objects.requireNonNull(passedConsumer);
+ mSupportedConsumer = Objects.requireNonNull(supportedConsumer);
+ mNotifySyncRunnable = Objects.requireNonNull(notifySyncRunnable);
+ }
+ }
+
+ /**
+ * Calls the health check service to request or cancel packages based on
+ * {@code newRequestedPackages}.
+ *
+ * <p> Supported packages in {@code newRequestedPackages} that have not been previously
+ * requested will be requested while supported packages not in {@code newRequestedPackages}
+ * but were previously requested will be cancelled.
+ *
+ * <p> This handles binding and unbinding to the health check service as required.
+ *
+ * <p> Note, calling this may modify {@code newRequestedPackages}.
+ *
+ * <p> Note, this method is not thread safe, all calls should be serialized.
+ */
+ public void syncRequests(Set<String> newRequestedPackages) {
+ boolean enabled;
+ synchronized (mLock) {
+ enabled = mEnabled;
+ }
+
+ if (!enabled) {
+ Slog.i(TAG, "Health checks disabled, no supported packages");
+ // Call outside lock
+ mSupportedConsumer.accept(Collections.emptyList());
+ return;
+ }
+
+ getSupportedPackages(supportedPackageConfigs -> {
+ // Notify the watchdog without lock held
+ mSupportedConsumer.accept(supportedPackageConfigs);
+ getRequestedPackages(previousRequestedPackages -> {
+ synchronized (mLock) {
+ // Hold lock so requests and cancellations are sent atomically.
+ // It is important we don't mix requests from multiple threads.
+
+ Set<String> supportedPackages = new ArraySet<>();
+ for (PackageConfig config : supportedPackageConfigs) {
+ supportedPackages.add(config.getPackageName());
+ }
+ // Note, this may modify newRequestedPackages
+ newRequestedPackages.retainAll(supportedPackages);
+
+ // Cancel packages no longer requested
+ actOnDifference(previousRequestedPackages,
+ newRequestedPackages, p -> cancel(p));
+ // Request packages not yet requested
+ actOnDifference(newRequestedPackages,
+ previousRequestedPackages, p -> request(p));
+
+ if (newRequestedPackages.isEmpty()) {
+ Slog.i(TAG, "No more health check requests, unbinding...");
+ unbindService();
+ return;
+ }
+ }
+ });
+ });
+ }
+
+ private void actOnDifference(Collection<String> collection1, Collection<String> collection2,
+ Consumer<String> action) {
+ Iterator<String> iterator = collection1.iterator();
+ while (iterator.hasNext()) {
+ String packageName = iterator.next();
+ if (!collection2.contains(packageName)) {
+ action.accept(packageName);
+ }
+ }
+ }
+
+ /**
+ * Requests an explicit health check for {@code packageName}.
+ * After this request, the callback registered on {@link #setCallbacks} can receive explicit
+ * health check passed results.
+ */
+ private void request(String packageName) {
+ synchronized (mLock) {
+ if (!prepareServiceLocked("request health check for " + packageName)) {
+ return;
+ }
+
+ Slog.i(TAG, "Requesting health check for package " + packageName);
+ try {
+ mRemoteService.request(packageName);
+ } catch (RemoteException e) {
+ Slog.w(TAG, "Failed to request health check for package " + packageName, e);
+ }
+ }
+ }
+
+ /**
+ * Cancels all explicit health checks for {@code packageName}.
+ * After this request, the callback registered on {@link #setCallbacks} can no longer receive
+ * explicit health check passed results.
+ */
+ private void cancel(String packageName) {
+ synchronized (mLock) {
+ if (!prepareServiceLocked("cancel health check for " + packageName)) {
+ return;
+ }
+
+ Slog.i(TAG, "Cancelling health check for package " + packageName);
+ try {
+ mRemoteService.cancel(packageName);
+ } catch (RemoteException e) {
+ // Do nothing, if the service is down, when it comes up, we will sync requests,
+ // if there's some other error, retrying wouldn't fix anyways.
+ Slog.w(TAG, "Failed to cancel health check for package " + packageName, e);
+ }
+ }
+ }
+
+ /**
+ * Returns the packages that we can request explicit health checks for.
+ * The packages will be returned to the {@code consumer}.
+ */
+ private void getSupportedPackages(Consumer<List<PackageConfig>> consumer) {
+ synchronized (mLock) {
+ if (!prepareServiceLocked("get health check supported packages")) {
+ return;
+ }
+
+ Slog.d(TAG, "Getting health check supported packages");
+ try {
+ mRemoteService.getSupportedPackages(new RemoteCallback(result -> {
+ List<PackageConfig> packages =
+ result.getParcelableArrayList(EXTRA_SUPPORTED_PACKAGES, android.service.watchdog.ExplicitHealthCheckService.PackageConfig.class);
+ Slog.i(TAG, "Explicit health check supported packages " + packages);
+ consumer.accept(packages);
+ }));
+ } catch (RemoteException e) {
+ // Request failed, treat as if all observed packages are supported, if any packages
+ // expire during this period, we may incorrectly treat it as failing health checks
+ // even if we don't support health checks for the package.
+ Slog.w(TAG, "Failed to get health check supported packages", e);
+ }
+ }
+ }
+
+ /**
+ * Returns the packages for which health checks are currently in progress.
+ * The packages will be returned to the {@code consumer}.
+ */
+ private void getRequestedPackages(Consumer<List<String>> consumer) {
+ synchronized (mLock) {
+ if (!prepareServiceLocked("get health check requested packages")) {
+ return;
+ }
+
+ Slog.d(TAG, "Getting health check requested packages");
+ try {
+ mRemoteService.getRequestedPackages(new RemoteCallback(result -> {
+ List<String> packages = result.getStringArrayList(EXTRA_REQUESTED_PACKAGES);
+ Slog.i(TAG, "Explicit health check requested packages " + packages);
+ consumer.accept(packages);
+ }));
+ } catch (RemoteException e) {
+ // Request failed, treat as if we haven't requested any packages, if any packages
+ // were actually requested, they will not be cancelled now. May be cancelled later
+ Slog.w(TAG, "Failed to get health check requested packages", e);
+ }
+ }
+ }
+
+ /**
+ * Binds to the explicit health check service if the controller is enabled and
+ * not already bound.
+ */
+ private void bindService() {
+ synchronized (mLock) {
+ if (!mEnabled || mConnection != null || mRemoteService != null) {
+ if (!mEnabled) {
+ Slog.i(TAG, "Not binding to service, service disabled");
+ } else if (mRemoteService != null) {
+ Slog.i(TAG, "Not binding to service, service already connected");
+ } else {
+ Slog.i(TAG, "Not binding to service, service already connecting");
+ }
+ return;
+ }
+ ComponentName component = getServiceComponentNameLocked();
+ if (component == null) {
+ Slog.wtf(TAG, "Explicit health check service not found");
+ return;
+ }
+
+ Intent intent = new Intent();
+ intent.setComponent(component);
+ mConnection = new ServiceConnection() {
+ @Override
+ public void onServiceConnected(ComponentName name, IBinder service) {
+ Slog.i(TAG, "Explicit health check service is connected " + name);
+ initState(service);
+ }
+
+ @Override
+ @MainThread
+ public void onServiceDisconnected(ComponentName name) {
+ // Service crashed or process was killed, #onServiceConnected will be called.
+ // Don't need to re-bind.
+ Slog.i(TAG, "Explicit health check service is disconnected " + name);
+ synchronized (mLock) {
+ mRemoteService = null;
+ }
+ }
+
+ @Override
+ public void onBindingDied(ComponentName name) {
+ // Application hosting service probably got updated
+ // Need to re-bind.
+ Slog.i(TAG, "Explicit health check service binding is dead. Rebind: " + name);
+ unbindService();
+ bindService();
+ }
+
+ @Override
+ public void onNullBinding(ComponentName name) {
+ // Should never happen. Service returned null from #onBind.
+ Slog.wtf(TAG, "Explicit health check service binding is null?? " + name);
+ }
+ };
+
+ mContext.bindServiceAsUser(intent, mConnection,
+ Context.BIND_AUTO_CREATE, UserHandle.SYSTEM);
+ Slog.i(TAG, "Explicit health check service is bound");
+ }
+ }
+
+ /** Unbinds the explicit health check service. */
+ private void unbindService() {
+ synchronized (mLock) {
+ if (mRemoteService != null) {
+ mContext.unbindService(mConnection);
+ mRemoteService = null;
+ mConnection = null;
+ }
+ Slog.i(TAG, "Explicit health check service is unbound");
+ }
+ }
+
+ @GuardedBy("mLock")
+ @Nullable
+ private ServiceInfo getServiceInfoLocked() {
+ final String packageName =
+ mContext.getPackageManager().getServicesSystemSharedLibraryPackageName();
+ if (packageName == null) {
+ Slog.w(TAG, "no external services package!");
+ return null;
+ }
+
+ final Intent intent = new Intent(ExplicitHealthCheckService.SERVICE_INTERFACE);
+ intent.setPackage(packageName);
+ final ResolveInfo resolveInfo = mContext.getPackageManager().resolveService(intent,
+ PackageManager.GET_SERVICES | PackageManager.GET_META_DATA);
+ if (resolveInfo == null || resolveInfo.serviceInfo == null) {
+ Slog.w(TAG, "No valid components found.");
+ return null;
+ }
+ return resolveInfo.serviceInfo;
+ }
+
+ @GuardedBy("mLock")
+ @Nullable
+ private ComponentName getServiceComponentNameLocked() {
+ final ServiceInfo serviceInfo = getServiceInfoLocked();
+ if (serviceInfo == null) {
+ return null;
+ }
+
+ final ComponentName name = new ComponentName(serviceInfo.packageName, serviceInfo.name);
+ if (!Manifest.permission.BIND_EXPLICIT_HEALTH_CHECK_SERVICE
+ .equals(serviceInfo.permission)) {
+ Slog.w(TAG, name.flattenToShortString() + " does not require permission "
+ + Manifest.permission.BIND_EXPLICIT_HEALTH_CHECK_SERVICE);
+ return null;
+ }
+ return name;
+ }
+
+ private void initState(IBinder service) {
+ synchronized (mLock) {
+ if (!mEnabled) {
+ Slog.w(TAG, "Attempting to connect disabled service?? Unbinding...");
+ // Very unlikely, but we disabled the service after binding but before we connected
+ unbindService();
+ return;
+ }
+ mRemoteService = IExplicitHealthCheckService.Stub.asInterface(service);
+ try {
+ mRemoteService.setCallback(new RemoteCallback(result -> {
+ String packageName = result.getString(EXTRA_HEALTH_CHECK_PASSED_PACKAGE);
+ if (!TextUtils.isEmpty(packageName)) {
+ if (mPassedConsumer == null) {
+ Slog.wtf(TAG, "Health check passed for package " + packageName
+ + "but no consumer registered.");
+ } else {
+ // Call without lock held
+ mPassedConsumer.accept(packageName);
+ }
+ } else {
+ Slog.wtf(TAG, "Empty package passed explicit health check?");
+ }
+ }));
+ Slog.i(TAG, "Service initialized, syncing requests");
+ } catch (RemoteException e) {
+ Slog.wtf(TAG, "Could not setCallback on explicit health check service");
+ }
+ }
+ // Calling outside lock
+ mNotifySyncRunnable.run();
+ }
+
+ /**
+ * Prepares the health check service to receive requests.
+ *
+ * @return {@code true} if it is ready and we can proceed with a request,
+ * {@code false} otherwise. If it is not ready, and the service is enabled,
+ * we will bind and the request should be automatically attempted later.
+ */
+ @GuardedBy("mLock")
+ private boolean prepareServiceLocked(String action) {
+ if (mRemoteService != null && mEnabled) {
+ return true;
+ }
+ Slog.i(TAG, "Service not ready to " + action
+ + (mEnabled ? ". Binding..." : ". Disabled"));
+ if (mEnabled) {
+ bindService();
+ }
+ return false;
+ }
+}
diff --git a/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java b/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java
new file mode 100644
index 0000000..d256aea
--- /dev/null
+++ b/packages/CrashRecovery/services/java/com/android/server/PackageWatchdog.java
@@ -0,0 +1,1792 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server;
+
+import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;
+
+import static java.lang.annotation.RetentionPolicy.SOURCE;
+
+import android.annotation.IntDef;
+import android.annotation.Nullable;
+import android.content.Context;
+import android.content.pm.PackageInfo;
+import android.content.pm.PackageManager;
+import android.content.pm.VersionedPackage;
+import android.net.ConnectivityModuleConnector;
+import android.os.Environment;
+import android.os.Handler;
+import android.os.Looper;
+import android.os.Process;
+import android.os.SystemProperties;
+import android.provider.DeviceConfig;
+import android.text.TextUtils;
+import android.util.ArrayMap;
+import android.util.ArraySet;
+import android.util.AtomicFile;
+import android.util.LongArrayQueue;
+import android.util.MathUtils;
+import android.util.Slog;
+import android.util.Xml;
+
+import com.android.internal.annotations.GuardedBy;
+import com.android.internal.annotations.VisibleForTesting;
+import com.android.internal.os.BackgroundThread;
+import com.android.internal.util.IndentingPrintWriter;
+import com.android.internal.util.XmlUtils;
+import com.android.modules.utils.TypedXmlPullParser;
+import com.android.modules.utils.TypedXmlSerializer;
+
+import libcore.io.IoUtils;
+
+import org.xmlpull.v1.XmlPullParserException;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Monitors the health of packages on the system and notifies interested observers when packages
+ * fail. On failure, the registered observer with the least user impacting mitigation will
+ * be notified.
+ */
+public class PackageWatchdog {
+ private static final String TAG = "PackageWatchdog";
+
+ static final String PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS =
+ "watchdog_trigger_failure_duration_millis";
+ static final String PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT =
+ "watchdog_trigger_failure_count";
+ static final String PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED =
+ "watchdog_explicit_health_check_enabled";
+
+ // TODO: make the following values configurable via DeviceConfig
+ private static final long NATIVE_CRASH_POLLING_INTERVAL_MILLIS =
+ TimeUnit.SECONDS.toMillis(30);
+ private static final long NUMBER_OF_NATIVE_CRASH_POLLS = 10;
+
+
+ public static final int FAILURE_REASON_UNKNOWN = 0;
+ public static final int FAILURE_REASON_NATIVE_CRASH = 1;
+ public static final int FAILURE_REASON_EXPLICIT_HEALTH_CHECK = 2;
+ public static final int FAILURE_REASON_APP_CRASH = 3;
+ public static final int FAILURE_REASON_APP_NOT_RESPONDING = 4;
+
+ @IntDef(prefix = { "FAILURE_REASON_" }, value = {
+ FAILURE_REASON_UNKNOWN,
+ FAILURE_REASON_NATIVE_CRASH,
+ FAILURE_REASON_EXPLICIT_HEALTH_CHECK,
+ FAILURE_REASON_APP_CRASH,
+ FAILURE_REASON_APP_NOT_RESPONDING
+ })
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface FailureReasons {}
+
+ // Duration to count package failures before it resets to 0
+ @VisibleForTesting
+ static final int DEFAULT_TRIGGER_FAILURE_DURATION_MS =
+ (int) TimeUnit.MINUTES.toMillis(1);
+ // Number of package failures within the duration above before we notify observers
+ @VisibleForTesting
+ static final int DEFAULT_TRIGGER_FAILURE_COUNT = 5;
+ @VisibleForTesting
+ static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2);
+ // Sliding window for tracking how many mitigation calls were made for a package.
+ @VisibleForTesting
+ static final long DEFAULT_DEESCALATION_WINDOW_MS = TimeUnit.HOURS.toMillis(1);
+ // Whether explicit health checks are enabled or not
+ private static final boolean DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED = true;
+
+ @VisibleForTesting
+ static final int DEFAULT_BOOT_LOOP_TRIGGER_COUNT = 5;
+ static final long DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS = TimeUnit.MINUTES.toMillis(10);
+
+ // These properties track individual system server boot events, and are reset once the boot
+ // threshold is met, or the boot loop trigger window is exceeded between boot events.
+ private static final String PROP_RESCUE_BOOT_COUNT = "sys.rescue_boot_count";
+ private static final String PROP_RESCUE_BOOT_START = "sys.rescue_boot_start";
+
+ // These properties track multiple calls made to observers tracking boot loops. They are reset
+ // when the de-escalation window is exceeded between boot events.
+ private static final String PROP_BOOT_MITIGATION_WINDOW_START = "sys.boot_mitigation_start";
+ private static final String PROP_BOOT_MITIGATION_COUNT = "sys.boot_mitigation_count";
+
+ private long mNumberOfNativeCrashPollsRemaining;
+
+ private static final int DB_VERSION = 1;
+ private static final String TAG_PACKAGE_WATCHDOG = "package-watchdog";
+ private static final String TAG_PACKAGE = "package";
+ private static final String TAG_OBSERVER = "observer";
+ private static final String ATTR_VERSION = "version";
+ private static final String ATTR_NAME = "name";
+ private static final String ATTR_DURATION = "duration";
+ private static final String ATTR_EXPLICIT_HEALTH_CHECK_DURATION = "health-check-duration";
+ private static final String ATTR_PASSED_HEALTH_CHECK = "passed-health-check";
+ private static final String ATTR_MITIGATION_CALLS = "mitigation-calls";
+
+ // A file containing information about the current mitigation count in the case of a boot loop.
+ // This allows boot loop information to persist in the case of an fs-checkpoint being
+ // aborted.
+ private static final String METADATA_FILE = "/metadata/watchdog/mitigation_count.txt";
+
+ @GuardedBy("PackageWatchdog.class")
+ private static PackageWatchdog sPackageWatchdog;
+
+ private final Object mLock = new Object();
+ // System server context
+ private final Context mContext;
+ // Handler to run short running tasks
+ private final Handler mShortTaskHandler;
+ // Handler for processing IO and long running tasks
+ private final Handler mLongTaskHandler;
+ // Contains (observer-name -> observer-handle) that have ever been registered from
+ // previous boots. Observers with all packages expired are periodically pruned.
+ // It is saved to disk on system shutdown and repouplated on startup so it survives reboots.
+ @GuardedBy("mLock")
+ private final ArrayMap<String, ObserverInternal> mAllObservers = new ArrayMap<>();
+ // File containing the XML data of monitored packages /data/system/package-watchdog.xml
+ private final AtomicFile mPolicyFile;
+ private final ExplicitHealthCheckController mHealthCheckController;
+ private final ConnectivityModuleConnector mConnectivityModuleConnector;
+ private final Runnable mSyncRequests = this::syncRequests;
+ private final Runnable mSyncStateWithScheduledReason = this::syncStateWithScheduledReason;
+ private final Runnable mSaveToFile = this::saveToFile;
+ private final SystemClock mSystemClock;
+ private final BootThreshold mBootThreshold;
+ private final DeviceConfig.OnPropertiesChangedListener
+ mOnPropertyChangedListener = this::onPropertyChanged;
+
+ // The set of packages that have been synced with the ExplicitHealthCheckController
+ @GuardedBy("mLock")
+ private Set<String> mRequestedHealthCheckPackages = new ArraySet<>();
+ @GuardedBy("mLock")
+ private boolean mIsPackagesReady;
+ // Flag to control whether explicit health checks are supported or not
+ @GuardedBy("mLock")
+ private boolean mIsHealthCheckEnabled = DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED;
+ @GuardedBy("mLock")
+ private int mTriggerFailureDurationMs = DEFAULT_TRIGGER_FAILURE_DURATION_MS;
+ @GuardedBy("mLock")
+ private int mTriggerFailureCount = DEFAULT_TRIGGER_FAILURE_COUNT;
+ // SystemClock#uptimeMillis when we last executed #syncState
+ // 0 if no prune is scheduled.
+ @GuardedBy("mLock")
+ private long mUptimeAtLastStateSync;
+ // If true, sync explicit health check packages with the ExplicitHealthCheckController.
+ @GuardedBy("mLock")
+ private boolean mSyncRequired = false;
+
+ @FunctionalInterface
+ @VisibleForTesting
+ interface SystemClock {
+ long uptimeMillis();
+ }
+
+ private PackageWatchdog(Context context) {
+ // Needs to be constructed inline
+ this(context, new AtomicFile(
+ new File(new File(Environment.getDataDirectory(), "system"),
+ "package-watchdog.xml")),
+ new Handler(Looper.myLooper()), BackgroundThread.getHandler(),
+ new ExplicitHealthCheckController(context),
+ ConnectivityModuleConnector.getInstance(),
+ android.os.SystemClock::uptimeMillis);
+ }
+
+ /**
+ * Creates a PackageWatchdog that allows injecting dependencies.
+ */
+ @VisibleForTesting
+ PackageWatchdog(Context context, AtomicFile policyFile, Handler shortTaskHandler,
+ Handler longTaskHandler, ExplicitHealthCheckController controller,
+ ConnectivityModuleConnector connectivityModuleConnector, SystemClock clock) {
+ mContext = context;
+ mPolicyFile = policyFile;
+ mShortTaskHandler = shortTaskHandler;
+ mLongTaskHandler = longTaskHandler;
+ mHealthCheckController = controller;
+ mConnectivityModuleConnector = connectivityModuleConnector;
+ mSystemClock = clock;
+ mNumberOfNativeCrashPollsRemaining = NUMBER_OF_NATIVE_CRASH_POLLS;
+ mBootThreshold = new BootThreshold(DEFAULT_BOOT_LOOP_TRIGGER_COUNT,
+ DEFAULT_BOOT_LOOP_TRIGGER_WINDOW_MS);
+ loadFromFile();
+ sPackageWatchdog = this;
+ }
+
+ /** Creates or gets singleton instance of PackageWatchdog. */
+ public static PackageWatchdog getInstance(Context context) {
+ synchronized (PackageWatchdog.class) {
+ if (sPackageWatchdog == null) {
+ new PackageWatchdog(context);
+ }
+ return sPackageWatchdog;
+ }
+ }
+
+ /**
+ * Called during boot to notify when packages are ready on the device so we can start
+ * binding.
+ */
+ public void onPackagesReady() {
+ synchronized (mLock) {
+ mIsPackagesReady = true;
+ mHealthCheckController.setCallbacks(packageName -> onHealthCheckPassed(packageName),
+ packages -> onSupportedPackages(packages),
+ this::onSyncRequestNotified);
+ setPropertyChangedListenerLocked();
+ updateConfigs();
+ registerConnectivityModuleHealthListener();
+ }
+ }
+
+ /**
+ * Registers {@code observer} to listen for package failures. Add a new ObserverInternal for
+ * this observer if it does not already exist.
+ *
+ * <p>Observers are expected to call this on boot. It does not specify any packages but
+ * it will resume observing any packages requested from a previous boot.
+ */
+ public void registerHealthObserver(PackageHealthObserver observer) {
+ synchronized (mLock) {
+ ObserverInternal internalObserver = mAllObservers.get(observer.getName());
+ if (internalObserver != null) {
+ internalObserver.registeredObserver = observer;
+ } else {
+ internalObserver = new ObserverInternal(observer.getName(), new ArrayList<>());
+ internalObserver.registeredObserver = observer;
+ mAllObservers.put(observer.getName(), internalObserver);
+ syncState("added new observer");
+ }
+ }
+ }
+
+ /**
+ * Starts observing the health of the {@code packages} for {@code observer} and notifies
+ * {@code observer} of any package failures within the monitoring duration.
+ *
+ * <p>If monitoring a package supporting explicit health check, at the end of the monitoring
+ * duration if {@link #onHealthCheckPassed} was never called,
+ * {@link PackageHealthObserver#execute} will be called as if the package failed.
+ *
+ * <p>If {@code observer} is already monitoring a package in {@code packageNames},
+ * the monitoring window of that package will be reset to {@code durationMs} and the health
+ * check state will be reset to a default depending on if the package is contained in
+ * {@link mPackagesWithExplicitHealthCheckEnabled}.
+ *
+ * <p>If {@code packageNames} is empty, this will be a no-op.
+ *
+ * <p>If {@code durationMs} is less than 1, a default monitoring duration
+ * {@link #DEFAULT_OBSERVING_DURATION_MS} will be used.
+ */
+ public void startObservingHealth(PackageHealthObserver observer, List<String> packageNames,
+ long durationMs) {
+ if (packageNames.isEmpty()) {
+ Slog.wtf(TAG, "No packages to observe, " + observer.getName());
+ return;
+ }
+ if (durationMs < 1) {
+ Slog.wtf(TAG, "Invalid duration " + durationMs + "ms for observer "
+ + observer.getName() + ". Not observing packages " + packageNames);
+ durationMs = DEFAULT_OBSERVING_DURATION_MS;
+ }
+
+ List<MonitoredPackage> packages = new ArrayList<>();
+ for (int i = 0; i < packageNames.size(); i++) {
+ // Health checks not available yet so health check state will start INACTIVE
+ MonitoredPackage pkg = newMonitoredPackage(packageNames.get(i), durationMs, false);
+ if (pkg != null) {
+ packages.add(pkg);
+ } else {
+ Slog.w(TAG, "Failed to create MonitoredPackage for pkg=" + packageNames.get(i));
+ }
+ }
+
+ if (packages.isEmpty()) {
+ return;
+ }
+
+ // Sync before we add the new packages to the observers. This will #pruneObservers,
+ // causing any elapsed time to be deducted from all existing packages before we add new
+ // packages. This maintains the invariant that the elapsed time for ALL (new and existing)
+ // packages is the same.
+ mLongTaskHandler.post(() -> {
+ syncState("observing new packages");
+
+ synchronized (mLock) {
+ ObserverInternal oldObserver = mAllObservers.get(observer.getName());
+ if (oldObserver == null) {
+ Slog.d(TAG, observer.getName() + " started monitoring health "
+ + "of packages " + packageNames);
+ mAllObservers.put(observer.getName(),
+ new ObserverInternal(observer.getName(), packages));
+ } else {
+ Slog.d(TAG, observer.getName() + " added the following "
+ + "packages to monitor " + packageNames);
+ oldObserver.updatePackagesLocked(packages);
+ }
+ }
+
+ // Register observer in case not already registered
+ registerHealthObserver(observer);
+
+ // Sync after we add the new packages to the observers. We may have received packges
+ // requiring an earlier schedule than we are currently scheduled for.
+ syncState("updated observers");
+ });
+
+ }
+
+ /**
+ * Unregisters {@code observer} from listening to package failure.
+ * Additionally, this stops observing any packages that may have previously been observed
+ * even from a previous boot.
+ */
+ public void unregisterHealthObserver(PackageHealthObserver observer) {
+ mLongTaskHandler.post(() -> {
+ synchronized (mLock) {
+ mAllObservers.remove(observer.getName());
+ }
+ syncState("unregistering observer: " + observer.getName());
+ });
+ }
+
+ /**
+ * Called when a process fails due to a crash, ANR or explicit health check.
+ *
+ * <p>For each package contained in the process, one registered observer with the least user
+ * impact will be notified for mitigation.
+ *
+ * <p>This method could be called frequently if there is a severe problem on the device.
+ */
+ public void onPackageFailure(List<VersionedPackage> packages,
+ @FailureReasons int failureReason) {
+ if (packages == null) {
+ Slog.w(TAG, "Could not resolve a list of failing packages");
+ return;
+ }
+ mLongTaskHandler.post(() -> {
+ synchronized (mLock) {
+ if (mAllObservers.isEmpty()) {
+ return;
+ }
+ boolean requiresImmediateAction = (failureReason == FAILURE_REASON_NATIVE_CRASH
+ || failureReason == FAILURE_REASON_EXPLICIT_HEALTH_CHECK);
+ if (requiresImmediateAction) {
+ handleFailureImmediately(packages, failureReason);
+ } else {
+ for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
+ VersionedPackage versionedPackage = packages.get(pIndex);
+ // Observer that will receive failure for versionedPackage
+ PackageHealthObserver currentObserverToNotify = null;
+ int currentObserverImpact = Integer.MAX_VALUE;
+ MonitoredPackage currentMonitoredPackage = null;
+
+ // Find observer with least user impact
+ for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
+ ObserverInternal observer = mAllObservers.valueAt(oIndex);
+ PackageHealthObserver registeredObserver = observer.registeredObserver;
+ if (registeredObserver != null
+ && observer.onPackageFailureLocked(
+ versionedPackage.getPackageName())) {
+ MonitoredPackage p = observer.getMonitoredPackage(
+ versionedPackage.getPackageName());
+ int mitigationCount = 1;
+ if (p != null) {
+ mitigationCount = p.getMitigationCountLocked() + 1;
+ }
+ int impact = registeredObserver.onHealthCheckFailed(
+ versionedPackage, failureReason, mitigationCount);
+ if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0
+ && impact < currentObserverImpact) {
+ currentObserverToNotify = registeredObserver;
+ currentObserverImpact = impact;
+ currentMonitoredPackage = p;
+ }
+ }
+ }
+
+ // Execute action with least user impact
+ if (currentObserverToNotify != null) {
+ int mitigationCount = 1;
+ if (currentMonitoredPackage != null) {
+ currentMonitoredPackage.noteMitigationCallLocked();
+ mitigationCount =
+ currentMonitoredPackage.getMitigationCountLocked();
+ }
+ currentObserverToNotify.execute(versionedPackage,
+ failureReason, mitigationCount);
+ }
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * For native crashes or explicit health check failures, call directly into each observer to
+ * mitigate the error without going through failure threshold logic.
+ */
+ private void handleFailureImmediately(List<VersionedPackage> packages,
+ @FailureReasons int failureReason) {
+ VersionedPackage failingPackage = packages.size() > 0 ? packages.get(0) : null;
+ PackageHealthObserver currentObserverToNotify = null;
+ int currentObserverImpact = Integer.MAX_VALUE;
+ for (ObserverInternal observer: mAllObservers.values()) {
+ PackageHealthObserver registeredObserver = observer.registeredObserver;
+ if (registeredObserver != null) {
+ int impact = registeredObserver.onHealthCheckFailed(
+ failingPackage, failureReason, 1);
+ if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0
+ && impact < currentObserverImpact) {
+ currentObserverToNotify = registeredObserver;
+ currentObserverImpact = impact;
+ }
+ }
+ }
+ if (currentObserverToNotify != null) {
+ currentObserverToNotify.execute(failingPackage, failureReason, 1);
+ }
+ }
+
+ /**
+ * Called when the system server boots. If the system server is detected to be in a boot loop,
+ * query each observer and perform the mitigation action with the lowest user impact.
+ */
+ public void noteBoot() {
+ synchronized (mLock) {
+ if (mBootThreshold.incrementAndTest()) {
+ mBootThreshold.reset();
+ int mitigationCount = mBootThreshold.getMitigationCount() + 1;
+ PackageHealthObserver currentObserverToNotify = null;
+ int currentObserverImpact = Integer.MAX_VALUE;
+ for (int i = 0; i < mAllObservers.size(); i++) {
+ final ObserverInternal observer = mAllObservers.valueAt(i);
+ PackageHealthObserver registeredObserver = observer.registeredObserver;
+ if (registeredObserver != null) {
+ int impact = registeredObserver.onBootLoop(mitigationCount);
+ if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0
+ && impact < currentObserverImpact) {
+ currentObserverToNotify = registeredObserver;
+ currentObserverImpact = impact;
+ }
+ }
+ }
+ if (currentObserverToNotify != null) {
+ mBootThreshold.setMitigationCount(mitigationCount);
+ mBootThreshold.saveMitigationCountToMetadata();
+ currentObserverToNotify.executeBootLoopMitigation(mitigationCount);
+ }
+ }
+ }
+ }
+
+ // TODO(b/120598832): Optimize write? Maybe only write a separate smaller file? Also
+ // avoid holding lock?
+ // This currently adds about 7ms extra to shutdown thread
+ /** Writes the package information to file during shutdown. */
+ public void writeNow() {
+ synchronized (mLock) {
+ // Must only run synchronous tasks as this runs on the ShutdownThread and no other
+ // thread is guaranteed to run during shutdown.
+ if (!mAllObservers.isEmpty()) {
+ mLongTaskHandler.removeCallbacks(mSaveToFile);
+ pruneObserversLocked();
+ saveToFile();
+ Slog.i(TAG, "Last write to update package durations");
+ }
+ }
+ }
+
+ /**
+ * Enables or disables explicit health checks.
+ * <p> If explicit health checks are enabled, the health check service is started.
+ * <p> If explicit health checks are disabled, pending explicit health check requests are
+ * passed and the health check service is stopped.
+ */
+ private void setExplicitHealthCheckEnabled(boolean enabled) {
+ synchronized (mLock) {
+ mIsHealthCheckEnabled = enabled;
+ mHealthCheckController.setEnabled(enabled);
+ mSyncRequired = true;
+ // Prune to update internal state whenever health check is enabled/disabled
+ syncState("health check state " + (enabled ? "enabled" : "disabled"));
+ }
+ }
+
+ /**
+ * This method should be only called on mShortTaskHandler, since it modifies
+ * {@link #mNumberOfNativeCrashPollsRemaining}.
+ */
+ private void checkAndMitigateNativeCrashes() {
+ mNumberOfNativeCrashPollsRemaining--;
+ // Check if native watchdog reported a crash
+ if ("1".equals(SystemProperties.get("sys.init.updatable_crashing"))) {
+ // We rollback everything available when crash is unattributable
+ onPackageFailure(Collections.EMPTY_LIST, FAILURE_REASON_NATIVE_CRASH);
+ // we stop polling after an attempt to execute rollback, regardless of whether the
+ // attempt succeeds or not
+ } else {
+ if (mNumberOfNativeCrashPollsRemaining > 0) {
+ mShortTaskHandler.postDelayed(() -> checkAndMitigateNativeCrashes(),
+ NATIVE_CRASH_POLLING_INTERVAL_MILLIS);
+ }
+ }
+ }
+
+ /**
+ * Since this method can eventually trigger a rollback, it should be called
+ * only once boot has completed {@code onBootCompleted} and not earlier, because the install
+ * session must be entirely completed before we try to rollback.
+ */
+ public void scheduleCheckAndMitigateNativeCrashes() {
+ Slog.i(TAG, "Scheduling " + mNumberOfNativeCrashPollsRemaining + " polls to check "
+ + "and mitigate native crashes");
+ mShortTaskHandler.post(()->checkAndMitigateNativeCrashes());
+ }
+
+ /** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */
+ @Retention(SOURCE)
+ @IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_LEVEL_0,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_10,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_30,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_50,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_70,
+ PackageHealthObserverImpact.USER_IMPACT_LEVEL_100})
+ public @interface PackageHealthObserverImpact {
+ /** No action to take. */
+ int USER_IMPACT_LEVEL_0 = 0;
+ /* Action has low user impact, user of a device will barely notice. */
+ int USER_IMPACT_LEVEL_10 = 10;
+ /* Actions having medium user impact, user of a device will likely notice. */
+ int USER_IMPACT_LEVEL_30 = 30;
+ int USER_IMPACT_LEVEL_50 = 50;
+ int USER_IMPACT_LEVEL_70 = 70;
+ /* Action has high user impact, a last resort, user of a device will be very frustrated. */
+ int USER_IMPACT_LEVEL_100 = 100;
+ }
+
+ /** Register instances of this interface to receive notifications on package failure. */
+ public interface PackageHealthObserver {
+ /**
+ * Called when health check fails for the {@code versionedPackage}.
+ *
+ * @param versionedPackage the package that is failing. This may be null if a native
+ * service is crashing.
+ * @param failureReason the type of failure that is occurring.
+ * @param mitigationCount the number of times mitigation has been called for this package
+ * (including this time).
+ *
+ *
+ * @return any one of {@link PackageHealthObserverImpact} to express the impact
+ * to the user on {@link #execute}
+ */
+ @PackageHealthObserverImpact int onHealthCheckFailed(
+ @Nullable VersionedPackage versionedPackage,
+ @FailureReasons int failureReason,
+ int mitigationCount);
+
+ /**
+ * Executes mitigation for {@link #onHealthCheckFailed}.
+ *
+ * @param versionedPackage the package that is failing. This may be null if a native
+ * service is crashing.
+ * @param failureReason the type of failure that is occurring.
+ * @param mitigationCount the number of times mitigation has been called for this package
+ * (including this time).
+ * @return {@code true} if action was executed successfully, {@code false} otherwise
+ */
+ boolean execute(@Nullable VersionedPackage versionedPackage,
+ @FailureReasons int failureReason, int mitigationCount);
+
+
+ /**
+ * Called when the system server has booted several times within a window of time, defined
+ * by {@link #mBootThreshold}
+ *
+ * @param mitigationCount the number of times mitigation has been attempted for this
+ * boot loop (including this time).
+ */
+ default @PackageHealthObserverImpact int onBootLoop(int mitigationCount) {
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ }
+
+ /**
+ * Executes mitigation for {@link #onBootLoop}
+ * @param mitigationCount the number of times mitigation has been attempted for this
+ * boot loop (including this time).
+ */
+ default boolean executeBootLoopMitigation(int mitigationCount) {
+ return false;
+ }
+
+ // TODO(b/120598832): Ensure uniqueness?
+ /**
+ * Identifier for the observer, should not change across device updates otherwise the
+ * watchdog may drop observing packages with the old name.
+ */
+ String getName();
+
+ /**
+ * An observer will not be pruned if this is set, even if the observer is not explicitly
+ * monitoring any packages.
+ */
+ default boolean isPersistent() {
+ return false;
+ }
+
+ /**
+ * Returns {@code true} if this observer wishes to observe the given package, {@code false}
+ * otherwise
+ *
+ * <p> A persistent observer may choose to start observing certain failing packages, even if
+ * it has not explicitly asked to watch the package with {@link #startObservingHealth}.
+ */
+ default boolean mayObservePackage(String packageName) {
+ return false;
+ }
+ }
+
+ @VisibleForTesting
+ long getTriggerFailureCount() {
+ synchronized (mLock) {
+ return mTriggerFailureCount;
+ }
+ }
+
+ @VisibleForTesting
+ long getTriggerFailureDurationMs() {
+ synchronized (mLock) {
+ return mTriggerFailureDurationMs;
+ }
+ }
+
+ /**
+ * Serializes and syncs health check requests with the {@link ExplicitHealthCheckController}.
+ */
+ private void syncRequestsAsync() {
+ mShortTaskHandler.removeCallbacks(mSyncRequests);
+ mShortTaskHandler.post(mSyncRequests);
+ }
+
+ /**
+ * Syncs health check requests with the {@link ExplicitHealthCheckController}.
+ * Calls to this must be serialized.
+ *
+ * @see #syncRequestsAsync
+ */
+ private void syncRequests() {
+ boolean syncRequired = false;
+ synchronized (mLock) {
+ if (mIsPackagesReady) {
+ Set<String> packages = getPackagesPendingHealthChecksLocked();
+ if (mSyncRequired || !packages.equals(mRequestedHealthCheckPackages)
+ || packages.isEmpty()) {
+ syncRequired = true;
+ mRequestedHealthCheckPackages = packages;
+ }
+ } // else, we will sync requests when packages become ready
+ }
+
+ // Call outside lock to avoid holding lock when calling into the controller.
+ if (syncRequired) {
+ Slog.i(TAG, "Syncing health check requests for packages: "
+ + mRequestedHealthCheckPackages);
+ mHealthCheckController.syncRequests(mRequestedHealthCheckPackages);
+ mSyncRequired = false;
+ }
+ }
+
+ /**
+ * Updates the observers monitoring {@code packageName} that explicit health check has passed.
+ *
+ * <p> This update is strictly for registered observers at the time of the call
+ * Observers that register after this signal will have no knowledge of prior signals and will
+ * effectively behave as if the explicit health check hasn't passed for {@code packageName}.
+ *
+ * <p> {@code packageName} can still be considered failed if reported by
+ * {@link #onPackageFailureLocked} before the package expires.
+ *
+ * <p> Triggered by components outside the system server when they are fully functional after an
+ * update.
+ */
+ private void onHealthCheckPassed(String packageName) {
+ Slog.i(TAG, "Health check passed for package: " + packageName);
+ boolean isStateChanged = false;
+
+ synchronized (mLock) {
+ for (int observerIdx = 0; observerIdx < mAllObservers.size(); observerIdx++) {
+ ObserverInternal observer = mAllObservers.valueAt(observerIdx);
+ MonitoredPackage monitoredPackage = observer.getMonitoredPackage(packageName);
+
+ if (monitoredPackage != null) {
+ int oldState = monitoredPackage.getHealthCheckStateLocked();
+ int newState = monitoredPackage.tryPassHealthCheckLocked();
+ isStateChanged |= oldState != newState;
+ }
+ }
+ }
+
+ if (isStateChanged) {
+ syncState("health check passed for " + packageName);
+ }
+ }
+
+ private void onSupportedPackages(List<PackageConfig> supportedPackages) {
+ boolean isStateChanged = false;
+
+ Map<String, Long> supportedPackageTimeouts = new ArrayMap<>();
+ Iterator<PackageConfig> it = supportedPackages.iterator();
+ while (it.hasNext()) {
+ PackageConfig info = it.next();
+ supportedPackageTimeouts.put(info.getPackageName(), info.getHealthCheckTimeoutMillis());
+ }
+
+ synchronized (mLock) {
+ Slog.d(TAG, "Received supported packages " + supportedPackages);
+ Iterator<ObserverInternal> oit = mAllObservers.values().iterator();
+ while (oit.hasNext()) {
+ Iterator<MonitoredPackage> pit = oit.next().getMonitoredPackages()
+ .values().iterator();
+ while (pit.hasNext()) {
+ MonitoredPackage monitoredPackage = pit.next();
+ String packageName = monitoredPackage.getName();
+ int oldState = monitoredPackage.getHealthCheckStateLocked();
+ int newState;
+
+ if (supportedPackageTimeouts.containsKey(packageName)) {
+ // Supported packages become ACTIVE if currently INACTIVE
+ newState = monitoredPackage.setHealthCheckActiveLocked(
+ supportedPackageTimeouts.get(packageName));
+ } else {
+ // Unsupported packages are marked as PASSED unless already FAILED
+ newState = monitoredPackage.tryPassHealthCheckLocked();
+ }
+ isStateChanged |= oldState != newState;
+ }
+ }
+ }
+
+ if (isStateChanged) {
+ syncState("updated health check supported packages " + supportedPackages);
+ }
+ }
+
+ private void onSyncRequestNotified() {
+ synchronized (mLock) {
+ mSyncRequired = true;
+ syncRequestsAsync();
+ }
+ }
+
+ @GuardedBy("mLock")
+ private Set<String> getPackagesPendingHealthChecksLocked() {
+ Set<String> packages = new ArraySet<>();
+ Iterator<ObserverInternal> oit = mAllObservers.values().iterator();
+ while (oit.hasNext()) {
+ ObserverInternal observer = oit.next();
+ Iterator<MonitoredPackage> pit =
+ observer.getMonitoredPackages().values().iterator();
+ while (pit.hasNext()) {
+ MonitoredPackage monitoredPackage = pit.next();
+ String packageName = monitoredPackage.getName();
+ if (monitoredPackage.isPendingHealthChecksLocked()) {
+ packages.add(packageName);
+ }
+ }
+ }
+ return packages;
+ }
+
+ /**
+ * Syncs the state of the observers.
+ *
+ * <p> Prunes all observers, saves new state to disk, syncs health check requests with the
+ * health check service and schedules the next state sync.
+ */
+ private void syncState(String reason) {
+ synchronized (mLock) {
+ Slog.i(TAG, "Syncing state, reason: " + reason);
+ pruneObserversLocked();
+
+ saveToFileAsync();
+ syncRequestsAsync();
+
+ // Done syncing state, schedule the next state sync
+ scheduleNextSyncStateLocked();
+ }
+ }
+
+ private void syncStateWithScheduledReason() {
+ syncState("scheduled");
+ }
+
+ @GuardedBy("mLock")
+ private void scheduleNextSyncStateLocked() {
+ long durationMs = getNextStateSyncMillisLocked();
+ mShortTaskHandler.removeCallbacks(mSyncStateWithScheduledReason);
+ if (durationMs == Long.MAX_VALUE) {
+ Slog.i(TAG, "Cancelling state sync, nothing to sync");
+ mUptimeAtLastStateSync = 0;
+ } else {
+ mUptimeAtLastStateSync = mSystemClock.uptimeMillis();
+ mShortTaskHandler.postDelayed(mSyncStateWithScheduledReason, durationMs);
+ }
+ }
+
+ /**
+ * Returns the next duration in millis to sync the watchdog state.
+ *
+ * @returns Long#MAX_VALUE if there are no observed packages.
+ */
+ @GuardedBy("mLock")
+ private long getNextStateSyncMillisLocked() {
+ long shortestDurationMs = Long.MAX_VALUE;
+ for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
+ ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex)
+ .getMonitoredPackages();
+ for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
+ MonitoredPackage mp = packages.valueAt(pIndex);
+ long duration = mp.getShortestScheduleDurationMsLocked();
+ if (duration < shortestDurationMs) {
+ shortestDurationMs = duration;
+ }
+ }
+ }
+ return shortestDurationMs;
+ }
+
+ /**
+ * Removes {@code elapsedMs} milliseconds from all durations on monitored packages
+ * and updates other internal state.
+ */
+ @GuardedBy("mLock")
+ private void pruneObserversLocked() {
+ long elapsedMs = mUptimeAtLastStateSync == 0
+ ? 0 : mSystemClock.uptimeMillis() - mUptimeAtLastStateSync;
+ if (elapsedMs <= 0) {
+ Slog.i(TAG, "Not pruning observers, elapsed time: " + elapsedMs + "ms");
+ return;
+ }
+
+ Iterator<ObserverInternal> it = mAllObservers.values().iterator();
+ while (it.hasNext()) {
+ ObserverInternal observer = it.next();
+ Set<MonitoredPackage> failedPackages =
+ observer.prunePackagesLocked(elapsedMs);
+ if (!failedPackages.isEmpty()) {
+ onHealthCheckFailed(observer, failedPackages);
+ }
+ if (observer.getMonitoredPackages().isEmpty() && (observer.registeredObserver == null
+ || !observer.registeredObserver.isPersistent())) {
+ Slog.i(TAG, "Discarding observer " + observer.name + ". All packages expired");
+ it.remove();
+ }
+ }
+ }
+
+ private void onHealthCheckFailed(ObserverInternal observer,
+ Set<MonitoredPackage> failedPackages) {
+ mLongTaskHandler.post(() -> {
+ synchronized (mLock) {
+ PackageHealthObserver registeredObserver = observer.registeredObserver;
+ if (registeredObserver != null) {
+ Iterator<MonitoredPackage> it = failedPackages.iterator();
+ while (it.hasNext()) {
+ VersionedPackage versionedPkg = getVersionedPackage(it.next().getName());
+ if (versionedPkg != null) {
+ Slog.i(TAG,
+ "Explicit health check failed for package " + versionedPkg);
+ registeredObserver.execute(versionedPkg,
+ PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK, 1);
+ }
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * Gets PackageInfo for the given package. Matches any user and apex.
+ *
+ * @throws PackageManager.NameNotFoundException if no such package is installed.
+ */
+ private PackageInfo getPackageInfo(String packageName)
+ throws PackageManager.NameNotFoundException {
+ PackageManager pm = mContext.getPackageManager();
+ try {
+ // The MATCH_ANY_USER flag doesn't mix well with the MATCH_APEX
+ // flag, so make two separate attempts to get the package info.
+ // We don't need both flags at the same time because we assume
+ // apex files are always installed for all users.
+ return pm.getPackageInfo(packageName, PackageManager.MATCH_ANY_USER);
+ } catch (PackageManager.NameNotFoundException e) {
+ return pm.getPackageInfo(packageName, PackageManager.MATCH_APEX);
+ }
+ }
+
+ @Nullable
+ private VersionedPackage getVersionedPackage(String packageName) {
+ final PackageManager pm = mContext.getPackageManager();
+ if (pm == null || TextUtils.isEmpty(packageName)) {
+ return null;
+ }
+ try {
+ final long versionCode = getPackageInfo(packageName).getLongVersionCode();
+ return new VersionedPackage(packageName, versionCode);
+ } catch (PackageManager.NameNotFoundException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Loads mAllObservers from file.
+ *
+ * <p>Note that this is <b>not</b> thread safe and should only called be called
+ * from the constructor.
+ */
+ private void loadFromFile() {
+ InputStream infile = null;
+ mAllObservers.clear();
+ try {
+ infile = mPolicyFile.openRead();
+ final TypedXmlPullParser parser = Xml.resolvePullParser(infile);
+ XmlUtils.beginDocument(parser, TAG_PACKAGE_WATCHDOG);
+ int outerDepth = parser.getDepth();
+ while (XmlUtils.nextElementWithin(parser, outerDepth)) {
+ ObserverInternal observer = ObserverInternal.read(parser, this);
+ if (observer != null) {
+ mAllObservers.put(observer.name, observer);
+ }
+ }
+ } catch (FileNotFoundException e) {
+ // Nothing to monitor
+ } catch (IOException | NumberFormatException | XmlPullParserException e) {
+ Slog.wtf(TAG, "Unable to read monitored packages, deleting file", e);
+ mPolicyFile.delete();
+ } finally {
+ IoUtils.closeQuietly(infile);
+ }
+ }
+
+ private void onPropertyChanged(DeviceConfig.Properties properties) {
+ try {
+ updateConfigs();
+ } catch (Exception ignore) {
+ Slog.w(TAG, "Failed to reload device config changes");
+ }
+ }
+
+ /** Adds a {@link DeviceConfig#OnPropertiesChangedListener}. */
+ private void setPropertyChangedListenerLocked() {
+ DeviceConfig.addOnPropertiesChangedListener(
+ DeviceConfig.NAMESPACE_ROLLBACK,
+ mContext.getMainExecutor(),
+ mOnPropertyChangedListener);
+ }
+
+ @VisibleForTesting
+ void removePropertyChangedListener() {
+ DeviceConfig.removeOnPropertiesChangedListener(mOnPropertyChangedListener);
+ }
+
+ /**
+ * Health check is enabled or disabled after reading the flags
+ * from DeviceConfig.
+ */
+ @VisibleForTesting
+ void updateConfigs() {
+ synchronized (mLock) {
+ mTriggerFailureCount = DeviceConfig.getInt(
+ DeviceConfig.NAMESPACE_ROLLBACK,
+ PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
+ DEFAULT_TRIGGER_FAILURE_COUNT);
+ if (mTriggerFailureCount <= 0) {
+ mTriggerFailureCount = DEFAULT_TRIGGER_FAILURE_COUNT;
+ }
+
+ mTriggerFailureDurationMs = DeviceConfig.getInt(
+ DeviceConfig.NAMESPACE_ROLLBACK,
+ PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS,
+ DEFAULT_TRIGGER_FAILURE_DURATION_MS);
+ if (mTriggerFailureDurationMs <= 0) {
+ mTriggerFailureDurationMs = DEFAULT_TRIGGER_FAILURE_DURATION_MS;
+ }
+
+ setExplicitHealthCheckEnabled(DeviceConfig.getBoolean(
+ DeviceConfig.NAMESPACE_ROLLBACK,
+ PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED,
+ DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED));
+ }
+ }
+
+ private void registerConnectivityModuleHealthListener() {
+ // TODO: have an internal method to trigger a rollback by reporting high severity errors,
+ // and rely on ActivityManager to inform the watchdog of severe network stack crashes
+ // instead of having this listener in parallel.
+ mConnectivityModuleConnector.registerHealthListener(
+ packageName -> {
+ final VersionedPackage pkg = getVersionedPackage(packageName);
+ if (pkg == null) {
+ Slog.wtf(TAG, "NetworkStack failed but could not find its package");
+ return;
+ }
+ final List<VersionedPackage> pkgList = Collections.singletonList(pkg);
+ onPackageFailure(pkgList, FAILURE_REASON_EXPLICIT_HEALTH_CHECK);
+ });
+ }
+
+ /**
+ * Persists mAllObservers to file. Threshold information is ignored.
+ */
+ private boolean saveToFile() {
+ Slog.i(TAG, "Saving observer state to file");
+ synchronized (mLock) {
+ FileOutputStream stream;
+ try {
+ stream = mPolicyFile.startWrite();
+ } catch (IOException e) {
+ Slog.w(TAG, "Cannot update monitored packages", e);
+ return false;
+ }
+
+ try {
+ TypedXmlSerializer out = Xml.resolveSerializer(stream);
+ out.startDocument(null, true);
+ out.startTag(null, TAG_PACKAGE_WATCHDOG);
+ out.attributeInt(null, ATTR_VERSION, DB_VERSION);
+ for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
+ mAllObservers.valueAt(oIndex).writeLocked(out);
+ }
+ out.endTag(null, TAG_PACKAGE_WATCHDOG);
+ out.endDocument();
+ mPolicyFile.finishWrite(stream);
+ return true;
+ } catch (IOException e) {
+ Slog.w(TAG, "Failed to save monitored packages, restoring backup", e);
+ mPolicyFile.failWrite(stream);
+ return false;
+ } finally {
+ IoUtils.closeQuietly(stream);
+ }
+ }
+ }
+
+ private void saveToFileAsync() {
+ if (!mLongTaskHandler.hasCallbacks(mSaveToFile)) {
+ mLongTaskHandler.post(mSaveToFile);
+ }
+ }
+
+ /** Convert a {@code LongArrayQueue} to a String of comma-separated values. */
+ public static String longArrayQueueToString(LongArrayQueue queue) {
+ if (queue.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(queue.get(0));
+ for (int i = 1; i < queue.size(); i++) {
+ sb.append(",");
+ sb.append(queue.get(i));
+ }
+ return sb.toString();
+ }
+ return "";
+ }
+
+ /** Parse a comma-separated String of longs into a LongArrayQueue. */
+ public static LongArrayQueue parseLongArrayQueue(String commaSeparatedValues) {
+ LongArrayQueue result = new LongArrayQueue();
+ if (!TextUtils.isEmpty(commaSeparatedValues)) {
+ String[] values = commaSeparatedValues.split(",");
+ for (String value : values) {
+ result.addLast(Long.parseLong(value));
+ }
+ }
+ return result;
+ }
+
+
+ /** Dump status of every observer in mAllObservers. */
+ public void dump(IndentingPrintWriter pw) {
+ pw.println("Package Watchdog status");
+ pw.increaseIndent();
+ synchronized (mLock) {
+ for (String observerName : mAllObservers.keySet()) {
+ pw.println("Observer name: " + observerName);
+ pw.increaseIndent();
+ ObserverInternal observerInternal = mAllObservers.get(observerName);
+ observerInternal.dump(pw);
+ pw.decreaseIndent();
+ }
+ }
+ }
+
+ /**
+ * Represents an observer monitoring a set of packages along with the failure thresholds for
+ * each package.
+ *
+ * <p> Note, the PackageWatchdog#mLock must always be held when reading or writing
+ * instances of this class.
+ */
+ private static class ObserverInternal {
+ public final String name;
+ @GuardedBy("mLock")
+ private final ArrayMap<String, MonitoredPackage> mPackages = new ArrayMap<>();
+ @Nullable
+ @GuardedBy("mLock")
+ public PackageHealthObserver registeredObserver;
+
+ ObserverInternal(String name, List<MonitoredPackage> packages) {
+ this.name = name;
+ updatePackagesLocked(packages);
+ }
+
+ /**
+ * Writes important {@link MonitoredPackage} details for this observer to file.
+ * Does not persist any package failure thresholds.
+ */
+ @GuardedBy("mLock")
+ public boolean writeLocked(TypedXmlSerializer out) {
+ try {
+ out.startTag(null, TAG_OBSERVER);
+ out.attribute(null, ATTR_NAME, name);
+ for (int i = 0; i < mPackages.size(); i++) {
+ MonitoredPackage p = mPackages.valueAt(i);
+ p.writeLocked(out);
+ }
+ out.endTag(null, TAG_OBSERVER);
+ return true;
+ } catch (IOException e) {
+ Slog.w(TAG, "Cannot save observer", e);
+ return false;
+ }
+ }
+
+ @GuardedBy("mLock")
+ public void updatePackagesLocked(List<MonitoredPackage> packages) {
+ for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
+ MonitoredPackage p = packages.get(pIndex);
+ MonitoredPackage existingPackage = getMonitoredPackage(p.getName());
+ if (existingPackage != null) {
+ existingPackage.updateHealthCheckDuration(p.mDurationMs);
+ } else {
+ putMonitoredPackage(p);
+ }
+ }
+ }
+
+ /**
+ * Reduces the monitoring durations of all packages observed by this observer by
+ * {@code elapsedMs}. If any duration is less than 0, the package is removed from
+ * observation. If any health check duration is less than 0, the health check result
+ * is evaluated.
+ *
+ * @return a {@link Set} of packages that were removed from the observer without explicit
+ * health check passing, or an empty list if no package expired for which an explicit health
+ * check was still pending
+ */
+ @GuardedBy("mLock")
+ private Set<MonitoredPackage> prunePackagesLocked(long elapsedMs) {
+ Set<MonitoredPackage> failedPackages = new ArraySet<>();
+ Iterator<MonitoredPackage> it = mPackages.values().iterator();
+ while (it.hasNext()) {
+ MonitoredPackage p = it.next();
+ int oldState = p.getHealthCheckStateLocked();
+ int newState = p.handleElapsedTimeLocked(elapsedMs);
+ if (oldState != HealthCheckState.FAILED
+ && newState == HealthCheckState.FAILED) {
+ Slog.i(TAG, "Package " + p.getName() + " failed health check");
+ failedPackages.add(p);
+ }
+ if (p.isExpiredLocked()) {
+ it.remove();
+ }
+ }
+ return failedPackages;
+ }
+
+ /**
+ * Increments failure counts of {@code packageName}.
+ * @returns {@code true} if failure threshold is exceeded, {@code false} otherwise
+ */
+ @GuardedBy("mLock")
+ public boolean onPackageFailureLocked(String packageName) {
+ if (getMonitoredPackage(packageName) == null && registeredObserver.isPersistent()
+ && registeredObserver.mayObservePackage(packageName)) {
+ putMonitoredPackage(sPackageWatchdog.newMonitoredPackage(
+ packageName, DEFAULT_OBSERVING_DURATION_MS, false));
+ }
+ MonitoredPackage p = getMonitoredPackage(packageName);
+ if (p != null) {
+ return p.onFailureLocked();
+ }
+ return false;
+ }
+
+ /**
+ * Returns the map of packages monitored by this observer.
+ *
+ * @return a mapping of package names to {@link MonitoredPackage} objects.
+ */
+ @GuardedBy("mLock")
+ public ArrayMap<String, MonitoredPackage> getMonitoredPackages() {
+ return mPackages;
+ }
+
+ /**
+ * Returns the {@link MonitoredPackage} associated with a given package name if the
+ * package is being monitored by this observer.
+ *
+ * @param packageName: the name of the package.
+ * @return the {@link MonitoredPackage} object associated with the package name if one
+ * exists, {@code null} otherwise.
+ */
+ @GuardedBy("mLock")
+ @Nullable
+ public MonitoredPackage getMonitoredPackage(String packageName) {
+ return mPackages.get(packageName);
+ }
+
+ /**
+ * Associates a {@link MonitoredPackage} with the observer.
+ *
+ * @param p: the {@link MonitoredPackage} to store.
+ */
+ @GuardedBy("mLock")
+ public void putMonitoredPackage(MonitoredPackage p) {
+ mPackages.put(p.getName(), p);
+ }
+
+ /**
+ * Returns one ObserverInternal from the {@code parser} and advances its state.
+ *
+ * <p>Note that this method is <b>not</b> thread safe. It should only be called from
+ * #loadFromFile which in turn is only called on construction of the
+ * singleton PackageWatchdog.
+ **/
+ public static ObserverInternal read(TypedXmlPullParser parser, PackageWatchdog watchdog) {
+ String observerName = null;
+ if (TAG_OBSERVER.equals(parser.getName())) {
+ observerName = parser.getAttributeValue(null, ATTR_NAME);
+ if (TextUtils.isEmpty(observerName)) {
+ Slog.wtf(TAG, "Unable to read observer name");
+ return null;
+ }
+ }
+ List<MonitoredPackage> packages = new ArrayList<>();
+ int innerDepth = parser.getDepth();
+ try {
+ while (XmlUtils.nextElementWithin(parser, innerDepth)) {
+ if (TAG_PACKAGE.equals(parser.getName())) {
+ try {
+ MonitoredPackage pkg = watchdog.parseMonitoredPackage(parser);
+ if (pkg != null) {
+ packages.add(pkg);
+ }
+ } catch (NumberFormatException e) {
+ Slog.wtf(TAG, "Skipping package for observer " + observerName, e);
+ continue;
+ }
+ }
+ }
+ } catch (XmlPullParserException | IOException e) {
+ Slog.wtf(TAG, "Unable to read observer " + observerName, e);
+ return null;
+ }
+ if (packages.isEmpty()) {
+ return null;
+ }
+ return new ObserverInternal(observerName, packages);
+ }
+
+ /** Dumps information about this observer and the packages it watches. */
+ public void dump(IndentingPrintWriter pw) {
+ boolean isPersistent = registeredObserver != null && registeredObserver.isPersistent();
+ pw.println("Persistent: " + isPersistent);
+ for (String packageName : mPackages.keySet()) {
+ MonitoredPackage p = getMonitoredPackage(packageName);
+ pw.println(packageName + ": ");
+ pw.increaseIndent();
+ pw.println("# Failures: " + p.mFailureHistory.size());
+ pw.println("Monitoring duration remaining: " + p.mDurationMs + "ms");
+ pw.println("Explicit health check duration: " + p.mHealthCheckDurationMs + "ms");
+ pw.println("Health check state: " + p.toString(p.mHealthCheckState));
+ pw.decreaseIndent();
+ }
+ }
+ }
+
+ @Retention(SOURCE)
+ @IntDef(value = {
+ HealthCheckState.ACTIVE,
+ HealthCheckState.INACTIVE,
+ HealthCheckState.PASSED,
+ HealthCheckState.FAILED})
+ public @interface HealthCheckState {
+ // The package has not passed health check but has requested a health check
+ int ACTIVE = 0;
+ // The package has not passed health check and has not requested a health check
+ int INACTIVE = 1;
+ // The package has passed health check
+ int PASSED = 2;
+ // The package has failed health check
+ int FAILED = 3;
+ }
+
+ MonitoredPackage newMonitoredPackage(
+ String name, long durationMs, boolean hasPassedHealthCheck) {
+ return newMonitoredPackage(name, durationMs, Long.MAX_VALUE, hasPassedHealthCheck,
+ new LongArrayQueue());
+ }
+
+ MonitoredPackage newMonitoredPackage(String name, long durationMs, long healthCheckDurationMs,
+ boolean hasPassedHealthCheck, LongArrayQueue mitigationCalls) {
+ return new MonitoredPackage(name, durationMs, healthCheckDurationMs,
+ hasPassedHealthCheck, mitigationCalls);
+ }
+
+ MonitoredPackage parseMonitoredPackage(TypedXmlPullParser parser)
+ throws XmlPullParserException {
+ String packageName = parser.getAttributeValue(null, ATTR_NAME);
+ long duration = parser.getAttributeLong(null, ATTR_DURATION);
+ long healthCheckDuration = parser.getAttributeLong(null,
+ ATTR_EXPLICIT_HEALTH_CHECK_DURATION);
+ boolean hasPassedHealthCheck = parser.getAttributeBoolean(null, ATTR_PASSED_HEALTH_CHECK);
+ LongArrayQueue mitigationCalls = parseLongArrayQueue(
+ parser.getAttributeValue(null, ATTR_MITIGATION_CALLS));
+ return newMonitoredPackage(packageName,
+ duration, healthCheckDuration, hasPassedHealthCheck, mitigationCalls);
+ }
+
+ /**
+ * Represents a package and its health check state along with the time
+ * it should be monitored for.
+ *
+ * <p> Note, the PackageWatchdog#mLock must always be held when reading or writing
+ * instances of this class.
+ */
+ class MonitoredPackage {
+ private final String mPackageName;
+ // Times when package failures happen sorted in ascending order
+ @GuardedBy("mLock")
+ private final LongArrayQueue mFailureHistory = new LongArrayQueue();
+ // Times when an observer was called to mitigate this package's failure. Sorted in
+ // ascending order.
+ @GuardedBy("mLock")
+ private final LongArrayQueue mMitigationCalls;
+ // One of STATE_[ACTIVE|INACTIVE|PASSED|FAILED]. Updated on construction and after
+ // methods that could change the health check state: handleElapsedTimeLocked and
+ // tryPassHealthCheckLocked
+ private int mHealthCheckState = HealthCheckState.INACTIVE;
+ // Whether an explicit health check has passed.
+ // This value in addition with mHealthCheckDurationMs determines the health check state
+ // of the package, see #getHealthCheckStateLocked
+ @GuardedBy("mLock")
+ private boolean mHasPassedHealthCheck;
+ // System uptime duration to monitor package.
+ @GuardedBy("mLock")
+ private long mDurationMs;
+ // System uptime duration to check the result of an explicit health check
+ // Initially, MAX_VALUE until we get a value from the health check service
+ // and request health checks.
+ // This value in addition with mHasPassedHealthCheck determines the health check state
+ // of the package, see #getHealthCheckStateLocked
+ @GuardedBy("mLock")
+ private long mHealthCheckDurationMs = Long.MAX_VALUE;
+
+ MonitoredPackage(String packageName, long durationMs,
+ long healthCheckDurationMs, boolean hasPassedHealthCheck,
+ LongArrayQueue mitigationCalls) {
+ mPackageName = packageName;
+ mDurationMs = durationMs;
+ mHealthCheckDurationMs = healthCheckDurationMs;
+ mHasPassedHealthCheck = hasPassedHealthCheck;
+ mMitigationCalls = mitigationCalls;
+ updateHealthCheckStateLocked();
+ }
+
+ /** Writes the salient fields to disk using {@code out}. */
+ @GuardedBy("mLock")
+ public void writeLocked(TypedXmlSerializer out) throws IOException {
+ out.startTag(null, TAG_PACKAGE);
+ out.attribute(null, ATTR_NAME, getName());
+ out.attributeLong(null, ATTR_DURATION, mDurationMs);
+ out.attributeLong(null, ATTR_EXPLICIT_HEALTH_CHECK_DURATION, mHealthCheckDurationMs);
+ out.attributeBoolean(null, ATTR_PASSED_HEALTH_CHECK, mHasPassedHealthCheck);
+ LongArrayQueue normalizedCalls = normalizeMitigationCalls();
+ out.attribute(null, ATTR_MITIGATION_CALLS, longArrayQueueToString(normalizedCalls));
+ out.endTag(null, TAG_PACKAGE);
+ }
+
+ /**
+ * Increment package failures or resets failure count depending on the last package failure.
+ *
+ * @return {@code true} if failure count exceeds a threshold, {@code false} otherwise
+ */
+ @GuardedBy("mLock")
+ public boolean onFailureLocked() {
+ // Sliding window algorithm: find out if there exists a window containing failures >=
+ // mTriggerFailureCount.
+ final long now = mSystemClock.uptimeMillis();
+ mFailureHistory.addLast(now);
+ while (now - mFailureHistory.peekFirst() > mTriggerFailureDurationMs) {
+ // Prune values falling out of the window
+ mFailureHistory.removeFirst();
+ }
+ boolean failed = mFailureHistory.size() >= mTriggerFailureCount;
+ if (failed) {
+ mFailureHistory.clear();
+ }
+ return failed;
+ }
+
+ /**
+ * Notes the timestamp of a mitigation call into the observer.
+ */
+ @GuardedBy("mLock")
+ public void noteMitigationCallLocked() {
+ mMitigationCalls.addLast(mSystemClock.uptimeMillis());
+ }
+
+ /**
+ * Prunes any mitigation calls outside of the de-escalation window, and returns the
+ * number of calls that are in the window afterwards.
+ *
+ * @return the number of mitigation calls made in the de-escalation window.
+ */
+ @GuardedBy("mLock")
+ public int getMitigationCountLocked() {
+ try {
+ final long now = mSystemClock.uptimeMillis();
+ while (now - mMitigationCalls.peekFirst() > DEFAULT_DEESCALATION_WINDOW_MS) {
+ mMitigationCalls.removeFirst();
+ }
+ } catch (NoSuchElementException ignore) {
+ }
+
+ return mMitigationCalls.size();
+ }
+
+ /**
+ * Before writing to disk, make the mitigation call timestamps relative to the current
+ * system uptime. This is because they need to be relative to the uptime which will reset
+ * at the next boot.
+ *
+ * @return a LongArrayQueue of the mitigation calls relative to the current system uptime.
+ */
+ @GuardedBy("mLock")
+ public LongArrayQueue normalizeMitigationCalls() {
+ LongArrayQueue normalized = new LongArrayQueue();
+ final long now = mSystemClock.uptimeMillis();
+ for (int i = 0; i < mMitigationCalls.size(); i++) {
+ normalized.addLast(mMitigationCalls.get(i) - now);
+ }
+ return normalized;
+ }
+
+ /**
+ * Sets the initial health check duration.
+ *
+ * @return the new health check state
+ */
+ @GuardedBy("mLock")
+ public int setHealthCheckActiveLocked(long initialHealthCheckDurationMs) {
+ if (initialHealthCheckDurationMs <= 0) {
+ Slog.wtf(TAG, "Cannot set non-positive health check duration "
+ + initialHealthCheckDurationMs + "ms for package " + getName()
+ + ". Using total duration " + mDurationMs + "ms instead");
+ initialHealthCheckDurationMs = mDurationMs;
+ }
+ if (mHealthCheckState == HealthCheckState.INACTIVE) {
+ // Transitions to ACTIVE
+ mHealthCheckDurationMs = initialHealthCheckDurationMs;
+ }
+ return updateHealthCheckStateLocked();
+ }
+
+ /**
+ * Updates the monitoring durations of the package.
+ *
+ * @return the new health check state
+ */
+ @GuardedBy("mLock")
+ public int handleElapsedTimeLocked(long elapsedMs) {
+ if (elapsedMs <= 0) {
+ Slog.w(TAG, "Cannot handle non-positive elapsed time for package " + getName());
+ return mHealthCheckState;
+ }
+ // Transitions to FAILED if now <= 0 and health check not passed
+ mDurationMs -= elapsedMs;
+ if (mHealthCheckState == HealthCheckState.ACTIVE) {
+ // We only update health check durations if we have #setHealthCheckActiveLocked
+ // This ensures we don't leave the INACTIVE state for an unexpected elapsed time
+ // Transitions to FAILED if now <= 0 and health check not passed
+ mHealthCheckDurationMs -= elapsedMs;
+ }
+ return updateHealthCheckStateLocked();
+ }
+
+ /** Explicitly update the monitoring duration of the package. */
+ @GuardedBy("mLock")
+ public void updateHealthCheckDuration(long newDurationMs) {
+ mDurationMs = newDurationMs;
+ }
+
+ /**
+ * Marks the health check as passed and transitions to {@link HealthCheckState.PASSED}
+ * if not yet {@link HealthCheckState.FAILED}.
+ *
+ * @return the new {@link HealthCheckState health check state}
+ */
+ @GuardedBy("mLock")
+ @HealthCheckState
+ public int tryPassHealthCheckLocked() {
+ if (mHealthCheckState != HealthCheckState.FAILED) {
+ // FAILED is a final state so only pass if we haven't failed
+ // Transition to PASSED
+ mHasPassedHealthCheck = true;
+ }
+ return updateHealthCheckStateLocked();
+ }
+
+ /** Returns the monitored package name. */
+ private String getName() {
+ return mPackageName;
+ }
+
+ /**
+ * Returns the current {@link HealthCheckState health check state}.
+ */
+ @GuardedBy("mLock")
+ @HealthCheckState
+ public int getHealthCheckStateLocked() {
+ return mHealthCheckState;
+ }
+
+ /**
+ * Returns the shortest duration before the package should be scheduled for a prune.
+ *
+ * @return the duration or {@link Long#MAX_VALUE} if the package should not be scheduled
+ */
+ @GuardedBy("mLock")
+ public long getShortestScheduleDurationMsLocked() {
+ // Consider health check duration only if #isPendingHealthChecksLocked is true
+ return Math.min(toPositive(mDurationMs),
+ isPendingHealthChecksLocked()
+ ? toPositive(mHealthCheckDurationMs) : Long.MAX_VALUE);
+ }
+
+ /**
+ * Returns {@code true} if the total duration left to monitor the package is less than or
+ * equal to 0 {@code false} otherwise.
+ */
+ @GuardedBy("mLock")
+ public boolean isExpiredLocked() {
+ return mDurationMs <= 0;
+ }
+
+ /**
+ * Returns {@code true} if the package, {@link #getName} is expecting health check results
+ * {@code false} otherwise.
+ */
+ @GuardedBy("mLock")
+ public boolean isPendingHealthChecksLocked() {
+ return mHealthCheckState == HealthCheckState.ACTIVE
+ || mHealthCheckState == HealthCheckState.INACTIVE;
+ }
+
+ /**
+ * Updates the health check state based on {@link #mHasPassedHealthCheck}
+ * and {@link #mHealthCheckDurationMs}.
+ *
+ * @return the new {@link HealthCheckState health check state}
+ */
+ @GuardedBy("mLock")
+ @HealthCheckState
+ private int updateHealthCheckStateLocked() {
+ int oldState = mHealthCheckState;
+ if (mHasPassedHealthCheck) {
+ // Set final state first to avoid ambiguity
+ mHealthCheckState = HealthCheckState.PASSED;
+ } else if (mHealthCheckDurationMs <= 0 || mDurationMs <= 0) {
+ // Set final state first to avoid ambiguity
+ mHealthCheckState = HealthCheckState.FAILED;
+ } else if (mHealthCheckDurationMs == Long.MAX_VALUE) {
+ mHealthCheckState = HealthCheckState.INACTIVE;
+ } else {
+ mHealthCheckState = HealthCheckState.ACTIVE;
+ }
+
+ if (oldState != mHealthCheckState) {
+ Slog.i(TAG, "Updated health check state for package " + getName() + ": "
+ + toString(oldState) + " -> " + toString(mHealthCheckState));
+ }
+ return mHealthCheckState;
+ }
+
+ /** Returns a {@link String} representation of the current health check state. */
+ private String toString(@HealthCheckState int state) {
+ switch (state) {
+ case HealthCheckState.ACTIVE:
+ return "ACTIVE";
+ case HealthCheckState.INACTIVE:
+ return "INACTIVE";
+ case HealthCheckState.PASSED:
+ return "PASSED";
+ case HealthCheckState.FAILED:
+ return "FAILED";
+ default:
+ return "UNKNOWN";
+ }
+ }
+
+ /** Returns {@code value} if it is greater than 0 or {@link Long#MAX_VALUE} otherwise. */
+ private long toPositive(long value) {
+ return value > 0 ? value : Long.MAX_VALUE;
+ }
+
+ /** Compares the equality of this object with another {@link MonitoredPackage}. */
+ @VisibleForTesting
+ boolean isEqualTo(MonitoredPackage pkg) {
+ return (getName().equals(pkg.getName()))
+ && mDurationMs == pkg.mDurationMs
+ && mHasPassedHealthCheck == pkg.mHasPassedHealthCheck
+ && mHealthCheckDurationMs == pkg.mHealthCheckDurationMs
+ && (mMitigationCalls.toString()).equals(pkg.mMitigationCalls.toString());
+ }
+ }
+
+ /**
+ * Handles the thresholding logic for system server boots.
+ */
+ class BootThreshold {
+
+ private final int mBootTriggerCount;
+ private final long mTriggerWindow;
+
+ BootThreshold(int bootTriggerCount, long triggerWindow) {
+ this.mBootTriggerCount = bootTriggerCount;
+ this.mTriggerWindow = triggerWindow;
+ }
+
+ public void reset() {
+ setStart(0);
+ setCount(0);
+ }
+
+ private int getCount() {
+ return SystemProperties.getInt(PROP_RESCUE_BOOT_COUNT, 0);
+ }
+
+ private void setCount(int count) {
+ SystemProperties.set(PROP_RESCUE_BOOT_COUNT, Integer.toString(count));
+ }
+
+ public long getStart() {
+ return SystemProperties.getLong(PROP_RESCUE_BOOT_START, 0);
+ }
+
+ public int getMitigationCount() {
+ return SystemProperties.getInt(PROP_BOOT_MITIGATION_COUNT, 0);
+ }
+
+ public void setStart(long start) {
+ setPropertyStart(PROP_RESCUE_BOOT_START, start);
+ }
+
+ public void setMitigationStart(long start) {
+ setPropertyStart(PROP_BOOT_MITIGATION_WINDOW_START, start);
+ }
+
+ public long getMitigationStart() {
+ return SystemProperties.getLong(PROP_BOOT_MITIGATION_WINDOW_START, 0);
+ }
+
+ public void setMitigationCount(int count) {
+ SystemProperties.set(PROP_BOOT_MITIGATION_COUNT, Integer.toString(count));
+ }
+
+ public void setPropertyStart(String property, long start) {
+ final long now = mSystemClock.uptimeMillis();
+ final long newStart = MathUtils.constrain(start, 0, now);
+ SystemProperties.set(property, Long.toString(newStart));
+ }
+
+ public void saveMitigationCountToMetadata() {
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(METADATA_FILE))) {
+ writer.write(String.valueOf(getMitigationCount()));
+ } catch (Exception e) {
+ Slog.e(TAG, "Could not save metadata to file: " + e);
+ }
+ }
+
+ public void readMitigationCountFromMetadataIfNecessary() {
+ File bootPropsFile = new File(METADATA_FILE);
+ if (bootPropsFile.exists()) {
+ try (BufferedReader reader = new BufferedReader(new FileReader(METADATA_FILE))) {
+ String mitigationCount = reader.readLine();
+ setMitigationCount(Integer.parseInt(mitigationCount));
+ bootPropsFile.delete();
+ } catch (Exception e) {
+ Slog.i(TAG, "Could not read metadata file: " + e);
+ }
+ }
+ }
+
+
+ /** Increments the boot counter, and returns whether the device is bootlooping. */
+ public boolean incrementAndTest() {
+ readMitigationCountFromMetadataIfNecessary();
+ final long now = mSystemClock.uptimeMillis();
+ if (now - getStart() < 0) {
+ Slog.e(TAG, "Window was less than zero. Resetting start to current time.");
+ setStart(now);
+ setMitigationStart(now);
+ }
+ if (now - getMitigationStart() > DEFAULT_DEESCALATION_WINDOW_MS) {
+ setMitigationCount(0);
+ setMitigationStart(now);
+ }
+ final long window = now - getStart();
+ if (window >= mTriggerWindow) {
+ setCount(1);
+ setStart(now);
+ return false;
+ } else {
+ int count = getCount() + 1;
+ setCount(count);
+ EventLogTags.writeRescueNote(Process.ROOT_UID, count, window);
+ return count >= mBootTriggerCount;
+ }
+ }
+
+ }
+}
diff --git a/packages/CrashRecovery/services/java/com/android/server/RescueParty.java b/packages/CrashRecovery/services/java/com/android/server/RescueParty.java
new file mode 100644
index 0000000..eb65b2a
--- /dev/null
+++ b/packages/CrashRecovery/services/java/com/android/server/RescueParty.java
@@ -0,0 +1,823 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server;
+
+import static android.provider.DeviceConfig.Properties;
+
+import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo;
+
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.content.ContentResolver;
+import android.content.Context;
+import android.content.pm.ApplicationInfo;
+import android.content.pm.PackageManager;
+import android.content.pm.VersionedPackage;
+import android.os.Build;
+import android.os.Environment;
+import android.os.FileUtils;
+import android.os.PowerManager;
+import android.os.RecoverySystem;
+import android.os.SystemClock;
+import android.os.SystemProperties;
+import android.os.UserHandle;
+import android.provider.DeviceConfig;
+import android.provider.Settings;
+import android.text.TextUtils;
+import android.util.ArraySet;
+import android.util.ExceptionUtils;
+import android.util.Log;
+import android.util.Slog;
+
+import com.android.internal.annotations.GuardedBy;
+import com.android.internal.annotations.VisibleForTesting;
+import com.android.internal.util.ArrayUtils;
+import com.android.internal.util.FrameworkStatsLog;
+import com.android.server.PackageWatchdog.FailureReasons;
+import com.android.server.PackageWatchdog.PackageHealthObserver;
+import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
+import com.android.server.am.SettingsToPropertiesMapper;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Utilities to help rescue the system from crash loops. Callers are expected to
+ * report boot events and persistent app crashes, and if they happen frequently
+ * enough this class will slowly escalate through several rescue operations
+ * before finally rebooting and prompting the user if they want to wipe data as
+ * a last resort.
+ *
+ * @hide
+ */
+public class RescueParty {
+ @VisibleForTesting
+ static final String PROP_ENABLE_RESCUE = "persist.sys.enable_rescue";
+ static final String PROP_ATTEMPTING_FACTORY_RESET = "sys.attempting_factory_reset";
+ static final String PROP_ATTEMPTING_REBOOT = "sys.attempting_reboot";
+ static final String PROP_MAX_RESCUE_LEVEL_ATTEMPTED = "sys.max_rescue_level_attempted";
+ static final String PROP_LAST_FACTORY_RESET_TIME_MS = "persist.sys.last_factory_reset";
+ @VisibleForTesting
+ static final int LEVEL_NONE = 0;
+ @VisibleForTesting
+ static final int LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS = 1;
+ @VisibleForTesting
+ static final int LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES = 2;
+ @VisibleForTesting
+ static final int LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS = 3;
+ @VisibleForTesting
+ static final int LEVEL_WARM_REBOOT = 4;
+ @VisibleForTesting
+ static final int LEVEL_FACTORY_RESET = 5;
+ @VisibleForTesting
+ static final String PROP_RESCUE_BOOT_COUNT = "sys.rescue_boot_count";
+ @VisibleForTesting
+ static final String TAG = "RescueParty";
+ @VisibleForTesting
+ static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2);
+ @VisibleForTesting
+ static final int DEVICE_CONFIG_RESET_MODE = Settings.RESET_MODE_TRUSTED_DEFAULTS;
+ // The DeviceConfig namespace containing all RescueParty switches.
+ @VisibleForTesting
+ static final String NAMESPACE_CONFIGURATION = "configuration";
+ @VisibleForTesting
+ static final String NAMESPACE_TO_PACKAGE_MAPPING_FLAG =
+ "namespace_to_package_mapping";
+ @VisibleForTesting
+ static final long DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN = 10;
+
+ private static final String NAME = "rescue-party-observer";
+
+ private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue";
+ private static final String PROP_VIRTUAL_DEVICE = "ro.hardware.virtual_device";
+ private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG =
+ "persist.device_config.configuration.disable_rescue_party";
+ private static final String PROP_DISABLE_FACTORY_RESET_FLAG =
+ "persist.device_config.configuration.disable_rescue_party_factory_reset";
+ private static final String PROP_THROTTLE_DURATION_MIN_FLAG =
+ "persist.device_config.configuration.rescue_party_throttle_duration_min";
+
+ private static final int PERSISTENT_MASK = ApplicationInfo.FLAG_PERSISTENT
+ | ApplicationInfo.FLAG_SYSTEM;
+
+ /** Register the Rescue Party observer as a Package Watchdog health observer */
+ public static void registerHealthObserver(Context context) {
+ PackageWatchdog.getInstance(context).registerHealthObserver(
+ RescuePartyObserver.getInstance(context));
+ }
+
+ private static boolean isDisabled() {
+ // Check if we're explicitly enabled for testing
+ if (SystemProperties.getBoolean(PROP_ENABLE_RESCUE, false)) {
+ return false;
+ }
+
+ // We're disabled if the DeviceConfig disable flag is set to true.
+ // This is in case that an emergency rollback of the feature is needed.
+ if (SystemProperties.getBoolean(PROP_DEVICE_CONFIG_DISABLE_FLAG, false)) {
+ Slog.v(TAG, "Disabled because of DeviceConfig flag");
+ return true;
+ }
+
+ // We're disabled on all engineering devices
+ if (Build.IS_ENG) {
+ Slog.v(TAG, "Disabled because of eng build");
+ return true;
+ }
+
+ // We're disabled on userdebug devices connected over USB, since that's
+ // a decent signal that someone is actively trying to debug the device,
+ // or that it's in a lab environment.
+ if (Build.IS_USERDEBUG && isUsbActive()) {
+ Slog.v(TAG, "Disabled because of active USB connection");
+ return true;
+ }
+
+ // One last-ditch check
+ if (SystemProperties.getBoolean(PROP_DISABLE_RESCUE, false)) {
+ Slog.v(TAG, "Disabled because of manual property");
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * Check if we're currently attempting to reboot for a factory reset. This method must
+ * return true if RescueParty tries to reboot early during a boot loop, since the device
+ * will not be fully booted at this time.
+ *
+ * TODO(gavincorkery): Rename method since its scope has expanded.
+ */
+ public static boolean isAttemptingFactoryReset() {
+ return isFactoryResetPropertySet() || isRebootPropertySet();
+ }
+
+ static boolean isFactoryResetPropertySet() {
+ return SystemProperties.getBoolean(PROP_ATTEMPTING_FACTORY_RESET, false);
+ }
+
+ static boolean isRebootPropertySet() {
+ return SystemProperties.getBoolean(PROP_ATTEMPTING_REBOOT, false);
+ }
+
+ /**
+ * Called when {@code SettingsProvider} has been published, which is a good
+ * opportunity to reset any settings depending on our rescue level.
+ */
+ public static void onSettingsProviderPublished(Context context) {
+ handleNativeRescuePartyResets();
+ ContentResolver contentResolver = context.getContentResolver();
+ DeviceConfig.setMonitorCallback(
+ contentResolver,
+ Executors.newSingleThreadExecutor(),
+ new RescuePartyMonitorCallback(context));
+ }
+
+
+ /**
+ * Called when {@code RollbackManager} performs Mainline module rollbacks,
+ * to avoid rolled back modules consuming flag values only expected to work
+ * on modules of newer versions.
+ */
+ public static void resetDeviceConfigForPackages(List<String> packageNames) {
+ if (packageNames == null) {
+ return;
+ }
+ Set<String> namespacesToReset = new ArraySet<String>();
+ Iterator<String> it = packageNames.iterator();
+ RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstanceIfCreated();
+ // Get runtime package to namespace mapping if created.
+ if (rescuePartyObserver != null) {
+ while (it.hasNext()) {
+ String packageName = it.next();
+ Set<String> runtimeAffectedNamespaces =
+ rescuePartyObserver.getAffectedNamespaceSet(packageName);
+ if (runtimeAffectedNamespaces != null) {
+ namespacesToReset.addAll(runtimeAffectedNamespaces);
+ }
+ }
+ }
+ // Get preset package to namespace mapping if created.
+ Set<String> presetAffectedNamespaces = getPresetNamespacesForPackages(
+ packageNames);
+ if (presetAffectedNamespaces != null) {
+ namespacesToReset.addAll(presetAffectedNamespaces);
+ }
+
+ // Clear flags under the namespaces mapped to these packages.
+ // Using setProperties since DeviceConfig.resetToDefaults bans the current flag set.
+ Iterator<String> namespaceIt = namespacesToReset.iterator();
+ while (namespaceIt.hasNext()) {
+ String namespaceToReset = namespaceIt.next();
+ Properties properties = new Properties.Builder(namespaceToReset).build();
+ try {
+ if (!DeviceConfig.setProperties(properties)) {
+ logCriticalInfo(Log.ERROR, "Failed to clear properties under "
+ + namespaceToReset
+ + ". Running `device_config get_sync_disabled_for_tests` will confirm"
+ + " if config-bulk-update is enabled.");
+ }
+ } catch (DeviceConfig.BadConfigException exception) {
+ logCriticalInfo(Log.WARN, "namespace " + namespaceToReset
+ + " is already banned, skip reset.");
+ }
+ }
+ }
+
+ private static Set<String> getPresetNamespacesForPackages(List<String> packageNames) {
+ Set<String> resultSet = new ArraySet<String>();
+ try {
+ String flagVal = DeviceConfig.getString(NAMESPACE_CONFIGURATION,
+ NAMESPACE_TO_PACKAGE_MAPPING_FLAG, "");
+ String[] mappingEntries = flagVal.split(",");
+ for (int i = 0; i < mappingEntries.length; i++) {
+ if (TextUtils.isEmpty(mappingEntries[i])) {
+ continue;
+ }
+ String[] splittedEntry = mappingEntries[i].split(":");
+ if (splittedEntry.length != 2) {
+ throw new RuntimeException("Invalid mapping entry: " + mappingEntries[i]);
+ }
+ String namespace = splittedEntry[0];
+ String packageName = splittedEntry[1];
+
+ if (packageNames.contains(packageName)) {
+ resultSet.add(namespace);
+ }
+ }
+ } catch (Exception e) {
+ resultSet.clear();
+ Slog.e(TAG, "Failed to read preset package to namespaces mapping.", e);
+ } finally {
+ return resultSet;
+ }
+ }
+
+ @VisibleForTesting
+ static long getElapsedRealtime() {
+ return SystemClock.elapsedRealtime();
+ }
+
+ private static class RescuePartyMonitorCallback implements DeviceConfig.MonitorCallback {
+ Context mContext;
+
+ RescuePartyMonitorCallback(Context context) {
+ this.mContext = context;
+ }
+
+ public void onNamespaceUpdate(@NonNull String updatedNamespace) {
+ startObservingPackages(mContext, updatedNamespace);
+ }
+
+ public void onDeviceConfigAccess(@NonNull String callingPackage,
+ @NonNull String namespace) {
+ RescuePartyObserver.getInstance(mContext).recordDeviceConfigAccess(
+ callingPackage,
+ namespace);
+ }
+ }
+
+ private static void startObservingPackages(Context context, @NonNull String updatedNamespace) {
+ RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstance(context);
+ Set<String> callingPackages = rescuePartyObserver.getCallingPackagesSet(updatedNamespace);
+ if (callingPackages == null) {
+ return;
+ }
+ List<String> callingPackageList = new ArrayList<>();
+ callingPackageList.addAll(callingPackages);
+ Slog.i(TAG, "Starting to observe: " + callingPackageList + ", updated namespace: "
+ + updatedNamespace);
+ PackageWatchdog.getInstance(context).startObservingHealth(
+ rescuePartyObserver,
+ callingPackageList,
+ DEFAULT_OBSERVING_DURATION_MS);
+ }
+
+ private static void handleNativeRescuePartyResets() {
+ if (SettingsToPropertiesMapper.isNativeFlagsResetPerformed()) {
+ String[] resetNativeCategories = SettingsToPropertiesMapper.getResetNativeCategories();
+ for (int i = 0; i < resetNativeCategories.length; i++) {
+ // Don't let RescueParty reset the namespace for RescueParty switches.
+ if (NAMESPACE_CONFIGURATION.equals(resetNativeCategories[i])) {
+ continue;
+ }
+ DeviceConfig.resetToDefaults(DEVICE_CONFIG_RESET_MODE,
+ resetNativeCategories[i]);
+ }
+ }
+ }
+
+ private static int getMaxRescueLevel(boolean mayPerformReboot) {
+ if (!mayPerformReboot
+ || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {
+ return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;
+ }
+ return LEVEL_FACTORY_RESET;
+ }
+
+ /**
+ * Get the rescue level to perform if this is the n-th attempt at mitigating failure.
+ *
+ * @param mitigationCount: the mitigation attempt number (1 = first attempt etc.)
+ * @param mayPerformReboot: whether or not a reboot and factory reset may be performed
+ * for the given failure.
+ * @return the rescue level for the n-th mitigation attempt.
+ */
+ private static int getRescueLevel(int mitigationCount, boolean mayPerformReboot) {
+ if (mitigationCount == 1) {
+ return LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS;
+ } else if (mitigationCount == 2) {
+ return LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES;
+ } else if (mitigationCount == 3) {
+ return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;
+ } else if (mitigationCount == 4) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot), LEVEL_WARM_REBOOT);
+ } else if (mitigationCount >= 5) {
+ return Math.min(getMaxRescueLevel(mayPerformReboot), LEVEL_FACTORY_RESET);
+ } else {
+ Slog.w(TAG, "Expected positive mitigation count, was " + mitigationCount);
+ return LEVEL_NONE;
+ }
+ }
+
+ private static void executeRescueLevel(Context context, @Nullable String failedPackage,
+ int level) {
+ Slog.w(TAG, "Attempting rescue level " + levelToString(level));
+ try {
+ executeRescueLevelInternal(context, level, failedPackage);
+ EventLogTags.writeRescueSuccess(level);
+ String successMsg = "Finished rescue level " + levelToString(level);
+ if (!TextUtils.isEmpty(failedPackage)) {
+ successMsg += " for package " + failedPackage;
+ }
+ logCriticalInfo(Log.DEBUG, successMsg);
+ } catch (Throwable t) {
+ logRescueException(level, failedPackage, t);
+ }
+ }
+
+ private static void executeRescueLevelInternal(Context context, int level, @Nullable
+ String failedPackage) throws Exception {
+ FrameworkStatsLog.write(FrameworkStatsLog.RESCUE_PARTY_RESET_REPORTED, level);
+ // Try our best to reset all settings possible, and once finished
+ // rethrow any exception that we encountered
+ Exception res = null;
+ Runnable runnable;
+ Thread thread;
+ switch (level) {
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ try {
+ resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_DEFAULTS,
+ level);
+ } catch (Exception e) {
+ res = e;
+ }
+ try {
+ resetDeviceConfig(context, /*isScoped=*/true, failedPackage);
+ } catch (Exception e) {
+ res = e;
+ }
+ break;
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ try {
+ resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_CHANGES,
+ level);
+ } catch (Exception e) {
+ res = e;
+ }
+ try {
+ resetDeviceConfig(context, /*isScoped=*/true, failedPackage);
+ } catch (Exception e) {
+ res = e;
+ }
+ break;
+ case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ try {
+ resetAllSettingsIfNecessary(context, Settings.RESET_MODE_TRUSTED_DEFAULTS,
+ level);
+ } catch (Exception e) {
+ res = e;
+ }
+ try {
+ resetDeviceConfig(context, /*isScoped=*/false, failedPackage);
+ } catch (Exception e) {
+ res = e;
+ }
+ break;
+ case LEVEL_WARM_REBOOT:
+ // Request the reboot from a separate thread to avoid deadlock on PackageWatchdog
+ // when device shutting down.
+ SystemProperties.set(PROP_ATTEMPTING_REBOOT, "true");
+ runnable = () -> {
+ try {
+ PowerManager pm = context.getSystemService(PowerManager.class);
+ if (pm != null) {
+ pm.reboot(TAG);
+ }
+ } catch (Throwable t) {
+ logRescueException(level, failedPackage, t);
+ }
+ };
+ thread = new Thread(runnable);
+ thread.start();
+ break;
+ case LEVEL_FACTORY_RESET:
+ // Before the completion of Reboot, if any crash happens then PackageWatchdog
+ // escalates to next level i.e. factory reset, as they happen in separate threads.
+ // Adding a check to prevent factory reset to execute before above reboot completes.
+ // Note: this reboot property is not persistent resets after reboot is completed.
+ if (isRebootPropertySet()) {
+ break;
+ }
+ SystemProperties.set(PROP_ATTEMPTING_FACTORY_RESET, "true");
+ long now = System.currentTimeMillis();
+ SystemProperties.set(PROP_LAST_FACTORY_RESET_TIME_MS, Long.toString(now));
+ runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ RecoverySystem.rebootPromptAndWipeUserData(context, TAG);
+ } catch (Throwable t) {
+ logRescueException(level, failedPackage, t);
+ }
+ }
+ };
+ thread = new Thread(runnable);
+ thread.start();
+ break;
+ }
+
+ if (res != null) {
+ throw res;
+ }
+ }
+
+ private static void logRescueException(int level, @Nullable String failedPackageName,
+ Throwable t) {
+ final String msg = ExceptionUtils.getCompleteMessage(t);
+ EventLogTags.writeRescueFailure(level, msg);
+ String failureMsg = "Failed rescue level " + levelToString(level);
+ if (!TextUtils.isEmpty(failedPackageName)) {
+ failureMsg += " for package " + failedPackageName;
+ }
+ logCriticalInfo(Log.ERROR, failureMsg + ": " + msg);
+ }
+
+ private static int mapRescueLevelToUserImpact(int rescueLevel) {
+ switch(rescueLevel) {
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10;
+ case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
+ case LEVEL_WARM_REBOOT:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
+ case LEVEL_FACTORY_RESET:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100;
+ default:
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ }
+ }
+
+ private static void resetAllSettingsIfNecessary(Context context, int mode,
+ int level) throws Exception {
+ // No need to reset Settings again if they are already reset in the current level once.
+ if (SystemProperties.getInt(PROP_MAX_RESCUE_LEVEL_ATTEMPTED, LEVEL_NONE) >= level) {
+ return;
+ }
+ SystemProperties.set(PROP_MAX_RESCUE_LEVEL_ATTEMPTED, Integer.toString(level));
+ // Try our best to reset all settings possible, and once finished
+ // rethrow any exception that we encountered
+ Exception res = null;
+ final ContentResolver resolver = context.getContentResolver();
+ try {
+ Settings.Global.resetToDefaultsAsUser(resolver, null, mode,
+ UserHandle.SYSTEM.getIdentifier());
+ } catch (Exception e) {
+ res = new RuntimeException("Failed to reset global settings", e);
+ }
+ for (int userId : getAllUserIds()) {
+ try {
+ Settings.Secure.resetToDefaultsAsUser(resolver, null, mode, userId);
+ } catch (Exception e) {
+ res = new RuntimeException("Failed to reset secure settings for " + userId, e);
+ }
+ }
+ if (res != null) {
+ throw res;
+ }
+ }
+
+ private static void resetDeviceConfig(Context context, boolean isScoped,
+ @Nullable String failedPackage) throws Exception {
+ final ContentResolver resolver = context.getContentResolver();
+ try {
+ if (!isScoped || failedPackage == null) {
+ resetAllAffectedNamespaces(context);
+ } else {
+ performScopedReset(context, failedPackage);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to reset config settings", e);
+ }
+ }
+
+ private static void resetAllAffectedNamespaces(Context context) {
+ RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstance(context);
+ Set<String> allAffectedNamespaces = rescuePartyObserver.getAllAffectedNamespaceSet();
+
+ Slog.w(TAG,
+ "Performing reset for all affected namespaces: "
+ + Arrays.toString(allAffectedNamespaces.toArray()));
+ Iterator<String> it = allAffectedNamespaces.iterator();
+ while (it.hasNext()) {
+ String namespace = it.next();
+ // Don't let RescueParty reset the namespace for RescueParty switches.
+ if (NAMESPACE_CONFIGURATION.equals(namespace)) {
+ continue;
+ }
+ DeviceConfig.resetToDefaults(DEVICE_CONFIG_RESET_MODE, namespace);
+ }
+ }
+
+ private static void performScopedReset(Context context, @NonNull String failedPackage) {
+ RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstance(context);
+ Set<String> affectedNamespaces = rescuePartyObserver.getAffectedNamespaceSet(
+ failedPackage);
+ // If we can't find namespaces affected for current package,
+ // skip this round of reset.
+ if (affectedNamespaces != null) {
+ Slog.w(TAG,
+ "Performing scoped reset for package: " + failedPackage
+ + ", affected namespaces: "
+ + Arrays.toString(affectedNamespaces.toArray()));
+ Iterator<String> it = affectedNamespaces.iterator();
+ while (it.hasNext()) {
+ String namespace = it.next();
+ // Don't let RescueParty reset the namespace for RescueParty switches.
+ if (NAMESPACE_CONFIGURATION.equals(namespace)) {
+ continue;
+ }
+ DeviceConfig.resetToDefaults(DEVICE_CONFIG_RESET_MODE, namespace);
+ }
+ }
+ }
+
+ /**
+ * Handle mitigation action for package failures. This observer will be register to Package
+ * Watchdog and will receive calls about package failures. This observer is persistent so it
+ * may choose to mitigate failures for packages it has not explicitly asked to observe.
+ */
+ public static class RescuePartyObserver implements PackageHealthObserver {
+
+ private final Context mContext;
+ private final Map<String, Set<String>> mCallingPackageNamespaceSetMap = new HashMap<>();
+ private final Map<String, Set<String>> mNamespaceCallingPackageSetMap = new HashMap<>();
+
+ @GuardedBy("RescuePartyObserver.class")
+ static RescuePartyObserver sRescuePartyObserver;
+
+ private RescuePartyObserver(Context context) {
+ mContext = context;
+ }
+
+ /** Creates or gets singleton instance of RescueParty. */
+ public static RescuePartyObserver getInstance(Context context) {
+ synchronized (RescuePartyObserver.class) {
+ if (sRescuePartyObserver == null) {
+ sRescuePartyObserver = new RescuePartyObserver(context);
+ }
+ return sRescuePartyObserver;
+ }
+ }
+
+ /** Gets singleton instance. It returns null if the instance is not created yet.*/
+ @Nullable
+ public static RescuePartyObserver getInstanceIfCreated() {
+ synchronized (RescuePartyObserver.class) {
+ return sRescuePartyObserver;
+ }
+ }
+
+ @VisibleForTesting
+ static void reset() {
+ synchronized (RescuePartyObserver.class) {
+ sRescuePartyObserver = null;
+ }
+ }
+
+ @Override
+ public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,
+ @FailureReasons int failureReason, int mitigationCount) {
+ if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
+ || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) {
+ return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
+ mayPerformReboot(failedPackage)));
+ } else {
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ }
+ }
+
+ @Override
+ public boolean execute(@Nullable VersionedPackage failedPackage,
+ @FailureReasons int failureReason, int mitigationCount) {
+ if (isDisabled()) {
+ return false;
+ }
+ if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
+ || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {
+ final int level = getRescueLevel(mitigationCount,
+ mayPerformReboot(failedPackage));
+ executeRescueLevel(mContext,
+ failedPackage == null ? null : failedPackage.getPackageName(), level);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public boolean isPersistent() {
+ return true;
+ }
+
+ @Override
+ public boolean mayObservePackage(String packageName) {
+ PackageManager pm = mContext.getPackageManager();
+ try {
+ // A package is a module if this is non-null
+ if (pm.getModuleInfo(packageName, 0) != null) {
+ return true;
+ }
+ } catch (PackageManager.NameNotFoundException ignore) {
+ }
+
+ return isPersistentSystemApp(packageName);
+ }
+
+ @Override
+ public int onBootLoop(int mitigationCount) {
+ if (isDisabled()) {
+ return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+ }
+ return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, true));
+ }
+
+ @Override
+ public boolean executeBootLoopMitigation(int mitigationCount) {
+ if (isDisabled()) {
+ return false;
+ }
+ boolean mayPerformReboot = !shouldThrottleReboot();
+ executeRescueLevel(mContext, /*failedPackage=*/ null,
+ getRescueLevel(mitigationCount, mayPerformReboot));
+ return true;
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ /**
+ * Returns {@code true} if the failing package is non-null and performing a reboot or
+ * prompting a factory reset is an acceptable mitigation strategy for the package's
+ * failure, {@code false} otherwise.
+ */
+ private boolean mayPerformReboot(@Nullable VersionedPackage failingPackage) {
+ if (failingPackage == null) {
+ return false;
+ }
+ if (shouldThrottleReboot()) {
+ return false;
+ }
+
+ return isPersistentSystemApp(failingPackage.getPackageName());
+ }
+
+ /**
+ * Returns {@code true} if Rescue Party is allowed to attempt a reboot or factory reset.
+ * Will return {@code false} if a factory reset was already offered recently.
+ */
+ private boolean shouldThrottleReboot() {
+ Long lastResetTime = SystemProperties.getLong(PROP_LAST_FACTORY_RESET_TIME_MS, 0);
+ long now = System.currentTimeMillis();
+ long throttleDurationMin = SystemProperties.getLong(PROP_THROTTLE_DURATION_MIN_FLAG,
+ DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN);
+ return now < lastResetTime + TimeUnit.MINUTES.toMillis(throttleDurationMin);
+ }
+
+ private boolean isPersistentSystemApp(@NonNull String packageName) {
+ PackageManager pm = mContext.getPackageManager();
+ try {
+ ApplicationInfo info = pm.getApplicationInfo(packageName, 0);
+ return (info.flags & PERSISTENT_MASK) == PERSISTENT_MASK;
+ } catch (PackageManager.NameNotFoundException e) {
+ return false;
+ }
+ }
+
+ private synchronized void recordDeviceConfigAccess(@NonNull String callingPackage,
+ @NonNull String namespace) {
+ // Record it in calling packages to namespace map
+ Set<String> namespaceSet = mCallingPackageNamespaceSetMap.get(callingPackage);
+ if (namespaceSet == null) {
+ namespaceSet = new ArraySet<>();
+ mCallingPackageNamespaceSetMap.put(callingPackage, namespaceSet);
+ }
+ namespaceSet.add(namespace);
+ // Record it in namespace to calling packages map
+ Set<String> callingPackageSet = mNamespaceCallingPackageSetMap.get(namespace);
+ if (callingPackageSet == null) {
+ callingPackageSet = new ArraySet<>();
+ }
+ callingPackageSet.add(callingPackage);
+ mNamespaceCallingPackageSetMap.put(namespace, callingPackageSet);
+ }
+
+ private synchronized Set<String> getAffectedNamespaceSet(String failedPackage) {
+ return mCallingPackageNamespaceSetMap.get(failedPackage);
+ }
+
+ private synchronized Set<String> getAllAffectedNamespaceSet() {
+ return new HashSet<String>(mNamespaceCallingPackageSetMap.keySet());
+ }
+
+ private synchronized Set<String> getCallingPackagesSet(String namespace) {
+ return mNamespaceCallingPackageSetMap.get(namespace);
+ }
+ }
+
+ private static int[] getAllUserIds() {
+ int systemUserId = UserHandle.SYSTEM.getIdentifier();
+ int[] userIds = { systemUserId };
+ try {
+ for (File file : FileUtils.listFilesOrEmpty(Environment.getDataSystemDeDirectory())) {
+ try {
+ final int userId = Integer.parseInt(file.getName());
+ if (userId != systemUserId) {
+ userIds = ArrayUtils.appendInt(userIds, userId);
+ }
+ } catch (NumberFormatException ignored) {
+ }
+ }
+ } catch (Throwable t) {
+ Slog.w(TAG, "Trouble discovering users", t);
+ }
+ return userIds;
+ }
+
+ /**
+ * Hacky test to check if the device has an active USB connection, which is
+ * a good proxy for someone doing local development work.
+ */
+ private static boolean isUsbActive() {
+ if (SystemProperties.getBoolean(PROP_VIRTUAL_DEVICE, false)) {
+ Slog.v(TAG, "Assuming virtual device is connected over USB");
+ return true;
+ }
+ try {
+ final String state = FileUtils
+ .readTextFile(new File("/sys/class/android_usb/android0/state"), 128, "");
+ return "CONFIGURED".equals(state.trim());
+ } catch (Throwable t) {
+ Slog.w(TAG, "Failed to determine if device was on USB", t);
+ return false;
+ }
+ }
+
+ private static String levelToString(int level) {
+ switch (level) {
+ case LEVEL_NONE: return "NONE";
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS: return "RESET_SETTINGS_UNTRUSTED_DEFAULTS";
+ case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES: return "RESET_SETTINGS_UNTRUSTED_CHANGES";
+ case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS: return "RESET_SETTINGS_TRUSTED_DEFAULTS";
+ case LEVEL_WARM_REBOOT: return "WARM_REBOOT";
+ case LEVEL_FACTORY_RESET: return "FACTORY_RESET";
+ default: return Integer.toString(level);
+ }
+ }
+}
diff --git a/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java
new file mode 100644
index 0000000..2007079
--- /dev/null
+++ b/packages/CrashRecovery/services/java/com/android/server/rollback/RollbackPackageHealthObserver.java
@@ -0,0 +1,529 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.rollback;
+
+import android.annotation.AnyThread;
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.annotation.WorkerThread;
+import android.content.Context;
+import android.content.Intent;
+import android.content.pm.ApplicationInfo;
+import android.content.pm.PackageManager;
+import android.content.pm.VersionedPackage;
+import android.content.rollback.PackageRollbackInfo;
+import android.content.rollback.RollbackInfo;
+import android.content.rollback.RollbackManager;
+import android.os.Environment;
+import android.os.FileUtils;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.os.PowerManager;
+import android.os.SystemProperties;
+import android.util.ArraySet;
+import android.util.Slog;
+import android.util.SparseArray;
+
+import com.android.internal.annotations.VisibleForTesting;
+import com.android.internal.util.FrameworkStatsLog;
+import com.android.internal.util.Preconditions;
+import com.android.server.PackageWatchdog;
+import com.android.server.PackageWatchdog.FailureReasons;
+import com.android.server.PackageWatchdog.PackageHealthObserver;
+import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
+import com.android.server.SystemConfig;
+import com.android.server.pm.ApexManager;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Consumer;
+
+/**
+ * {@link PackageHealthObserver} for {@link RollbackManagerService}.
+ * This class monitors crashes and triggers RollbackManager rollback accordingly.
+ * It also monitors native crashes for some short while after boot.
+ *
+ * @hide
+ */
+final class RollbackPackageHealthObserver implements PackageHealthObserver {
+ private static final String TAG = "RollbackPackageHealthObserver";
+ private static final String NAME = "rollback-observer";
+ private static final String PROP_ATTEMPTING_REBOOT = "sys.attempting_reboot";
+ private static final int PERSISTENT_MASK = ApplicationInfo.FLAG_PERSISTENT
+ | ApplicationInfo.FLAG_SYSTEM;
+
+ private final Context mContext;
+ private final Handler mHandler;
+ private final ApexManager mApexManager;
+ private final File mLastStagedRollbackIdsFile;
+ private final File mTwoPhaseRollbackEnabledFile;
+ // Staged rollback ids that have been committed but their session is not yet ready
+ private final Set<Integer> mPendingStagedRollbackIds = new ArraySet<>();
+ // True if needing to roll back only rebootless apexes when native crash happens
+ private boolean mTwoPhaseRollbackEnabled;
+
+ RollbackPackageHealthObserver(Context context) {
+ mContext = context;
+ HandlerThread handlerThread = new HandlerThread("RollbackPackageHealthObserver");
+ handlerThread.start();
+ mHandler = new Handler(handlerThread.getLooper());
+ File dataDir = new File(Environment.getDataDirectory(), "rollback-observer");
+ dataDir.mkdirs();
+ mLastStagedRollbackIdsFile = new File(dataDir, "last-staged-rollback-ids");
+ mTwoPhaseRollbackEnabledFile = new File(dataDir, "two-phase-rollback-enabled");
+ PackageWatchdog.getInstance(mContext).registerHealthObserver(this);
+ mApexManager = ApexManager.getInstance();
+
+ if (SystemProperties.getBoolean("sys.boot_completed", false)) {
+ // Load the value from the file if system server has crashed and restarted
+ mTwoPhaseRollbackEnabled = readBoolean(mTwoPhaseRollbackEnabledFile);
+ } else {
+ // Disable two-phase rollback for a normal reboot. We assume the rebootless apex
+ // installed before reboot is stable if native crash didn't happen.
+ mTwoPhaseRollbackEnabled = false;
+ writeBoolean(mTwoPhaseRollbackEnabledFile, false);
+ }
+ }
+
+ @Override
+ public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,
+ @FailureReasons int failureReason, int mitigationCount) {
+ boolean anyRollbackAvailable = !mContext.getSystemService(RollbackManager.class)
+ .getAvailableRollbacks().isEmpty();
+ int impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
+
+ if (failureReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH
+ && anyRollbackAvailable) {
+ // For native crashes, we will directly roll back any available rollbacks
+ // Note: For non-native crashes the rollback-all step has higher impact
+ impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_30;
+ } else if (getAvailableRollback(failedPackage) != null) {
+ // Rollback is available, we may get a callback into #execute
+ impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_30;
+ } else if (anyRollbackAvailable) {
+ // If any rollbacks are available, we will commit them
+ impact = PackageHealthObserverImpact.USER_IMPACT_LEVEL_70;
+ }
+
+ return impact;
+ }
+
+ @Override
+ public boolean execute(@Nullable VersionedPackage failedPackage,
+ @FailureReasons int rollbackReason, int mitigationCount) {
+ if (rollbackReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) {
+ mHandler.post(() -> rollbackAll(rollbackReason));
+ return true;
+ }
+
+ RollbackInfo rollback = getAvailableRollback(failedPackage);
+ if (rollback != null) {
+ mHandler.post(() -> rollbackPackage(rollback, failedPackage, rollbackReason));
+ } else {
+ mHandler.post(() -> rollbackAll(rollbackReason));
+ }
+
+ // Assume rollbacks executed successfully
+ return true;
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public boolean isPersistent() {
+ return true;
+ }
+
+ @Override
+ public boolean mayObservePackage(String packageName) {
+ if (mContext.getSystemService(RollbackManager.class)
+ .getAvailableRollbacks().isEmpty()) {
+ return false;
+ }
+ return isPersistentSystemApp(packageName);
+ }
+
+ private boolean isPersistentSystemApp(@NonNull String packageName) {
+ PackageManager pm = mContext.getPackageManager();
+ try {
+ ApplicationInfo info = pm.getApplicationInfo(packageName, 0);
+ return (info.flags & PERSISTENT_MASK) == PERSISTENT_MASK;
+ } catch (PackageManager.NameNotFoundException e) {
+ return false;
+ }
+ }
+
+ private void assertInWorkerThread() {
+ Preconditions.checkState(mHandler.getLooper().isCurrentThread());
+ }
+
+ /**
+ * Start observing health of {@code packages} for {@code durationMs}.
+ * This may cause {@code packages} to be rolled back if they crash too freqeuntly.
+ */
+ @AnyThread
+ void startObservingHealth(List<String> packages, long durationMs) {
+ PackageWatchdog.getInstance(mContext).startObservingHealth(this, packages, durationMs);
+ }
+
+ @AnyThread
+ void notifyRollbackAvailable(RollbackInfo rollback) {
+ mHandler.post(() -> {
+ // Enable two-phase rollback when a rebootless apex rollback is made available.
+ // We assume the rebootless apex is stable and is less likely to be the cause
+ // if native crash doesn't happen before reboot. So we will clear the flag and disable
+ // two-phase rollback after reboot.
+ if (isRebootlessApex(rollback)) {
+ mTwoPhaseRollbackEnabled = true;
+ writeBoolean(mTwoPhaseRollbackEnabledFile, true);
+ }
+ });
+ }
+
+ private static boolean isRebootlessApex(RollbackInfo rollback) {
+ if (!rollback.isStaged()) {
+ for (PackageRollbackInfo info : rollback.getPackages()) {
+ if (info.isApex()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /** Verifies the rollback state after a reboot and schedules polling for sometime after reboot
+ * to check for native crashes and mitigate them if needed.
+ */
+ @AnyThread
+ void onBootCompletedAsync() {
+ mHandler.post(()->onBootCompleted());
+ }
+
+ @WorkerThread
+ private void onBootCompleted() {
+ assertInWorkerThread();
+
+ RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);
+ if (!rollbackManager.getAvailableRollbacks().isEmpty()) {
+ // TODO(gavincorkery): Call into Package Watchdog from outside the observer
+ PackageWatchdog.getInstance(mContext).scheduleCheckAndMitigateNativeCrashes();
+ }
+
+ SparseArray<String> rollbackIds = popLastStagedRollbackIds();
+ for (int i = 0; i < rollbackIds.size(); i++) {
+ WatchdogRollbackLogger.logRollbackStatusOnBoot(mContext,
+ rollbackIds.keyAt(i), rollbackIds.valueAt(i),
+ rollbackManager.getRecentlyCommittedRollbacks());
+ }
+ }
+
+ @AnyThread
+ private RollbackInfo getAvailableRollback(VersionedPackage failedPackage) {
+ RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);
+ for (RollbackInfo rollback : rollbackManager.getAvailableRollbacks()) {
+ for (PackageRollbackInfo packageRollback : rollback.getPackages()) {
+ if (packageRollback.getVersionRolledBackFrom().equals(failedPackage)) {
+ return rollback;
+ }
+ // TODO(b/147666157): Extract version number of apk-in-apex so that we don't have
+ // to rely on complicated reasoning as below
+
+ // Due to b/147666157, for apk in apex, we do not know the version we are rolling
+ // back from. But if a package X is embedded in apex A exclusively (not embedded in
+ // any other apex), which is not guaranteed, then it is sufficient to check only
+ // package names here, as the version of failedPackage and the PackageRollbackInfo
+ // can't be different. If failedPackage has a higher version, then it must have
+ // been updated somehow. There are two ways: it was updated by an update of apex A
+ // or updated directly as apk. In both cases, this rollback would have gotten
+ // expired when onPackageReplaced() was called. Since the rollback exists, it has
+ // same version as failedPackage.
+ if (packageRollback.isApkInApex()
+ && packageRollback.getVersionRolledBackFrom().getPackageName()
+ .equals(failedPackage.getPackageName())) {
+ return rollback;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns {@code true} if staged session associated with {@code rollbackId} was marked
+ * as handled, {@code false} if already handled.
+ */
+ @WorkerThread
+ private boolean markStagedSessionHandled(int rollbackId) {
+ assertInWorkerThread();
+ return mPendingStagedRollbackIds.remove(rollbackId);
+ }
+
+ /**
+ * Returns {@code true} if all pending staged rollback sessions were marked as handled,
+ * {@code false} if there is any left.
+ */
+ @WorkerThread
+ private boolean isPendingStagedSessionsEmpty() {
+ assertInWorkerThread();
+ return mPendingStagedRollbackIds.isEmpty();
+ }
+
+ private static boolean readBoolean(File file) {
+ try (FileInputStream fis = new FileInputStream(file)) {
+ return fis.read() == 1;
+ } catch (IOException ignore) {
+ return false;
+ }
+ }
+
+ private static void writeBoolean(File file, boolean value) {
+ try (FileOutputStream fos = new FileOutputStream(file)) {
+ fos.write(value ? 1 : 0);
+ fos.flush();
+ FileUtils.sync(fos);
+ } catch (IOException ignore) {
+ }
+ }
+
+ @WorkerThread
+ private void saveStagedRollbackId(int stagedRollbackId, @Nullable VersionedPackage logPackage) {
+ assertInWorkerThread();
+ writeStagedRollbackId(mLastStagedRollbackIdsFile, stagedRollbackId, logPackage);
+ }
+
+ static void writeStagedRollbackId(File file, int stagedRollbackId,
+ @Nullable VersionedPackage logPackage) {
+ try {
+ FileOutputStream fos = new FileOutputStream(file, true);
+ PrintWriter pw = new PrintWriter(fos);
+ String logPackageName = logPackage != null ? logPackage.getPackageName() : "";
+ pw.append(String.valueOf(stagedRollbackId)).append(",").append(logPackageName);
+ pw.println();
+ pw.flush();
+ FileUtils.sync(fos);
+ pw.close();
+ } catch (IOException e) {
+ Slog.e(TAG, "Failed to save last staged rollback id", e);
+ file.delete();
+ }
+ }
+
+ @WorkerThread
+ private SparseArray<String> popLastStagedRollbackIds() {
+ assertInWorkerThread();
+ try {
+ return readStagedRollbackIds(mLastStagedRollbackIdsFile);
+ } finally {
+ mLastStagedRollbackIdsFile.delete();
+ }
+ }
+
+ static SparseArray<String> readStagedRollbackIds(File file) {
+ SparseArray<String> result = new SparseArray<>();
+ try {
+ String line;
+ BufferedReader reader = new BufferedReader(new FileReader(file));
+ while ((line = reader.readLine()) != null) {
+ // Each line is of the format: "id,logging_package"
+ String[] values = line.trim().split(",");
+ String rollbackId = values[0];
+ String logPackageName = "";
+ if (values.length > 1) {
+ logPackageName = values[1];
+ }
+ result.put(Integer.parseInt(rollbackId), logPackageName);
+ }
+ } catch (Exception ignore) {
+ return new SparseArray<>();
+ }
+ return result;
+ }
+
+
+ /**
+ * Returns true if the package name is the name of a module.
+ */
+ @AnyThread
+ private boolean isModule(String packageName) {
+ // Check if the package is an APK inside an APEX. If it is, use the parent APEX package when
+ // querying PackageManager.
+ String apexPackageName = mApexManager.getActiveApexPackageNameContainingPackage(
+ packageName);
+ if (apexPackageName != null) {
+ packageName = apexPackageName;
+ }
+
+ PackageManager pm = mContext.getPackageManager();
+ try {
+ return pm.getModuleInfo(packageName, 0) != null;
+ } catch (PackageManager.NameNotFoundException ignore) {
+ return false;
+ }
+ }
+
+ /**
+ * Rolls back the session that owns {@code failedPackage}
+ *
+ * @param rollback {@code rollbackInfo} of the {@code failedPackage}
+ * @param failedPackage the package that needs to be rolled back
+ */
+ @WorkerThread
+ private void rollbackPackage(RollbackInfo rollback, VersionedPackage failedPackage,
+ @FailureReasons int rollbackReason) {
+ assertInWorkerThread();
+
+ if (isAutomaticRollbackDenied(SystemConfig.getInstance(), failedPackage)) {
+ Slog.d(TAG, "Automatic rollback not allowed for package "
+ + failedPackage.getPackageName());
+ return;
+ }
+
+ final RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);
+ int reasonToLog = WatchdogRollbackLogger.mapFailureReasonToMetric(rollbackReason);
+ final String failedPackageToLog;
+ if (rollbackReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) {
+ failedPackageToLog = SystemProperties.get(
+ "sys.init.updatable_crashing_process_name", "");
+ } else {
+ failedPackageToLog = failedPackage.getPackageName();
+ }
+ VersionedPackage logPackageTemp = null;
+ if (isModule(failedPackage.getPackageName())) {
+ logPackageTemp = WatchdogRollbackLogger.getLogPackage(mContext, failedPackage);
+ }
+
+ final VersionedPackage logPackage = logPackageTemp;
+ WatchdogRollbackLogger.logEvent(logPackage,
+ FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_INITIATE,
+ reasonToLog, failedPackageToLog);
+
+ Consumer<Intent> onResult = result -> {
+ assertInWorkerThread();
+ int status = result.getIntExtra(RollbackManager.EXTRA_STATUS,
+ RollbackManager.STATUS_FAILURE);
+ if (status == RollbackManager.STATUS_SUCCESS) {
+ if (rollback.isStaged()) {
+ int rollbackId = rollback.getRollbackId();
+ saveStagedRollbackId(rollbackId, logPackage);
+ WatchdogRollbackLogger.logEvent(logPackage,
+ FrameworkStatsLog
+ .WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_BOOT_TRIGGERED,
+ reasonToLog, failedPackageToLog);
+
+ } else {
+ WatchdogRollbackLogger.logEvent(logPackage,
+ FrameworkStatsLog
+ .WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_SUCCESS,
+ reasonToLog, failedPackageToLog);
+ }
+ } else {
+ WatchdogRollbackLogger.logEvent(logPackage,
+ FrameworkStatsLog
+ .WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE,
+ reasonToLog, failedPackageToLog);
+ }
+ if (rollback.isStaged()) {
+ markStagedSessionHandled(rollback.getRollbackId());
+ // Wait for all pending staged sessions to get handled before rebooting.
+ if (isPendingStagedSessionsEmpty()) {
+ SystemProperties.set(PROP_ATTEMPTING_REBOOT, "true");
+ mContext.getSystemService(PowerManager.class).reboot("Rollback staged install");
+ }
+ }
+ };
+
+ final LocalIntentReceiver rollbackReceiver = new LocalIntentReceiver(result -> {
+ mHandler.post(() -> onResult.accept(result));
+ });
+
+ rollbackManager.commitRollback(rollback.getRollbackId(),
+ Collections.singletonList(failedPackage), rollbackReceiver.getIntentSender());
+ }
+
+ /**
+ * Returns true if this package is not eligible for automatic rollback.
+ */
+ @VisibleForTesting
+ @AnyThread
+ public static boolean isAutomaticRollbackDenied(SystemConfig systemConfig,
+ VersionedPackage versionedPackage) {
+ return systemConfig.getAutomaticRollbackDenylistedPackages()
+ .contains(versionedPackage.getPackageName());
+ }
+
+ /**
+ * Two-phase rollback:
+ * 1. roll back rebootless apexes first
+ * 2. roll back all remaining rollbacks if native crash doesn't stop after (1) is done
+ *
+ * This approach gives us a better chance to correctly attribute native crash to rebootless
+ * apex update without rolling back Mainline updates which might contains critical security
+ * fixes.
+ */
+ @WorkerThread
+ private boolean useTwoPhaseRollback(List<RollbackInfo> rollbacks) {
+ assertInWorkerThread();
+ if (!mTwoPhaseRollbackEnabled) {
+ return false;
+ }
+
+ Slog.i(TAG, "Rolling back all rebootless APEX rollbacks");
+ boolean found = false;
+ for (RollbackInfo rollback : rollbacks) {
+ if (isRebootlessApex(rollback)) {
+ VersionedPackage sample = rollback.getPackages().get(0).getVersionRolledBackFrom();
+ rollbackPackage(rollback, sample, PackageWatchdog.FAILURE_REASON_NATIVE_CRASH);
+ found = true;
+ }
+ }
+ return found;
+ }
+
+ @WorkerThread
+ private void rollbackAll(@FailureReasons int rollbackReason) {
+ assertInWorkerThread();
+ RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);
+ List<RollbackInfo> rollbacks = rollbackManager.getAvailableRollbacks();
+ if (useTwoPhaseRollback(rollbacks)) {
+ return;
+ }
+
+ Slog.i(TAG, "Rolling back all available rollbacks");
+ // Add all rollback ids to mPendingStagedRollbackIds, so that we do not reboot before all
+ // pending staged rollbacks are handled.
+ for (RollbackInfo rollback : rollbacks) {
+ if (rollback.isStaged()) {
+ mPendingStagedRollbackIds.add(rollback.getRollbackId());
+ }
+ }
+
+ for (RollbackInfo rollback : rollbacks) {
+ VersionedPackage sample = rollback.getPackages().get(0).getVersionRolledBackFrom();
+ rollbackPackage(rollback, sample, rollbackReason);
+ }
+ }
+}
diff --git a/packages/CrashRecovery/services/java/com/android/server/rollback/WatchdogRollbackLogger.java b/packages/CrashRecovery/services/java/com/android/server/rollback/WatchdogRollbackLogger.java
new file mode 100644
index 0000000..f9ef994
--- /dev/null
+++ b/packages/CrashRecovery/services/java/com/android/server/rollback/WatchdogRollbackLogger.java
@@ -0,0 +1,297 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.rollback;
+
+import static com.android.internal.util.FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_CRASH;
+import static com.android.internal.util.FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_NOT_RESPONDING;
+import static com.android.internal.util.FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_EXPLICIT_HEALTH_CHECK;
+import static com.android.internal.util.FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_NATIVE_CRASH;
+import static com.android.internal.util.FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_NATIVE_CRASH_DURING_BOOT;
+import static com.android.internal.util.FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN;
+import static com.android.internal.util.FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_BOOT_TRIGGERED;
+import static com.android.internal.util.FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE;
+import static com.android.internal.util.FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_INITIATE;
+import static com.android.internal.util.FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_SUCCESS;
+
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.content.Context;
+import android.content.pm.ApplicationInfo;
+import android.content.pm.PackageInstaller;
+import android.content.pm.PackageManager;
+import android.content.pm.VersionedPackage;
+import android.content.rollback.PackageRollbackInfo;
+import android.content.rollback.RollbackInfo;
+import android.os.SystemProperties;
+import android.text.TextUtils;
+import android.util.ArraySet;
+import android.util.Slog;
+
+import com.android.internal.annotations.VisibleForTesting;
+import com.android.internal.util.FrameworkStatsLog;
+import com.android.server.PackageWatchdog;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This class handles the logic for logging Watchdog-triggered rollback events.
+ */
+public final class WatchdogRollbackLogger {
+ private static final String TAG = "WatchdogRollbackLogger";
+
+ private static final String LOGGING_PARENT_KEY = "android.content.pm.LOGGING_PARENT";
+
+ private WatchdogRollbackLogger() {
+ }
+
+ @Nullable
+ private static String getLoggingParentName(Context context, @NonNull String packageName) {
+ PackageManager packageManager = context.getPackageManager();
+ try {
+ int flags = PackageManager.MATCH_APEX | PackageManager.GET_META_DATA;
+ ApplicationInfo ai = packageManager.getPackageInfo(packageName, flags).applicationInfo;
+ if (ai.metaData == null) {
+ return null;
+ }
+ return ai.metaData.getString(LOGGING_PARENT_KEY);
+ } catch (Exception e) {
+ Slog.w(TAG, "Unable to discover logging parent package: " + packageName, e);
+ return null;
+ }
+ }
+
+ /**
+ * Returns the logging parent of a given package if it exists, {@code null} otherwise.
+ *
+ * The logging parent is defined by the {@code android.content.pm.LOGGING_PARENT} field in the
+ * metadata of a package's AndroidManifest.xml.
+ */
+ @VisibleForTesting
+ @Nullable
+ static VersionedPackage getLogPackage(Context context,
+ @NonNull VersionedPackage failingPackage) {
+ String logPackageName;
+ VersionedPackage loggingParent;
+ logPackageName = getLoggingParentName(context, failingPackage.getPackageName());
+ if (logPackageName == null) {
+ return null;
+ }
+ try {
+ loggingParent = new VersionedPackage(logPackageName, context.getPackageManager()
+ .getPackageInfo(logPackageName, 0 /* flags */).getLongVersionCode());
+ } catch (PackageManager.NameNotFoundException e) {
+ return null;
+ }
+ return loggingParent;
+ }
+
+
+ /**
+ * Gets the set of parent packages for a given set of failed package names. In the case that
+ * multiple sessions have failed, we want to log failure for each of the parent packages.
+ * Even if multiple failed packages have the same parent, we only log the parent package once.
+ */
+ private static Set<VersionedPackage> getLogPackages(Context context,
+ @NonNull List<String> failedPackageNames) {
+ Set<VersionedPackage> parentPackages = new ArraySet<>();
+ for (String failedPackageName: failedPackageNames) {
+ parentPackages.add(getLogPackage(context, new VersionedPackage(failedPackageName, 0)));
+ }
+ return parentPackages;
+ }
+
+
+ static void logRollbackStatusOnBoot(Context context, int rollbackId, String logPackageName,
+ List<RollbackInfo> recentlyCommittedRollbacks) {
+ PackageInstaller packageInstaller = context.getPackageManager().getPackageInstaller();
+
+ RollbackInfo rollback = null;
+ for (RollbackInfo info : recentlyCommittedRollbacks) {
+ if (rollbackId == info.getRollbackId()) {
+ rollback = info;
+ break;
+ }
+ }
+
+ if (rollback == null) {
+ Slog.e(TAG, "rollback info not found for last staged rollback: " + rollbackId);
+ return;
+ }
+
+ // Use the version of the logging parent that was installed before
+ // we rolled back for logging purposes.
+ VersionedPackage oldLoggingPackage = null;
+ if (!TextUtils.isEmpty(logPackageName)) {
+ for (PackageRollbackInfo packageRollback : rollback.getPackages()) {
+ if (logPackageName.equals(packageRollback.getPackageName())) {
+ oldLoggingPackage = packageRollback.getVersionRolledBackFrom();
+ break;
+ }
+ }
+ }
+
+ int sessionId = rollback.getCommittedSessionId();
+ PackageInstaller.SessionInfo sessionInfo = packageInstaller.getSessionInfo(sessionId);
+ if (sessionInfo == null) {
+ Slog.e(TAG, "On boot completed, could not load session id " + sessionId);
+ return;
+ }
+
+ if (sessionInfo.isStagedSessionApplied()) {
+ logEvent(oldLoggingPackage,
+ WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_SUCCESS,
+ WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN, "");
+ } else if (sessionInfo.isStagedSessionFailed()) {
+ logEvent(oldLoggingPackage,
+ WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE,
+ WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN, "");
+ }
+ }
+
+ /**
+ * Logs that one or more apexd reverts have occurred, along with the crashing native process
+ * that caused apexd to revert during boot.
+ *
+ * @param context the context to use when determining the log packages
+ * @param failedPackageNames a list of names of packages which were reverted
+ * @param failingNativeProcess the crashing native process which caused a revert
+ */
+ public static void logApexdRevert(Context context, @NonNull List<String> failedPackageNames,
+ @NonNull String failingNativeProcess) {
+ Set<VersionedPackage> logPackages = getLogPackages(context, failedPackageNames);
+ for (VersionedPackage logPackage: logPackages) {
+ logEvent(logPackage,
+ WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_SUCCESS,
+ WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_NATIVE_CRASH_DURING_BOOT,
+ failingNativeProcess);
+ }
+ }
+
+ /**
+ * Log a Watchdog rollback event to statsd.
+ *
+ * @param logPackage the package to associate the rollback with.
+ * @param type the state of the rollback.
+ * @param rollbackReason the reason Watchdog triggered a rollback, if known.
+ * @param failingPackageName the failing package or process which triggered the rollback.
+ */
+ public static void logEvent(@Nullable VersionedPackage logPackage, int type,
+ int rollbackReason, @NonNull String failingPackageName) {
+ Slog.i(TAG, "Watchdog event occurred with type: " + rollbackTypeToString(type)
+ + " logPackage: " + logPackage
+ + " rollbackReason: " + rollbackReasonToString(rollbackReason)
+ + " failedPackageName: " + failingPackageName);
+ if (logPackage != null) {
+ FrameworkStatsLog.write(
+ FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED,
+ type,
+ logPackage.getPackageName(),
+ logPackage.getVersionCode(),
+ rollbackReason,
+ failingPackageName,
+ new byte[]{});
+ } else {
+ // In the case that the log package is null, still log an empty string as an
+ // indication that retrieving the logging parent failed.
+ FrameworkStatsLog.write(
+ FrameworkStatsLog.WATCHDOG_ROLLBACK_OCCURRED,
+ type,
+ "",
+ 0,
+ rollbackReason,
+ failingPackageName,
+ new byte[]{});
+ }
+
+ logTestProperties(logPackage, type, rollbackReason, failingPackageName);
+ }
+
+ /**
+ * Writes properties which will be used by rollback tests to check if particular rollback
+ * events have occurred.
+ *
+ * persist.sys.rollbacktest.enabled: true if rollback tests are running
+ * persist.sys.rollbacktest.EVENT_TYPE: true if a particular rollback event has occurred
+ * ex: persist.sys.rollbacktest.ROLLBACK_INITIATE is true if ROLLBACK_INITIATE has happened
+ * persist.sys.rollbacktest.EVENT_TYPE.logPackage: the package to associate the rollback with
+ * persist.sys.rollbacktest.EVENT_TYPE.rollbackReason: the reason Watchdog triggered a rollback
+ * persist.sys.rollbacktest.EVENT_TYPE.failedPackageName: the failing package or process which
+ * triggered the rollback
+ */
+ private static void logTestProperties(@Nullable VersionedPackage logPackage, int type,
+ int rollbackReason, @NonNull String failingPackageName) {
+ // This property should be on only during the tests
+ final String prefix = "persist.sys.rollbacktest.";
+ if (!SystemProperties.getBoolean(prefix + "enabled", false)) {
+ return;
+ }
+ String key = prefix + rollbackTypeToString(type);
+ SystemProperties.set(key, String.valueOf(true));
+ SystemProperties.set(key + ".logPackage", logPackage != null ? logPackage.toString() : "");
+ SystemProperties.set(key + ".rollbackReason", rollbackReasonToString(rollbackReason));
+ SystemProperties.set(key + ".failedPackageName", failingPackageName);
+ }
+
+ @VisibleForTesting
+ static int mapFailureReasonToMetric(@PackageWatchdog.FailureReasons int failureReason) {
+ switch (failureReason) {
+ case PackageWatchdog.FAILURE_REASON_NATIVE_CRASH:
+ return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_NATIVE_CRASH;
+ case PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK:
+ return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_EXPLICIT_HEALTH_CHECK;
+ case PackageWatchdog.FAILURE_REASON_APP_CRASH:
+ return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_CRASH;
+ case PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING:
+ return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_NOT_RESPONDING;
+ default:
+ return WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_UNKNOWN;
+ }
+ }
+
+ private static String rollbackTypeToString(int type) {
+ switch (type) {
+ case WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_INITIATE:
+ return "ROLLBACK_INITIATE";
+ case WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_SUCCESS:
+ return "ROLLBACK_SUCCESS";
+ case WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_FAILURE:
+ return "ROLLBACK_FAILURE";
+ case WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_TYPE__ROLLBACK_BOOT_TRIGGERED:
+ return "ROLLBACK_BOOT_TRIGGERED";
+ default:
+ return "UNKNOWN";
+ }
+ }
+
+ private static String rollbackReasonToString(int reason) {
+ switch (reason) {
+ case WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_NATIVE_CRASH:
+ return "REASON_NATIVE_CRASH";
+ case WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_EXPLICIT_HEALTH_CHECK:
+ return "REASON_EXPLICIT_HEALTH_CHECK";
+ case WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_CRASH:
+ return "REASON_APP_CRASH";
+ case WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_APP_NOT_RESPONDING:
+ return "REASON_APP_NOT_RESPONDING";
+ case WATCHDOG_ROLLBACK_OCCURRED__ROLLBACK_REASON__REASON_NATIVE_CRASH_DURING_BOOT:
+ return "REASON_NATIVE_CRASH_DURING_BOOT";
+ default:
+ return "UNKNOWN";
+ }
+ }
+}