Merge "Update logs for CrashRecovery Module" into main
diff --git a/services/core/java/com/android/server/PackageWatchdog.java b/services/core/java/com/android/server/PackageWatchdog.java
index 47203fb..fbe593f 100644
--- a/services/core/java/com/android/server/PackageWatchdog.java
+++ b/services/core/java/com/android/server/PackageWatchdog.java
@@ -20,6 +20,8 @@
import static android.content.Intent.ACTION_SHUTDOWN;
import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;
+import static com.android.server.crashrecovery.CrashRecoveryUtils.dumpCrashRecoveryEvents;
+
import static java.lang.annotation.RetentionPolicy.SOURCE;
import android.annotation.IntDef;
@@ -44,6 +46,7 @@
import android.util.ArrayMap;
import android.util.ArraySet;
import android.util.AtomicFile;
+import android.util.IndentingPrintWriter;
import android.util.LongArrayQueue;
import android.util.Slog;
import android.util.Xml;
@@ -51,7 +54,6 @@
import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.os.BackgroundThread;
-import com.android.internal.util.IndentingPrintWriter;
import com.android.internal.util.XmlUtils;
import com.android.modules.utils.TypedXmlPullParser;
import com.android.modules.utils.TypedXmlSerializer;
@@ -72,6 +74,7 @@
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.io.PrintWriter;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
@@ -1265,18 +1268,21 @@
/** Dump status of every observer in mAllObservers. */
- public void dump(IndentingPrintWriter pw) {
- pw.println("Package Watchdog status");
- pw.increaseIndent();
+ public void dump(PrintWriter pw) {
+ IndentingPrintWriter ipw = new IndentingPrintWriter(pw, " ");
+ ipw.println("Package Watchdog status");
+ ipw.increaseIndent();
synchronized (mLock) {
for (String observerName : mAllObservers.keySet()) {
- pw.println("Observer name: " + observerName);
- pw.increaseIndent();
+ ipw.println("Observer name: " + observerName);
+ ipw.increaseIndent();
ObserverInternal observerInternal = mAllObservers.get(observerName);
- observerInternal.dump(pw);
- pw.decreaseIndent();
+ observerInternal.dump(ipw);
+ ipw.decreaseIndent();
}
}
+ ipw.decreaseIndent();
+ dumpCrashRecoveryEvents(ipw);
}
@VisibleForTesting
diff --git a/services/core/java/com/android/server/RescueParty.java b/services/core/java/com/android/server/RescueParty.java
index bba97fa..cadceb5 100644
--- a/services/core/java/com/android/server/RescueParty.java
+++ b/services/core/java/com/android/server/RescueParty.java
@@ -18,7 +18,7 @@
import static android.provider.DeviceConfig.Properties;
-import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo;
+import static com.android.server.crashrecovery.CrashRecoveryUtils.logCrashRecoveryEvent;
import android.annotation.IntDef;
import android.annotation.NonNull;
@@ -291,13 +291,13 @@
Properties properties = new Properties.Builder(namespaceToReset).build();
try {
if (!DeviceConfig.setProperties(properties)) {
- logCriticalInfo(Log.ERROR, "Failed to clear properties under "
+ logCrashRecoveryEvent(Log.ERROR, "Failed to clear properties under "
+ namespaceToReset
+ ". Running `device_config get_sync_disabled_for_tests` will confirm"
+ " if config-bulk-update is enabled.");
}
} catch (DeviceConfig.BadConfigException exception) {
- logCriticalInfo(Log.WARN, "namespace " + namespaceToReset
+ logCrashRecoveryEvent(Log.WARN, "namespace " + namespaceToReset
+ " is already banned, skip reset.");
}
}
@@ -528,7 +528,7 @@
if (!TextUtils.isEmpty(failedPackage)) {
successMsg += " for package " + failedPackage;
}
- logCriticalInfo(Log.DEBUG, successMsg);
+ logCrashRecoveryEvent(Log.DEBUG, successMsg);
} catch (Throwable t) {
logRescueException(level, failedPackage, t);
}
@@ -687,7 +687,7 @@
if (!TextUtils.isEmpty(failedPackageName)) {
failureMsg += " for package " + failedPackageName;
}
- logCriticalInfo(Log.ERROR, failureMsg + ": " + msg);
+ logCrashRecoveryEvent(Log.ERROR, failureMsg + ": " + msg);
}
private static int mapRescueLevelToUserImpact(int rescueLevel) {
diff --git a/services/core/java/com/android/server/crashrecovery/CrashRecoveryUtils.java b/services/core/java/com/android/server/crashrecovery/CrashRecoveryUtils.java
new file mode 100644
index 0000000..3eb3380
--- /dev/null
+++ b/services/core/java/com/android/server/crashrecovery/CrashRecoveryUtils.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.crashrecovery;
+
+import android.os.Environment;
+import android.util.IndentingPrintWriter;
+import android.util.Slog;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+
+/**
+ * Class containing helper methods for the CrashRecoveryModule.
+ *
+ * @hide
+ */
+public class CrashRecoveryUtils {
+ private static final String TAG = "CrashRecoveryUtils";
+ private static final long MAX_CRITICAL_INFO_DUMP_SIZE = 1000 * 1000; // ~1MB
+ private static final Object sFileLock = new Object();
+
+ /** Persist recovery related events in crashrecovery events file.**/
+ public static void logCrashRecoveryEvent(int priority, String msg) {
+ Slog.println(priority, TAG, msg);
+ try {
+ File fname = getCrashRecoveryEventsFile();
+ synchronized (sFileLock) {
+ FileOutputStream out = new FileOutputStream(fname, true);
+ PrintWriter pw = new PrintWriter(out);
+ String dateString = LocalDateTime.now(ZoneId.systemDefault()).toString();
+ pw.println(dateString + ": " + msg);
+ pw.close();
+ }
+ } catch (IOException e) {
+ Slog.e(TAG, "Unable to log CrashRecoveryEvents " + e.getMessage());
+ }
+ }
+
+ /** Dump recovery related events from crashrecovery events file.**/
+ public static void dumpCrashRecoveryEvents(IndentingPrintWriter pw) {
+ pw.println("CrashRecovery Events: ");
+ pw.increaseIndent();
+ final File file = getCrashRecoveryEventsFile();
+ final long skipSize = file.length() - MAX_CRITICAL_INFO_DUMP_SIZE;
+ synchronized (sFileLock) {
+ try (BufferedReader in = new BufferedReader(new FileReader(file))) {
+ if (skipSize > 0) {
+ in.skip(skipSize);
+ }
+ String line;
+ while ((line = in.readLine()) != null) {
+ pw.println(line);
+ }
+ } catch (IOException e) {
+ Slog.e(TAG, "Unable to dump CrashRecoveryEvents " + e.getMessage());
+ }
+ }
+ pw.decreaseIndent();
+ }
+
+ private static File getCrashRecoveryEventsFile() {
+ File systemDir = new File(Environment.getDataDirectory(), "system");
+ return new File(systemDir, "crashrecovery-events.txt");
+ }
+}
diff --git a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java
index 1c786e6..68026ea 100644
--- a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java
+++ b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java
@@ -18,6 +18,8 @@
import static android.content.pm.Flags.provideInfoOfApkInApex;
+import static com.android.server.crashrecovery.CrashRecoveryUtils.logCrashRecoveryEvent;
+
import android.annotation.AnyThread;
import android.annotation.NonNull;
import android.annotation.Nullable;
@@ -40,6 +42,7 @@
import android.os.SystemProperties;
import android.sysprop.CrashRecoveryProperties;
import android.util.ArraySet;
+import android.util.Log;
import android.util.Slog;
import android.util.SparseArray;
@@ -532,11 +535,13 @@
private void rollbackPackage(RollbackInfo rollback, VersionedPackage failedPackage,
@FailureReasons int rollbackReason) {
assertInWorkerThread();
+ String failedPackageName = (failedPackage == null ? null : failedPackage.getPackageName());
Slog.i(TAG, "Rolling back package. RollbackId: " + rollback.getRollbackId()
- + " failedPackage: "
- + (failedPackage == null ? null : failedPackage.getPackageName())
+ + " failedPackage: " + failedPackageName
+ " rollbackReason: " + rollbackReason);
+ logCrashRecoveryEvent(Log.DEBUG, String.format("Rolling back %s. Reason: %s",
+ failedPackageName, rollbackReason));
final RollbackManager rollbackManager = mContext.getSystemService(RollbackManager.class);
int reasonToLog = WatchdogRollbackLogger.mapFailureReasonToMetric(rollbackReason);
final String failedPackageToLog;
@@ -724,6 +729,7 @@
}
Slog.i(TAG, "Rolling back all available low impact rollbacks");
+ logCrashRecoveryEvent(Log.DEBUG, "Rolling back all available. Reason: " + rollbackReason);
// Add all rollback ids to mPendingStagedRollbackIds, so that we do not reboot before all
// pending staged rollbacks are handled.
for (RollbackInfo rollback : lowImpactRollbacks) {
diff --git a/services/tests/mockingservicestests/src/com/android/server/crashrecovery/CrashRecoveryUtilsTest.java b/services/tests/mockingservicestests/src/com/android/server/crashrecovery/CrashRecoveryUtilsTest.java
new file mode 100644
index 0000000..6f38fca
--- /dev/null
+++ b/services/tests/mockingservicestests/src/com/android/server/crashrecovery/CrashRecoveryUtilsTest.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.crashrecovery;
+
+
+
+import static com.google.common.truth.Truth.assertThat;
+
+import static org.mockito.quality.Strictness.LENIENT;
+import static org.junit.Assert.fail;
+
+import android.content.Context;
+import android.os.Environment;
+import android.util.IndentingPrintWriter;
+import android.util.Log;
+
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.runner.AndroidJUnit4;
+
+import com.android.dx.mockito.inline.extended.ExtendedMockito;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.MockitoSession;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+
+
+/**
+ * Test CrashRecovery Utils.
+ */
+@RunWith(AndroidJUnit4.class)
+public class CrashRecoveryUtilsTest {
+
+ private MockitoSession mStaticMockSession;
+ private final String mLogMsg = "Logging from test";
+ private final String mCrashrecoveryEventTag = "CrashRecovery Events: ";
+ private File mCacheDir;
+
+ @Before
+ public void setup() throws IOException {
+ Context context = ApplicationProvider.getApplicationContext();
+ mCacheDir = context.getCacheDir();
+ mStaticMockSession = ExtendedMockito.mockitoSession()
+ .spyStatic(Environment.class)
+ .strictness(LENIENT)
+ .startMocking();
+ ExtendedMockito.doReturn(mCacheDir).when(() -> Environment.getDataDirectory());
+
+ createCrashRecoveryEventsTempDir();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ mStaticMockSession.finishMocking();
+ deleteCrashRecoveryEventsTempFile();
+ }
+
+ @Test
+ public void testCrashRecoveryUtils() {
+ testLogCrashRecoveryEvent();
+ testDumpCrashRecoveryEvents();
+ }
+
+ @Test
+ public void testDumpCrashRecoveryEventsWithoutAnyLogs() {
+ assertThat(getCrashRecoveryEventsTempFile().exists()).isFalse();
+ StringWriter sw = new StringWriter();
+ IndentingPrintWriter ipw = new IndentingPrintWriter(sw, " ");
+ CrashRecoveryUtils.dumpCrashRecoveryEvents(ipw);
+ ipw.close();
+
+ String dump = sw.getBuffer().toString();
+ assertThat(dump).contains(mCrashrecoveryEventTag);
+ assertThat(dump).doesNotContain(mLogMsg);
+ }
+
+ private void testLogCrashRecoveryEvent() {
+ assertThat(getCrashRecoveryEventsTempFile().exists()).isFalse();
+ CrashRecoveryUtils.logCrashRecoveryEvent(Log.WARN, mLogMsg);
+
+ assertThat(getCrashRecoveryEventsTempFile().exists()).isTrue();
+ String fileContent = null;
+ try {
+ File file = getCrashRecoveryEventsTempFile();
+ FileInputStream fis = new FileInputStream(file);
+ byte[] data = new byte[(int) file.length()];
+ fis.read(data);
+ fis.close();
+ fileContent = new String(data, StandardCharsets.UTF_8);
+ } catch (Exception e) {
+ fail("Unable to read the events file");
+ }
+ assertThat(fileContent).contains(mLogMsg);
+ }
+
+ private void testDumpCrashRecoveryEvents() {
+ StringWriter sw = new StringWriter();
+ IndentingPrintWriter ipw = new IndentingPrintWriter(sw, " ");
+ CrashRecoveryUtils.dumpCrashRecoveryEvents(ipw);
+ ipw.close();
+
+ String dump = sw.getBuffer().toString();
+ assertThat(dump).contains(mCrashrecoveryEventTag);
+ assertThat(dump).contains(mLogMsg);
+ }
+
+ private void createCrashRecoveryEventsTempDir() throws IOException {
+ Files.deleteIfExists(getCrashRecoveryEventsTempFile().toPath());
+ File mMockDirectory = new File(mCacheDir, "system");
+ if (!mMockDirectory.exists()) {
+ assertThat(mMockDirectory.mkdir()).isTrue();
+ }
+ }
+
+ private void deleteCrashRecoveryEventsTempFile() throws IOException {
+ Files.deleteIfExists(getCrashRecoveryEventsTempFile().toPath());
+ }
+
+ private File getCrashRecoveryEventsTempFile() {
+ File systemTempDir = new File(mCacheDir, "system");
+ return new File(systemTempDir, "crashrecovery-events.txt");
+ }
+}