Fix flaky test & add func runOnHostRetryingOnFailure
A part of flakiness is because Microdroid is offline & so rootMicrodroid
fails. Retry in those cases. Examples of failures:
https://screenshot.googleplex.com/7TMmMy6FYhWihxc
http://screen/AKhyjQaprrTsSFa
See runOnHostRetryingOnFailure
Test: atest MicrodroidHostTestCases
Bug: 233718604
Change-Id: I58ade18c4784a3f2057e69cb52657d2410d6de41
diff --git a/tests/hostside/helper/java/android/virt/test/VirtualizationTestCaseBase.java b/tests/hostside/helper/java/android/virt/test/VirtualizationTestCaseBase.java
index a55ebe1..10d1e8e 100644
--- a/tests/hostside/helper/java/android/virt/test/VirtualizationTestCaseBase.java
+++ b/tests/hostside/helper/java/android/virt/test/VirtualizationTestCaseBase.java
@@ -59,6 +59,11 @@
private static final long MICRODROID_MAX_LIFETIME_MINUTES = 20;
private static final long MICRODROID_ADB_CONNECT_TIMEOUT_MINUTES = 5;
+ protected static final long MICRODROID_COMMAND_TIMEOUT_MILLIS = 30000;
+ private static final long MICRODROID_COMMAND_RETRY_INTERVAL_MILLIS = 500;
+ protected static final int MICRODROID_ADB_CONNECT_MAX_ATTEMPTS =
+ (int) (MICRODROID_ADB_CONNECT_TIMEOUT_MINUTES * 60 * 1000
+ / MICRODROID_COMMAND_RETRY_INTERVAL_MILLIS);
public static void prepareVirtualizationTestSetup(ITestDevice androidDevice)
throws DeviceNotAvailableException {
@@ -128,12 +133,25 @@
return result.getStdout().trim();
}
- // Same as runOnMicrodroid, but keeps retrying on error till timeout
- private static String runOnMicrodroidRetryingOnFailure(String... cmd) {
- final long timeoutMs = 30000; // 30 sec. Microdroid is extremely slow on GCE-on-CF.
- int attempts = (int) MICRODROID_ADB_CONNECT_TIMEOUT_MINUTES * 60 * 1000 / 500;
+ // Same as runOnHost, but keeps retrying on error for maximum attempts times
+ // Each attempt with timeoutMs
+ public static String runOnHostRetryingOnFailure(long timeoutMs, int attempts, String... cmd) {
CommandResult result = RunUtil.getDefault()
- .runTimedCmdRetry(timeoutMs, 500, attempts,
+ .runTimedCmdRetry(timeoutMs,
+ MICRODROID_COMMAND_RETRY_INTERVAL_MILLIS, attempts, cmd);
+ assertWithMessage("Command `" + cmd + "` has failed")
+ .about(command_results())
+ .that(result)
+ .isSuccess();
+ return result.getStdout().trim();
+ }
+
+ // Same as runOnMicrodroid, but keeps retrying on error for maximum attempts times
+ // Each attempt with timeoutMs
+ public static String runOnMicrodroidRetryingOnFailure(
+ long timeoutMs, int attempts, String... cmd) {
+ CommandResult result = RunUtil.getDefault()
+ .runTimedCmdRetry(timeoutMs, MICRODROID_COMMAND_RETRY_INTERVAL_MILLIS, attempts,
"adb", "-s", MICRODROID_SERIAL, "shell", join(cmd));
assertWithMessage("Command `" + cmd + "` has failed")
.about(command_results())
@@ -328,18 +346,14 @@
android.run(VIRT_APEX + "bin/vm", "stop", cid);
}
- public static void rootMicrodroid() {
- runOnHost("adb", "-s", MICRODROID_SERIAL, "root");
- runOnHostWithTimeout(
- MICRODROID_ADB_CONNECT_TIMEOUT_MINUTES * 60 * 1000,
- "adb",
- "-s",
- MICRODROID_SERIAL,
- "wait-for-device");
- // There have been tests when adb wait-for-device succeeded but the following command
- // fails with error: closed. Hence, we run adb shell true in microdroid with retries
+ public static void rootMicrodroid() throws InterruptedException {
+ runOnHostRetryingOnFailure(MICRODROID_COMMAND_TIMEOUT_MILLIS,
+ MICRODROID_ADB_CONNECT_MAX_ATTEMPTS, "adb", "-s", MICRODROID_SERIAL, "root");
+ // adbd reboots after root. Some commands (including wait-for-device) following this fails
+ // with error: closed. Hence, we run adb shell true in microdroid with retries
// before returning.
- runOnMicrodroidRetryingOnFailure("true");
+ runOnMicrodroidRetryingOnFailure(MICRODROID_COMMAND_TIMEOUT_MILLIS,
+ MICRODROID_ADB_CONNECT_MAX_ATTEMPTS, "true");
}
// Establish an adb connection to microdroid by letting Android forward the connection to
diff --git a/tests/hostside/java/android/virt/test/MicrodroidTestCase.java b/tests/hostside/java/android/virt/test/MicrodroidTestCase.java
index ec2afaa..1b41d07 100644
--- a/tests/hostside/java/android/virt/test/MicrodroidTestCase.java
+++ b/tests/hostside/java/android/virt/test/MicrodroidTestCase.java
@@ -384,13 +384,16 @@
Optional.of(CPU_AFFINITY));
adbConnectToMicrodroid(getDevice(), cid);
waitForBootComplete();
- runOnMicrodroid("logcat -c");
+ runOnMicrodroidRetryingOnFailure(MICRODROID_COMMAND_TIMEOUT_MILLIS,
+ MICRODROID_ADB_CONNECT_MAX_ATTEMPTS,
+ "logcat -c");
// We need root permission to write to /data/tombstones/
rootMicrodroid();
// Write a test tombstone file in /data/tombstones
runOnMicrodroid("echo -n \'Test tombstone in VM with 34 bytes\'"
+ "> /data/tombstones/transmit.txt");
- // check if the tombstone have been tranferred from VM
+ // check if the tombstone have been tranferred from VM. This is a bit flaky - increasing
+ // timeout to 30s can result in SIGKILL inside microdroid due to logcat memory issue
assertNotEquals(runOnMicrodroid("timeout 15s logcat | grep -m 1 "
+ "'tombstone_transmit.microdroid:.*data/tombstones/transmit.txt'"),
"");