Automatically diagnosing known flakes
It often takes a long time to fix a flake; meanwhile it takes a lot of
time for sheriffs to identify whether to register a new flake for a
failure.
This CL adds automatic identifies identification of one know flake type.
Once identified, it rewrites the error so that:
1. Flakes clustering tool that has only rudimentary clustering, places
all flakes of this kind in the same cluster (not multiple ones like
now). This is a step towards using clustering tool for monitoring
flakes;
2. Sheriff immediately sees that the issue is known.
Change-Id: I86a0762665cb21434289e1be00b60bd76fec4142
diff --git a/tests/src/com/android/launcher3/util/rule/FailureInvestigator.java b/tests/src/com/android/launcher3/util/rule/FailureInvestigator.java
new file mode 100644
index 0000000..56c885d
--- /dev/null
+++ b/tests/src/com/android/launcher3/util/rule/FailureInvestigator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.launcher3.util.rule;
+
+import static androidx.test.InstrumentationRegistry.getInstrumentation;
+
+import androidx.test.uiautomator.UiDevice;
+
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+class FailureInvestigator {
+ private static boolean matches(String regex, CharSequence string) {
+ return Pattern.compile(regex).matcher(string).find();
+ }
+
+ static int getBugForFailure(CharSequence exception, String testsStartTime) {
+ final String logSinceTestsStart;
+ try {
+ logSinceTestsStart =
+ UiDevice.getInstance(getInstrumentation())
+ .executeShellCommand("logcat -d -t " + testsStartTime.replace(" ", ""));
+ } catch (IOException e) {
+ return 0;
+ }
+
+ if (matches(
+ "java.lang.AssertionError: http://go/tapl : Tests are broken by a non-Launcher "
+ + "system error: Phone is locked",
+ exception)) {
+ if (matches(
+ "BroadcastQueue: Can't deliver broadcast to com.android.systemui.*Crashing it",
+ logSinceTestsStart)) {
+ return 147845913;
+ }
+ }
+
+ return 0;
+ }
+}
diff --git a/tests/src/com/android/launcher3/util/rule/FailureWatcher.java b/tests/src/com/android/launcher3/util/rule/FailureWatcher.java
index cdda0f0..feb89b9 100644
--- a/tests/src/com/android/launcher3/util/rule/FailureWatcher.java
+++ b/tests/src/com/android/launcher3/util/rule/FailureWatcher.java
@@ -8,10 +8,13 @@
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
public class FailureWatcher extends TestWatcher {
private static final String TAG = "FailureWatcher";
@@ -35,6 +38,30 @@
}
}
+ private static final String testsStartTime =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(new Date());
+
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ try {
+ base.evaluate();
+ } catch (Throwable e) {
+ final int bug =
+ FailureInvestigator.getBugForFailure(e.toString(), testsStartTime);
+ if (bug == 0) throw e;
+
+ Log.e(TAG, "Known bug found for the original failure "
+ + android.util.Log.getStackTraceString(e));
+ throw new AssertionError(
+ "Detected a failure that matches a known bug b/" + bug);
+ }
+ }
+ };
+ }
+
@Override
protected void failed(Throwable e, Description description) {
onError(mDevice, description, e);