Handle failures from Before/AfterClass in the runner
- Failures from @BeforeClass / @AfterClass are not properly handled
by tradefed (at least by IsolatedHostTest).
- So let's detect them and propagate failures as if each tests reported
the failure, which is a workaround that tradefed uses in a different
host test runner.
- Also, change the lambdas RavenwoodAwareTestRunner to inner classes,
so that the stacktrace would be less cryptic.
Flag: EXEMPT host test change only
Bug: 364395552
Test: $ANDROID_BUILD_TOP/frameworks/base/ravenwood/scripts/run-ravenwood-tests.sh
Change-Id: I55299d19cd07af2fcfe71ff93b31a5b401e799e4
diff --git a/ravenwood/Android.bp b/ravenwood/Android.bp
index 333fe4c..eebe5e9f 100644
--- a/ravenwood/Android.bp
+++ b/ravenwood/Android.bp
@@ -94,6 +94,9 @@
libs: [
"ravenwood-runtime-common-ravenwood",
],
+ static_libs: [
+ "framework-annotations-lib", // should it be "libs" instead?
+ ],
visibility: ["//visibility:private"],
}
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodAfterClassFailureTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodAfterClassFailureTest.java
new file mode 100644
index 0000000..f9794ad
--- /dev/null
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodAfterClassFailureTest.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.ravenwoodtest.bivalenttest.listenertests;
+
+import static android.platform.test.ravenwood.RavenwoodRule.isOnRavenwood;
+
+import org.junit.AfterClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import platform.test.runner.parameterized.ParameterizedAndroidJunit4;
+import platform.test.runner.parameterized.Parameters;
+
+/**
+ * Test that throws from @AfterClass.
+ *
+ * Tradefed would ignore it, so instead RavenwoodAwareTestRunner would detect it and kill
+ * the self (test) process.
+ *
+ * Unfortunately, this behavior can't easily be tested from within this class, so for now
+ * it's only used for a manual test, which you can run by removing the @Ignore.
+ *
+ * TODO(b/364948126) Improve the tests and automate it.
+ */
+@Ignore
+@RunWith(ParameterizedAndroidJunit4.class)
+public class RavenwoodAfterClassFailureTest {
+ public RavenwoodAfterClassFailureTest(String param) {
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ if (!isOnRavenwood()) return; // Don't do anything on real device.
+
+ throw new RuntimeException("FAILURE");
+ }
+
+ @Parameters
+ public static List<String> getParams() {
+ var params = new ArrayList<String>();
+ params.add("foo");
+ params.add("bar");
+ return params;
+ }
+
+ @Test
+ public void test1() {
+ }
+
+ @Test
+ public void test2() {
+ }
+}
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassAssumptionFailureTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassAssumptionFailureTest.java
new file mode 100644
index 0000000..61fb068
--- /dev/null
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassAssumptionFailureTest.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.ravenwoodtest.bivalenttest.listenertests;
+
+import static android.platform.test.ravenwood.RavenwoodRule.isOnRavenwood;
+
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import platform.test.runner.parameterized.ParameterizedAndroidJunit4;
+import platform.test.runner.parameterized.Parameters;
+
+/**
+ * Test that fails in assumption in @BeforeClass.
+ *
+ * This is only used for manual tests. Make sure `atest` shows 4 test results with
+ * "ASSUMPTION_FAILED".
+ *
+ * TODO(b/364948126) Improve the tests and automate it.
+ */
+@RunWith(ParameterizedAndroidJunit4.class)
+public class RavenwoodBeforeClassAssumptionFailureTest {
+ public RavenwoodBeforeClassAssumptionFailureTest(String param) {
+ }
+
+ @BeforeClass
+ public static void beforeClass() {
+ if (!isOnRavenwood()) return; // Don't do anything on real device.
+
+ Assume.assumeTrue(false);
+ }
+
+ @Parameters
+ public static List<String> getParams() {
+ var params = new ArrayList<String>();
+ params.add("foo");
+ params.add("bar");
+ return params;
+ }
+
+ @Test
+ public void test1() {
+ }
+
+ @Test
+ public void test2() {
+ }
+}
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassFailureTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassFailureTest.java
new file mode 100644
index 0000000..626ce81
--- /dev/null
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassFailureTest.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.ravenwoodtest.bivalenttest.listenertests;
+
+import static android.platform.test.ravenwood.RavenwoodRule.isOnRavenwood;
+
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import platform.test.runner.parameterized.ParameterizedAndroidJunit4;
+import platform.test.runner.parameterized.Parameters;
+
+/**
+ * Test that fails throws from @BeforeClass.
+ *
+ * This is only used for manual tests. Make sure `atest` shows 4 test results with
+ * a "FAILURE" runtime exception.
+ *
+ * In order to run the test, you'll need to remove the @Ignore.
+ *
+ * TODO(b/364948126) Improve the tests and automate it.
+ */
+@Ignore
+@RunWith(ParameterizedAndroidJunit4.class)
+public class RavenwoodBeforeClassFailureTest {
+ public static final String TAG = "RavenwoodBeforeClassFailureTest";
+
+ public RavenwoodBeforeClassFailureTest(String param) {
+ }
+
+ @BeforeClass
+ public static void beforeClass() {
+ if (!isOnRavenwood()) return; // Don't do anything on real device.
+
+ throw new RuntimeException("FAILURE");
+ }
+
+ @Parameters
+ public static List<String> getParams() {
+ var params = new ArrayList<String>();
+ params.add("foo");
+ params.add("bar");
+ return params;
+ }
+
+ @Test
+ public void test1() {
+ }
+
+ @Test
+ public void test2() {
+ }
+}
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleAssumptionFailureTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleAssumptionFailureTest.java
new file mode 100644
index 0000000..dc949c4
--- /dev/null
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleAssumptionFailureTest.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.ravenwoodtest.bivalenttest.listenertests;
+
+import static android.platform.test.ravenwood.RavenwoodRule.isOnRavenwood;
+
+import static org.junit.Assume.assumeTrue;
+
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runner.RunWith;
+import org.junit.runners.model.Statement;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import platform.test.runner.parameterized.ParameterizedAndroidJunit4;
+import platform.test.runner.parameterized.Parameters;
+
+/**
+ * Test that fails in assumption from a class rule.
+ *
+ * This is only used for manual tests. Make sure `atest` shows 4 test results with
+ * "ASSUMPTION_FAILED".
+ *
+ * TODO(b/364948126) Improve the tests and automate it.
+ */
+@RunWith(ParameterizedAndroidJunit4.class)
+public class RavenwoodClassRuleAssumptionFailureTest {
+ public static final String TAG = "RavenwoodClassRuleFailureTest";
+
+ @ClassRule
+ public static final TestRule sClassRule = new TestRule() {
+ @Override
+ public Statement apply(Statement base, Description description) {
+ if (!isOnRavenwood()) {
+ return base; // Just run the test as-is on a real device.
+ }
+
+ assumeTrue(false);
+ return null; // unreachable
+ }
+ };
+
+ public RavenwoodClassRuleAssumptionFailureTest(String param) {
+ }
+
+ @Parameters
+ public static List<String> getParams() {
+ var params = new ArrayList<String>();
+ params.add("foo");
+ params.add("bar");
+ return params;
+ }
+
+ @Test
+ public void test1() {
+ }
+
+ @Test
+ public void test2() {
+ }
+}
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleFailureTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleFailureTest.java
new file mode 100644
index 0000000..9996bec
--- /dev/null
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleFailureTest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.ravenwoodtest.bivalenttest.listenertests;
+
+import static android.platform.test.ravenwood.RavenwoodRule.isOnRavenwood;
+
+import org.junit.ClassRule;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runner.RunWith;
+import org.junit.runners.model.Statement;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import platform.test.runner.parameterized.ParameterizedAndroidJunit4;
+import platform.test.runner.parameterized.Parameters;
+
+/**
+ * Test that fails throws from a class rule.
+ *
+ * This is only used for manual tests. Make sure `atest` shows 4 test results with
+ * a "FAILURE" runtime exception.
+ *
+ * In order to run the test, you'll need to remove the @Ignore.
+ *
+ * TODO(b/364948126) Improve the tests and automate it.
+ */
+@Ignore
+@RunWith(ParameterizedAndroidJunit4.class)
+public class RavenwoodClassRuleFailureTest {
+ public static final String TAG = "RavenwoodClassRuleFailureTest";
+
+ @ClassRule
+ public static final TestRule sClassRule = new TestRule() {
+ @Override
+ public Statement apply(Statement base, Description description) {
+ if (!isOnRavenwood()) {
+ return base; // Just run the test as-is on a real device.
+ }
+
+ throw new RuntimeException("FAILURE");
+ }
+ };
+
+ public RavenwoodClassRuleFailureTest(String param) {
+ }
+
+ @Parameters
+ public static List<String> getParams() {
+ var params = new ArrayList<String>();
+ params.add("foo");
+ params.add("bar");
+ return params;
+ }
+
+ @Test
+ public void test1() {
+ }
+
+ @Test
+ public void test2() {
+ }
+}
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
index 1d182da..6d21e440 100644
--- a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
@@ -34,6 +34,8 @@
import org.junit.runner.Runner;
import org.junit.runners.model.TestClass;
+import java.util.Stack;
+
/**
* Provide hook points created by {@link RavenwoodAwareTestRunner}.
*/
@@ -44,6 +46,11 @@
}
private static RavenwoodTestStats sStats; // lazy initialization.
+
+ // Keep track of the current class description.
+
+ // Test classes can be nested because of "Suite", so we need a stack to keep track.
+ private static final Stack<Description> sClassDescriptions = new Stack<>();
private static Description sCurrentClassDescription;
private static RavenwoodTestStats getStats() {
@@ -108,14 +115,15 @@
Scope scope, Order order) {
Log.v(TAG, "onBefore: description=" + description + ", " + scope + ", " + order);
- if (scope == Scope.Class && order == Order.First) {
+ if (scope == Scope.Class && order == Order.Outer) {
// Keep track of the current class.
sCurrentClassDescription = description;
+ sClassDescriptions.push(description);
}
// Class-level annotations are checked by the runner already, so we only check
// method-level annotations here.
- if (scope == Scope.Instance && order == Order.First) {
+ if (scope == Scope.Instance && order == Order.Outer) {
if (!RavenwoodEnablementChecker.shouldEnableOnRavenwood(
description, true)) {
getStats().onTestFinished(sCurrentClassDescription, description, Result.Skipped);
@@ -134,17 +142,20 @@
Scope scope, Order order, Throwable th) {
Log.v(TAG, "onAfter: description=" + description + ", " + scope + ", " + order + ", " + th);
- if (scope == Scope.Instance && order == Order.First) {
+ if (scope == Scope.Instance && order == Order.Outer) {
getStats().onTestFinished(sCurrentClassDescription, description,
th == null ? Result.Passed : Result.Failed);
- } else if (scope == Scope.Class && order == Order.Last) {
+ } else if (scope == Scope.Class && order == Order.Outer) {
getStats().onClassFinished(sCurrentClassDescription);
+ sClassDescriptions.pop();
+ sCurrentClassDescription =
+ sClassDescriptions.size() == 0 ? null : sClassDescriptions.peek();
}
// If RUN_DISABLED_TESTS is set, and the method did _not_ throw, make it an error.
if (RavenwoodRule.private$ravenwood().isRunningDisabledTests()
- && scope == Scope.Instance && order == Order.First) {
+ && scope == Scope.Instance && order == Order.Outer) {
boolean isTestEnabled = RavenwoodEnablementChecker.shouldEnableOnRavenwood(
description, false);
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java
index 631f68f..3ffabef 100644
--- a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java
@@ -127,7 +127,11 @@
int passed = 0;
int skipped = 0;
int failed = 0;
- for (var e : mStats.get(classDescription).values()) {
+ var stats = mStats.get(classDescription);
+ if (stats == null) {
+ return;
+ }
+ for (var e : stats.values()) {
switch (e) {
case Passed: passed++; break;
case Skipped: skipped++; break;
diff --git a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java
index bfde9cb..dffb263 100644
--- a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java
+++ b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java
@@ -15,20 +15,25 @@
*/
package android.platform.test.ravenwood;
+import static com.android.ravenwood.common.RavenwoodCommonUtils.RAVENWOOD_VERBOSE_LOGGING;
import static com.android.ravenwood.common.RavenwoodCommonUtils.ensureIsPublicVoidMethod;
import static com.android.ravenwood.common.RavenwoodCommonUtils.isOnRavenwood;
import static java.lang.annotation.ElementType.METHOD;
import static java.lang.annotation.ElementType.TYPE;
+import android.annotation.NonNull;
+import android.annotation.Nullable;
import android.util.Log;
import com.android.ravenwood.common.SneakyThrow;
import org.junit.Assume;
+import org.junit.AssumptionViolatedException;
import org.junit.internal.builders.AllDefaultPossibilitiesBuilder;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
+import org.junit.runner.Result;
import org.junit.runner.Runner;
import org.junit.runner.manipulation.Filter;
import org.junit.runner.manipulation.Filterable;
@@ -39,8 +44,11 @@
import org.junit.runner.manipulation.Sortable;
import org.junit.runner.manipulation.Sorter;
import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunListener;
import org.junit.runner.notification.RunNotifier;
+import org.junit.runner.notification.StoppedByUserException;
import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.model.MultipleFailureException;
import org.junit.runners.model.RunnerBuilder;
import org.junit.runners.model.Statement;
import org.junit.runners.model.TestClass;
@@ -51,6 +59,8 @@
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Stack;
/**
* A test runner used for Ravenwood.
@@ -61,7 +71,7 @@
* the inner runner gets a chance to run. This can be used to initialize stuff used by the
* inner runner.
* - Add hook points, which are handed by RavenwoodAwareTestRunnerHook, with help from
- * the four test rules such as {@link #sImplicitClassMinRule}, which are also injected by
+ * the four test rules such as {@link #sImplicitClassOuterRule}, which are also injected by
* the ravenizer tool.
*
* We use this runner to:
@@ -102,28 +112,50 @@
/** Order of a hook. */
public enum Order {
- First,
- Last,
+ Outer,
+ Inner,
}
// The following four rule instances will be injected to tests by the Ravenizer tool.
+ private static class RavenwoodClassOuterRule implements TestRule {
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return getCurrentRunner().updateStatement(base, description, Scope.Class, Order.Outer);
+ }
+ }
- public static final TestRule sImplicitClassMinRule = (base, description) ->
- getCurrentRunner().updateStatement(base, description, Scope.Class, Order.First);
+ private static class RavenwoodClassInnerRule implements TestRule {
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return getCurrentRunner().updateStatement(base, description, Scope.Class, Order.Inner);
+ }
+ }
- public static final TestRule sImplicitClassMaxRule = (base, description) ->
- getCurrentRunner().updateStatement(base, description, Scope.Class, Order.Last);
+ private static class RavenwoodInstanceOuterRule implements TestRule {
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return getCurrentRunner().updateStatement(
+ base, description, Scope.Instance, Order.Outer);
+ }
+ }
- public static final TestRule sImplicitInstMinRule = (base, description) ->
- getCurrentRunner().updateStatement(base, description, Scope.Instance, Order.First);
+ private static class RavenwoodInstanceInnerRule implements TestRule {
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return getCurrentRunner().updateStatement(
+ base, description, Scope.Instance, Order.Inner);
+ }
+ }
- public static final TestRule sImplicitInstMaxRule = (base, description) ->
- getCurrentRunner().updateStatement(base, description, Scope.Instance, Order.Last);
+ public static final TestRule sImplicitClassOuterRule = new RavenwoodClassOuterRule();
+ public static final TestRule sImplicitClassInnerRule = new RavenwoodClassInnerRule();
+ public static final TestRule sImplicitInstOuterRule = new RavenwoodInstanceOuterRule();
+ public static final TestRule sImplicitInstInnerRule = new RavenwoodInstanceOuterRule();
- public static final String IMPLICIT_CLASS_MIN_RULE_NAME = "sImplicitClassMinRule";
- public static final String IMPLICIT_CLASS_MAX_RULE_NAME = "sImplicitClassMaxRule";
- public static final String IMPLICIT_INST_MIN_RULE_NAME = "sImplicitInstMinRule";
- public static final String IMPLICIT_INST_MAX_RULE_NAME = "sImplicitInstMaxRule";
+ public static final String IMPLICIT_CLASS_OUTER_RULE_NAME = "sImplicitClassOuterRule";
+ public static final String IMPLICIT_CLASS_INNER_RULE_NAME = "sImplicitClassInnerRule";
+ public static final String IMPLICIT_INST_OUTER_RULE_NAME = "sImplicitInstOuterRule";
+ public static final String IMPLICIT_INST_INNER_RULE_NAME = "sImplicitInstInnerRule";
/** Keeps track of the runner on the current thread. */
private static final ThreadLocal<RavenwoodAwareTestRunner> sCurrentRunner = new ThreadLocal<>();
@@ -157,6 +189,8 @@
try {
mTestClass = new TestClass(testClass);
+ Log.v(TAG, "RavenwoodAwareTestRunner starting for " + testClass.getCanonicalName());
+
onRunnerInitializing();
/*
@@ -261,20 +295,27 @@
}
@Override
- public void run(RunNotifier notifier) {
+ public void run(RunNotifier realNotifier) {
+ final RunNotifier notifier = new RavenwoodRunNotifier(realNotifier);
+
if (mRealRunner instanceof ClassSkippingTestRunner) {
mRealRunner.run(notifier);
RavenwoodAwareTestRunnerHook.onClassSkipped(getDescription());
return;
}
+ Log.v(TAG, "Starting " + mTestClass.getJavaClass().getCanonicalName());
+ if (RAVENWOOD_VERBOSE_LOGGING) {
+ dumpDescription(getDescription());
+ }
+
if (maybeReportExceptionFromConstructor(notifier)) {
return;
}
sCurrentRunner.set(this);
try {
- runWithHooks(getDescription(), Scope.Runner, Order.First,
+ runWithHooks(getDescription(), Scope.Runner, Order.Outer,
() -> mRealRunner.run(notifier));
} finally {
sCurrentRunner.remove();
@@ -399,4 +440,217 @@
}
}
}
+
+ private void dumpDescription(Description desc) {
+ dumpDescription(desc, "[TestDescription]=", " ");
+ }
+
+ private void dumpDescription(Description desc, String header, String indent) {
+ Log.v(TAG, indent + header + desc);
+
+ var children = desc.getChildren();
+ var childrenIndent = " " + indent;
+ for (int i = 0; i < children.size(); i++) {
+ dumpDescription(children.get(i), "#" + i + ": ", childrenIndent);
+ }
+ }
+
+ /**
+ * A run notifier that wraps another notifier and provides the following features:
+ * - Handle a failure that happened before testStarted and testEnded (typically that means
+ * it's from @BeforeClass or @AfterClass, or a @ClassRule) and deliver it as if
+ * individual tests in the class reported it. This is for b/364395552.
+ *
+ * - Logging.
+ */
+ private class RavenwoodRunNotifier extends RunNotifier {
+ private final RunNotifier mRealNotifier;
+
+ private final Stack<Description> mSuiteStack = new Stack<>();
+ private Description mCurrentSuite = null;
+ private final ArrayList<Throwable> mOutOfTestFailures = new ArrayList<>();
+
+ private boolean mBeforeTest = true;
+ private boolean mAfterTest = false;
+
+ private RavenwoodRunNotifier(RunNotifier realNotifier) {
+ mRealNotifier = realNotifier;
+ }
+
+ private boolean isInTest() {
+ return !mBeforeTest && !mAfterTest;
+ }
+
+ @Override
+ public void addListener(RunListener listener) {
+ mRealNotifier.addListener(listener);
+ }
+
+ @Override
+ public void removeListener(RunListener listener) {
+ mRealNotifier.removeListener(listener);
+ }
+
+ @Override
+ public void addFirstListener(RunListener listener) {
+ mRealNotifier.addFirstListener(listener);
+ }
+
+ @Override
+ public void fireTestRunStarted(Description description) {
+ Log.i(TAG, "testRunStarted: " + description);
+ mRealNotifier.fireTestRunStarted(description);
+ }
+
+ @Override
+ public void fireTestRunFinished(Result result) {
+ Log.i(TAG, "testRunFinished: "
+ + result.getRunCount() + ","
+ + result.getFailureCount() + ","
+ + result.getAssumptionFailureCount() + ","
+ + result.getIgnoreCount());
+ mRealNotifier.fireTestRunFinished(result);
+ }
+
+ @Override
+ public void fireTestSuiteStarted(Description description) {
+ Log.i(TAG, "testSuiteStarted: " + description);
+ mRealNotifier.fireTestSuiteStarted(description);
+
+ mBeforeTest = true;
+ mAfterTest = false;
+
+ // Keep track of the current suite, needed if the outer test is a Suite,
+ // in which case its children are test classes. (not test methods)
+ mCurrentSuite = description;
+ mSuiteStack.push(description);
+
+ mOutOfTestFailures.clear();
+ }
+
+ @Override
+ public void fireTestSuiteFinished(Description description) {
+ Log.i(TAG, "testSuiteFinished: " + description);
+ mRealNotifier.fireTestSuiteFinished(description);
+
+ maybeHandleOutOfTestFailures();
+
+ mBeforeTest = true;
+ mAfterTest = false;
+
+ // Restore the upper suite.
+ mSuiteStack.pop();
+ mCurrentSuite = mSuiteStack.size() == 0 ? null : mSuiteStack.peek();
+ }
+
+ @Override
+ public void fireTestStarted(Description description) throws StoppedByUserException {
+ Log.i(TAG, "testStarted: " + description);
+ mRealNotifier.fireTestStarted(description);
+
+ mAfterTest = false;
+ mBeforeTest = false;
+ }
+
+ @Override
+ public void fireTestFailure(Failure failure) {
+ Log.i(TAG, "testFailure: " + failure);
+
+ if (isInTest()) {
+ mRealNotifier.fireTestFailure(failure);
+ } else {
+ mOutOfTestFailures.add(failure.getException());
+ }
+ }
+
+ @Override
+ public void fireTestAssumptionFailed(Failure failure) {
+ Log.i(TAG, "testAssumptionFailed: " + failure);
+
+ if (isInTest()) {
+ mRealNotifier.fireTestAssumptionFailed(failure);
+ } else {
+ mOutOfTestFailures.add(failure.getException());
+ }
+ }
+
+ @Override
+ public void fireTestIgnored(Description description) {
+ Log.i(TAG, "testIgnored: " + description);
+ mRealNotifier.fireTestIgnored(description);
+ }
+
+ @Override
+ public void fireTestFinished(Description description) {
+ Log.i(TAG, "testFinished: " + description);
+ mRealNotifier.fireTestFinished(description);
+
+ mAfterTest = true;
+ }
+
+ @Override
+ public void pleaseStop() {
+ Log.w(TAG, "pleaseStop:");
+ mRealNotifier.pleaseStop();
+ }
+
+ /**
+ * At the end of each Suite, we handle failures happened out of test methods.
+ * (typically in @BeforeClass or @AfterClasses)
+ *
+ * This is to work around b/364395552.
+ */
+ private boolean maybeHandleOutOfTestFailures() {
+ if (mOutOfTestFailures.size() == 0) {
+ return false;
+ }
+ Throwable th;
+ if (mOutOfTestFailures.size() == 1) {
+ th = mOutOfTestFailures.get(0);
+ } else {
+ th = new MultipleFailureException(mOutOfTestFailures);
+ }
+ if (mBeforeTest) {
+ reportBeforeTestFailure(mCurrentSuite, th);
+ return true;
+ }
+ if (mAfterTest) {
+ // Unfortunately, there's no good way to report it, so kill the own process.
+ onCriticalError(
+ "Failures detected in @AfterClass, which would be swalloed by tradefed",
+ th);
+ return true; // unreachable
+ }
+ return false;
+ }
+
+ private void reportBeforeTestFailure(Description suiteDesc, Throwable th) {
+ // If a failure happens befere running any tests, we'll need to pretend
+ // as if each test in the suite reported the failure, to work around b/364395552.
+ for (var child : suiteDesc.getChildren()) {
+ if (child.isSuite()) {
+ // If the chiil is still a "parent" -- a test class or a test suite
+ // -- propagate to its children.
+ mRealNotifier.fireTestSuiteStarted(child);
+ reportBeforeTestFailure(child, th);
+ mRealNotifier.fireTestSuiteFinished(child);
+ } else {
+ mRealNotifier.fireTestStarted(child);
+ Failure f = new Failure(child, th);
+ if (th instanceof AssumptionViolatedException) {
+ mRealNotifier.fireTestAssumptionFailed(f);
+ } else {
+ mRealNotifier.fireTestFailure(f);
+ }
+ mRealNotifier.fireTestFinished(child);
+ }
+ }
+ }
+ }
+
+ private void onCriticalError(@NonNull String message, @Nullable Throwable th) {
+ Log.e(TAG, "Critical error! Ravenwood cannot continue. Killing self process: "
+ + message, th);
+ System.exit(1);
+ }
}
diff --git a/ravenwood/runtime-common-src/com/android/ravenwood/common/RavenwoodCommonUtils.java b/ravenwood/runtime-common-src/com/android/ravenwood/common/RavenwoodCommonUtils.java
index 7b5bc5a..875ce71 100644
--- a/ravenwood/runtime-common-src/com/android/ravenwood/common/RavenwoodCommonUtils.java
+++ b/ravenwood/runtime-common-src/com/android/ravenwood/common/RavenwoodCommonUtils.java
@@ -15,12 +15,17 @@
*/
package com.android.ravenwood.common;
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+
import com.android.ravenwood.common.divergence.RavenwoodDivergence;
import java.io.File;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.Arrays;
@@ -33,6 +38,14 @@
private static final Object sLock = new Object();
+ /**
+ * If set to "1", we enable the verbose logging.
+ *
+ * (See also InitLogging() in http://ac/system/libbase/logging.cpp)
+ */
+ public static final boolean RAVENWOOD_VERBOSE_LOGGING = "1".equals(System.getenv(
+ "RAVENWOOD_VERBOSE"));
+
/** Name of `libravenwood_runtime` */
private static final String RAVENWOOD_NATIVE_RUNTIME_NAME = "ravenwood_runtime";
@@ -265,4 +278,12 @@
method.getDeclaringClass().getName(), method.getName(),
(isStatic ? "static " : "")));
}
+
+ @NonNull
+ public static String getStackTraceString(@Nullable Throwable th) {
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter writer = new PrintWriter(stringWriter);
+ th.printStackTrace(writer);
+ return stringWriter.toString();
+ }
}
diff --git a/ravenwood/runtime-helper-src/framework/com/android/platform/test/ravenwood/runtimehelper/ClassLoadHook.java b/ravenwood/runtime-helper-src/framework/com/android/platform/test/ravenwood/runtimehelper/ClassLoadHook.java
index c519204..01e19a7 100644
--- a/ravenwood/runtime-helper-src/framework/com/android/platform/test/ravenwood/runtimehelper/ClassLoadHook.java
+++ b/ravenwood/runtime-helper-src/framework/com/android/platform/test/ravenwood/runtimehelper/ClassLoadHook.java
@@ -15,6 +15,8 @@
*/
package com.android.platform.test.ravenwood.runtimehelper;
+import static com.android.ravenwood.common.RavenwoodCommonUtils.RAVENWOOD_VERBOSE_LOGGING;
+
import android.system.ErrnoException;
import android.system.Os;
@@ -40,14 +42,6 @@
private static final boolean SKIP_LOADING_LIBANDROID = "1".equals(System.getenv(
"RAVENWOOD_SKIP_LOADING_LIBANDROID"));
- /**
- * If set to 1, and if $ANDROID_LOG_TAGS isn't set, we enable the verbose logging.
- *
- * (See also InitLogging() in http://ac/system/libbase/logging.cpp)
- */
- private static final boolean RAVENWOOD_VERBOSE_LOGGING = "1".equals(System.getenv(
- "RAVENWOOD_VERBOSE"));
-
public static final String CORE_NATIVE_CLASSES = "core_native_classes";
public static final String ICU_DATA_PATH = "icu.data.path";
public static final String KEYBOARD_PATHS = "keyboard_paths";
diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/RunnerRewritingAdapter.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/RunnerRewritingAdapter.kt
index eaef2cf..bd9d96d 100644
--- a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/RunnerRewritingAdapter.kt
+++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/RunnerRewritingAdapter.kt
@@ -302,7 +302,7 @@
override fun visitCode() {
visitFieldInsn(Opcodes.GETSTATIC,
ravenwoodTestRunnerType.internlName,
- RavenwoodAwareTestRunner.IMPLICIT_CLASS_MIN_RULE_NAME,
+ RavenwoodAwareTestRunner.IMPLICIT_CLASS_OUTER_RULE_NAME,
testRuleType.desc
)
visitFieldInsn(Opcodes.PUTSTATIC,
@@ -313,7 +313,7 @@
visitFieldInsn(Opcodes.GETSTATIC,
ravenwoodTestRunnerType.internlName,
- RavenwoodAwareTestRunner.IMPLICIT_CLASS_MAX_RULE_NAME,
+ RavenwoodAwareTestRunner.IMPLICIT_CLASS_INNER_RULE_NAME,
testRuleType.desc
)
visitFieldInsn(Opcodes.PUTSTATIC,
@@ -361,7 +361,7 @@
visitVarInsn(ALOAD, 0)
visitFieldInsn(Opcodes.GETSTATIC,
ravenwoodTestRunnerType.internlName,
- RavenwoodAwareTestRunner.IMPLICIT_INST_MIN_RULE_NAME,
+ RavenwoodAwareTestRunner.IMPLICIT_INST_OUTER_RULE_NAME,
testRuleType.desc
)
visitFieldInsn(Opcodes.PUTFIELD,
@@ -373,7 +373,7 @@
visitVarInsn(ALOAD, 0)
visitFieldInsn(Opcodes.GETSTATIC,
ravenwoodTestRunnerType.internlName,
- RavenwoodAwareTestRunner.IMPLICIT_INST_MAX_RULE_NAME,
+ RavenwoodAwareTestRunner.IMPLICIT_INST_INNER_RULE_NAME,
testRuleType.desc
)
visitFieldInsn(Opcodes.PUTFIELD,