Move @DisabledOnRavenwood logic to the runner side
Now the whole logic about "@DisabledOnRavenwood" is moved to the runner,
with "RUN_DISABLED_TESTS" support too.
RavenwoodRule is no longer needed to handle @DisabledOnRavenwood.
Also create test result stats files, which contains number of tests
passed/failed/skipped.
Bug: 356918135
Test: $ANDROID_BUILD_TOP/frameworks/base/ravenwood/scripts/run-ravenwood-tests.sh
Test: atest RavenwoodBivalentTest_device
Flag: EXEMPT host test change only
Change-Id: I8e1ee027e7cac9014cdc2b67b8f748b0922a536e
diff --git a/ravenwood/Android.bp b/ravenwood/Android.bp
index be4cd76..9b0c8e5 100644
--- a/ravenwood/Android.bp
+++ b/ravenwood/Android.bp
@@ -160,6 +160,7 @@
"ravenwood-framework",
"services.core.ravenwood",
"junit",
+ "framework-annotations-lib",
],
sdk_version: "core_current",
visibility: ["//frameworks/base"],
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/RavenwoodClassRuleDeviceOnlyTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/RavenwoodClassRuleDeviceOnlyTest.java
index 3a24c0e..e8f59db 100644
--- a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/RavenwoodClassRuleDeviceOnlyTest.java
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/RavenwoodClassRuleDeviceOnlyTest.java
@@ -26,6 +26,10 @@
import org.junit.Test;
import org.junit.runner.RunWith;
+/**
+ * Test to ensure @DisabledOnRavenwood works. Note, now the DisabledOnRavenwood annotation
+ * is handled by the test runner, so it won't really need the class rule.
+ */
@RunWith(AndroidJUnit4.class)
@DisabledOnRavenwood
public class RavenwoodClassRuleDeviceOnlyTest {
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodImplicitClassRuleDeviceOnlyTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodImplicitClassRuleDeviceOnlyTest.java
index 0f8be0e..7ef672e 100644
--- a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodImplicitClassRuleDeviceOnlyTest.java
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodImplicitClassRuleDeviceOnlyTest.java
@@ -34,7 +34,12 @@
@BeforeClass
public static void beforeClass() {
- Assert.assertFalse(RavenwoodRule.isOnRavenwood());
+ // This method shouldn't be called -- unless RUN_DISABLED_TESTS is enabled.
+
+ // If we're doing RUN_DISABLED_TESTS, don't throw here, because that'd confuse junit.
+ if (!RavenwoodRule.private$ravenwood().isRunningDisabledTests()) {
+ Assert.assertFalse(RavenwoodRule.isOnRavenwood());
+ }
}
@Test
@@ -46,7 +51,10 @@
public static void afterClass() {
if (RavenwoodRule.isOnRavenwood()) {
Log.e(TAG, "Even @AfterClass shouldn't be executed!");
- System.exit(1);
+
+ if (!RavenwoodRule.private$ravenwood().isRunningDisabledTests()) {
+ System.exit(1);
+ }
}
}
}
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsReallyDisabledTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsReallyDisabledTest.java
new file mode 100644
index 0000000..c77841b
--- /dev/null
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsReallyDisabledTest.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.ravenwoodtest.bivalenttest.ravenizer;
+
+import static org.junit.Assert.fail;
+
+import android.platform.test.annotations.DisabledOnRavenwood;
+import android.platform.test.ravenwood.RavenwoodAwareTestRunner.RavenwoodTestRunnerInitializing;
+import android.platform.test.ravenwood.RavenwoodRule;
+import android.util.Log;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Test for "RAVENWOOD_RUN_DISABLED_TESTS" with "REALLY_DISABLED" set.
+ *
+ * This test is only executed on Ravenwood.
+ */
+@RunWith(AndroidJUnit4.class)
+public class RavenwoodRunDisabledTestsReallyDisabledTest {
+ private static final String TAG = "RavenwoodRunDisabledTestsTest";
+
+ private static final CallTracker sCallTracker = new CallTracker();
+
+ @RavenwoodTestRunnerInitializing
+ public static void ravenwoodRunnerInitializing() {
+ RavenwoodRule.private$ravenwood().overrideRunDisabledTest(true,
+ "\\#testReallyDisabled$");
+ }
+
+ /**
+ * This test gets to run with RAVENWOOD_RUN_DISABLED_TESTS set.
+ */
+ @Test
+ @DisabledOnRavenwood
+ public void testDisabledTestGetsToRun() {
+ if (!RavenwoodRule.isOnRavenwood()) {
+ return;
+ }
+ sCallTracker.incrementMethodCallCount();
+
+ fail("This test won't pass on Ravenwood.");
+ }
+
+ /**
+ * This will still not be executed due to the "really disabled" pattern.
+ */
+ @Test
+ @DisabledOnRavenwood
+ public void testReallyDisabled() {
+ if (!RavenwoodRule.isOnRavenwood()) {
+ return;
+ }
+ sCallTracker.incrementMethodCallCount();
+
+ fail("This test won't pass on Ravenwood.");
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ if (!RavenwoodRule.isOnRavenwood()) {
+ return;
+ }
+ Log.i(TAG, "afterClass called");
+
+ sCallTracker.assertCallsOrDie(
+ "testDisabledTestGetsToRun", 1,
+ "testReallyDisabled", 0
+ );
+ }
+}
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsTest.java
new file mode 100644
index 0000000..ea1a29d
--- /dev/null
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsTest.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.ravenwoodtest.bivalenttest.ravenizer;
+
+import static org.junit.Assert.fail;
+
+import android.platform.test.annotations.DisabledOnRavenwood;
+import android.platform.test.ravenwood.RavenwoodAwareTestRunner.RavenwoodTestRunnerInitializing;
+import android.platform.test.ravenwood.RavenwoodRule;
+import android.util.Log;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+
+import org.junit.AfterClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+
+/**
+ * Test for "RAVENWOOD_RUN_DISABLED_TESTS". (with no "REALLY_DISABLED" set.)
+ *
+ * This test is only executed on Ravenwood.
+ */
+@RunWith(AndroidJUnit4.class)
+public class RavenwoodRunDisabledTestsTest {
+ private static final String TAG = "RavenwoodRunDisabledTestsTest";
+
+ @Rule
+ public ExpectedException mExpectedException = ExpectedException.none();
+
+ private static final CallTracker sCallTracker = new CallTracker();
+
+ @RavenwoodTestRunnerInitializing
+ public static void ravenwoodRunnerInitializing() {
+ RavenwoodRule.private$ravenwood().overrideRunDisabledTest(true, null);
+ }
+
+ @Test
+ @DisabledOnRavenwood
+ public void testDisabledTestGetsToRun() {
+ if (!RavenwoodRule.isOnRavenwood()) {
+ return;
+ }
+ sCallTracker.incrementMethodCallCount();
+
+ fail("This test won't pass on Ravenwood.");
+ }
+
+ @Test
+ @DisabledOnRavenwood
+ public void testDisabledButPass() {
+ if (!RavenwoodRule.isOnRavenwood()) {
+ return;
+ }
+ sCallTracker.incrementMethodCallCount();
+
+ // When a @DisabledOnRavenwood actually passed, the runner should make fail().
+ mExpectedException.expectMessage("it actually passed under Ravenwood");
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ if (!RavenwoodRule.isOnRavenwood()) {
+ return;
+ }
+ Log.i(TAG, "afterClass called");
+
+ sCallTracker.assertCallsOrDie(
+ "testDisabledTestGetsToRun", 1,
+ "testDisabledButPass", 1
+ );
+ }
+}
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
index 03600ad..1da93eb 100644
--- a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
@@ -17,14 +17,16 @@
import static com.android.ravenwood.common.RavenwoodCommonUtils.RAVENWOOD_VERSION_JAVA_SYSPROP;
+import static org.junit.Assert.fail;
+
import android.os.Bundle;
import android.platform.test.ravenwood.RavenwoodAwareTestRunner.Order;
import android.platform.test.ravenwood.RavenwoodAwareTestRunner.Scope;
+import android.platform.test.ravenwood.RavenwoodTestStats.Result;
+import android.util.Log;
import androidx.test.platform.app.InstrumentationRegistry;
-import com.android.ravenwood.common.RavenwoodCommonUtils;
-
import org.junit.runner.Description;
import org.junit.runner.Runner;
import org.junit.runners.model.TestClass;
@@ -38,12 +40,24 @@
private RavenwoodAwareTestRunnerHook() {
}
- private static void log(String message) {
- RavenwoodCommonUtils.log(TAG, message);
+ private static RavenwoodTestStats sStats; // lazy initialization.
+ private static Description sCurrentClassDescription;
+
+ private static RavenwoodTestStats getStats() {
+ if (sStats == null) {
+ // We don't want to throw in the static initializer, because tradefed may not report
+ // it properly, so we initialize it here.
+ sStats = new RavenwoodTestStats();
+ }
+ return sStats;
}
+ /**
+ * Called when a runner starts, before the inner runner gets a chance to run.
+ */
public static void onRunnerInitializing(Runner runner, TestClass testClass) {
- log("onRunnerStart: testClass=" + testClass + " runner=" + runner);
+ // This log call also ensures the framework JNI is loaded.
+ Log.i(TAG, "onRunnerInitializing: testClass=" + testClass + " runner=" + runner);
// TODO: Move the initialization code to a better place.
@@ -52,26 +66,97 @@
"androidx.test.internal.runner.junit4.AndroidJUnit4ClassRunner");
System.setProperty(RAVENWOOD_VERSION_JAVA_SYSPROP, "1");
+
// This is needed to make AndroidJUnit4ClassRunner happy.
InstrumentationRegistry.registerInstance(null, Bundle.EMPTY);
}
+ /**
+ * Called when a whole test class is skipped.
+ */
+ public static void onClassSkipped(Description description) {
+ Log.i(TAG, "onClassSkipped: description=" + description);
+ getStats().onClassSkipped(description);
+ }
+
+ /**
+ * Called before a test / class.
+ *
+ * Return false if it should be skipped.
+ */
public static boolean onBefore(RavenwoodAwareTestRunner runner, Description description,
Scope scope, Order order) {
- log("onBefore: description=" + description + ", " + scope + ", " + order);
+ Log.i(TAG, "onBefore: description=" + description + ", " + scope + ", " + order);
+
+ if (scope == Scope.Class && order == Order.First) {
+ // Keep track of the current class.
+ sCurrentClassDescription = description;
+ }
// Class-level annotations are checked by the runner already, so we only check
// method-level annotations here.
if (scope == Scope.Instance && order == Order.First) {
- if (!RavenwoodRule.shouldEnableOnRavenwood(description)) {
+ if (!RavenwoodEnablementChecker.shouldEnableOnRavenwood(
+ description, true)) {
+ getStats().onTestFinished(sCurrentClassDescription, description, Result.Skipped);
return false;
}
}
return true;
}
- public static void onAfter(RavenwoodAwareTestRunner runner, Description description,
+ /**
+ * Called after a test / class.
+ *
+ * Return false if the exception should be ignored.
+ */
+ public static boolean onAfter(RavenwoodAwareTestRunner runner, Description description,
Scope scope, Order order, Throwable th) {
- log("onAfter: description=" + description + ", " + scope + ", " + order + ", " + th);
+ Log.i(TAG, "onAfter: description=" + description + ", " + scope + ", " + order + ", " + th);
+
+ if (scope == Scope.Instance && order == Order.First) {
+ getStats().onTestFinished(sCurrentClassDescription, description,
+ th == null ? Result.Passed : Result.Failed);
+
+ } else if (scope == Scope.Class && order == Order.Last) {
+ getStats().onClassFinished(sCurrentClassDescription);
+ }
+
+ // If RUN_DISABLED_TESTS is set, and the method did _not_ throw, make it an error.
+ if (RavenwoodRule.private$ravenwood().isRunningDisabledTests()
+ && scope == Scope.Instance && order == Order.First) {
+
+ boolean isTestEnabled = RavenwoodEnablementChecker.shouldEnableOnRavenwood(
+ description, false);
+ if (th == null) {
+ // Test passed. Is the test method supposed to be enabled?
+ if (isTestEnabled) {
+ // Enabled and didn't throw, okay.
+ return true;
+ } else {
+ // Disabled and didn't throw. We should report it.
+ fail("Test wasn't included under Ravenwood, but it actually "
+ + "passed under Ravenwood; consider updating annotations");
+ return true; // unreachable.
+ }
+ } else {
+ // Test failed.
+ if (isTestEnabled) {
+ // Enabled but failed. We should throw the exception.
+ return true;
+ } else {
+ // Disabled and failed. Expected. Don't throw.
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Called by {@link RavenwoodAwareTestRunner} to see if it should run a test class or not.
+ */
+ public static boolean shouldRunClassOnRavenwood(Class<?> clazz) {
+ return RavenwoodEnablementChecker.shouldRunClassOnRavenwood(clazz, true);
}
}
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodEnablementChecker.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodEnablementChecker.java
new file mode 100644
index 0000000..77275c4
--- /dev/null
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodEnablementChecker.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.platform.test.ravenwood;
+
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.platform.test.annotations.DisabledOnRavenwood;
+import android.platform.test.annotations.EnabledOnRavenwood;
+import android.platform.test.annotations.IgnoreUnderRavenwood;
+
+import org.junit.runner.Description;
+
+/**
+ * Calculates which tests need to be executed on Ravenwood.
+ */
+public class RavenwoodEnablementChecker {
+ private static final String TAG = "RavenwoodDisablementChecker";
+
+ private RavenwoodEnablementChecker() {
+ }
+
+ /**
+ * Determine if the given {@link Description} should be enabled when running on the
+ * Ravenwood test environment.
+ *
+ * A more specific method-level annotation always takes precedence over any class-level
+ * annotation, and an {@link EnabledOnRavenwood} annotation always takes precedence over
+ * an {@link DisabledOnRavenwood} annotation.
+ */
+ public static boolean shouldEnableOnRavenwood(Description description,
+ boolean takeIntoAccountRunDisabledTestsFlag) {
+ // First, consult any method-level annotations
+ if (description.isTest()) {
+ Boolean result = null;
+
+ // Stopgap for http://g/ravenwood/EPAD-N5ntxM
+ if (description.getMethodName().endsWith("$noRavenwood")) {
+ result = false;
+ } else if (description.getAnnotation(EnabledOnRavenwood.class) != null) {
+ result = true;
+ } else if (description.getAnnotation(DisabledOnRavenwood.class) != null) {
+ result = false;
+ } else if (description.getAnnotation(IgnoreUnderRavenwood.class) != null) {
+ result = false;
+ }
+ if (result != null) {
+ if (takeIntoAccountRunDisabledTestsFlag
+ && RavenwoodRule.private$ravenwood().isRunningDisabledTests()) {
+ result = !shouldStillIgnoreInProbeIgnoreMode(
+ description.getTestClass(), description.getMethodName());
+ }
+ }
+ if (result != null) {
+ return result;
+ }
+ }
+
+ // Otherwise, consult any class-level annotations
+ return shouldRunClassOnRavenwood(description.getTestClass(),
+ takeIntoAccountRunDisabledTestsFlag);
+ }
+
+ public static boolean shouldRunClassOnRavenwood(@NonNull Class<?> testClass,
+ boolean takeIntoAccountRunDisabledTestsFlag) {
+ boolean result = true;
+ if (testClass.getAnnotation(EnabledOnRavenwood.class) != null) {
+ result = true;
+ } else if (testClass.getAnnotation(DisabledOnRavenwood.class) != null) {
+ result = false;
+ } else if (testClass.getAnnotation(IgnoreUnderRavenwood.class) != null) {
+ result = false;
+ }
+ if (!result) {
+ if (takeIntoAccountRunDisabledTestsFlag
+ && RavenwoodRule.private$ravenwood().isRunningDisabledTests()) {
+ result = !shouldStillIgnoreInProbeIgnoreMode(testClass, null);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Check if a test should _still_ disabled even if {@code RUN_DISABLED_TESTS}
+ * is true, using {@code REALLY_DISABLED_PATTERN}.
+ *
+ * This only works on tests, not on classes.
+ */
+ static boolean shouldStillIgnoreInProbeIgnoreMode(
+ @NonNull Class<?> testClass, @Nullable String methodName) {
+ if (RavenwoodRule.private$ravenwood().getReallyDisabledPattern().pattern().isEmpty()) {
+ return false;
+ }
+
+ final var fullname = testClass.getName() + (methodName != null ? "#" + methodName : "");
+
+ System.out.println("XXX=" + fullname);
+
+ if (RavenwoodRule.private$ravenwood().getReallyDisabledPattern().matcher(fullname).find()) {
+ System.out.println("Still ignoring " + fullname);
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
index 7b4c173..a2088fd 100644
--- a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
@@ -40,7 +40,6 @@
import com.android.server.LocalServices;
import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
import java.io.File;
import java.io.IOException;
@@ -226,11 +225,6 @@
}
}
- public static void validate(Statement base, Description description,
- boolean enableOptionalValidation) {
- // Nothing to check, for now.
- }
-
/**
* Set the current configuration to the actual SystemProperties.
*/
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java
new file mode 100644
index 0000000..631f68f
--- /dev/null
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java
@@ -0,0 +1,163 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.platform.test.ravenwood;
+
+import android.util.Log;
+
+import org.junit.runner.Description;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Creats a "stats" CSV file containing the test results.
+ *
+ * The output file is created as `/tmp/Ravenwood-stats_[TEST-MODULE=NAME]_[TIMESTAMP].csv`.
+ * A symlink to the latest result will be created as
+ * `/tmp/Ravenwood-stats_[TEST-MODULE=NAME]_latest.csv`.
+ */
+public class RavenwoodTestStats {
+ private static final String TAG = "RavenwoodTestStats";
+ private static final String HEADER = "Module,Class,ClassDesc,Passed,Failed,Skipped";
+
+ public enum Result {
+ Passed,
+ Failed,
+ Skipped,
+ }
+
+ private final File mOutputFile;
+ private final PrintWriter mOutputWriter;
+ private final String mTestModuleName;
+
+ public final Map<Description, Map<Description, Result>> mStats = new HashMap<>();
+
+ /** Ctor */
+ public RavenwoodTestStats() {
+ mTestModuleName = guessTestModuleName();
+
+ var basename = "Ravenwood-stats_" + mTestModuleName + "_";
+
+ // Get the current time
+ LocalDateTime now = LocalDateTime.now();
+ DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss");
+
+ var tmpdir = System.getProperty("java.io.tmpdir");
+ mOutputFile = new File(tmpdir, basename + now.format(fmt) + ".csv");
+
+ try {
+ mOutputWriter = new PrintWriter(mOutputFile);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to crete logfile. File=" + mOutputFile, e);
+ }
+
+ // Crete the "latest" symlink.
+ Path symlink = Paths.get(tmpdir, basename + "latest.csv");
+ try {
+ if (Files.exists(symlink)) {
+ Files.delete(symlink);
+ }
+ Files.createSymbolicLink(symlink, Paths.get(mOutputFile.getName()));
+
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to crete logfile. File=" + mOutputFile, e);
+ }
+
+ Log.i(TAG, "Test result stats file: " + mOutputFile);
+
+ // Print the header.
+ mOutputWriter.println(HEADER);
+ mOutputWriter.flush();
+ }
+
+ private String guessTestModuleName() {
+ // Assume the current directory name is the test module name.
+ File cwd;
+ try {
+ cwd = new File(".").getCanonicalFile();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to get the current directory", e);
+ }
+ return cwd.getName();
+ }
+
+ private void addResult(Description classDescription, Description methodDescription,
+ Result result) {
+ mStats.compute(classDescription, (classDesc, value) -> {
+ if (value == null) {
+ value = new HashMap<>();
+ }
+ value.put(methodDescription, result);
+ return value;
+ });
+ }
+
+ public void onClassSkipped(Description classDescription) {
+ addResult(classDescription, Description.EMPTY, Result.Skipped);
+ onClassFinished(classDescription);
+ }
+
+ public void onTestFinished(Description classDescription, Description testDescription,
+ Result result) {
+ addResult(classDescription, testDescription, result);
+ }
+
+ public void onClassFinished(Description classDescription) {
+ int passed = 0;
+ int skipped = 0;
+ int failed = 0;
+ for (var e : mStats.get(classDescription).values()) {
+ switch (e) {
+ case Passed: passed++; break;
+ case Skipped: skipped++; break;
+ case Failed: failed++; break;
+ }
+ }
+
+ var testClass = extractTestClass(classDescription);
+
+ mOutputWriter.printf("%s,%s,%s,%d,%d,%d\n",
+ mTestModuleName, (testClass == null ? "?" : testClass.getCanonicalName()),
+ classDescription, passed, failed, skipped);
+ mOutputWriter.flush();
+ }
+
+ /**
+ * Try to extract the class from a description, which is needed because
+ * ParameterizedAndroidJunit4's description doesn't contain a class.
+ */
+ private Class<?> extractTestClass(Description desc) {
+ if (desc.getTestClass() != null) {
+ return desc.getTestClass();
+ }
+ // Look into the children.
+ for (var child : desc.getChildren()) {
+ var fromChild = extractTestClass(child);
+ if (fromChild != null) {
+ return fromChild;
+ }
+ }
+ return null;
+ }
+}
diff --git a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java
index a4fa41a..2b55ac5 100644
--- a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java
+++ b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java
@@ -15,8 +15,6 @@
*/
package android.platform.test.ravenwood;
-import static android.platform.test.ravenwood.RavenwoodRule.shouldRunCassOnRavenwood;
-
import static com.android.ravenwood.common.RavenwoodCommonUtils.ensureIsPublicVoidMethod;
import static com.android.ravenwood.common.RavenwoodCommonUtils.isOnRavenwood;
@@ -164,7 +162,8 @@
* If the class has @DisabledOnRavenwood, then we'll delegate to ClassSkippingTestRunner,
* which simply skips it.
*/
- if (isOnRavenwood() && !shouldRunCassOnRavenwood(mTestClsas.getJavaClass())) {
+ if (isOnRavenwood() && !RavenwoodAwareTestRunnerHook.shouldRunClassOnRavenwood(
+ mTestClsas.getJavaClass())) {
mRealRunner = new ClassSkippingTestRunner(mTestClsas);
return;
}
@@ -238,6 +237,7 @@
public void run(RunNotifier notifier) {
if (mRealRunner instanceof ClassSkippingTestRunner) {
mRealRunner.run(notifier);
+ RavenwoodAwareTestRunnerHook.onClassSkipped(getDescription());
return;
}
@@ -294,19 +294,23 @@
}
private void runWithHooks(Description description, Scope scope, Order order, Statement s) {
- Throwable th = null;
if (isOnRavenwood()) {
Assume.assumeTrue(
RavenwoodAwareTestRunnerHook.onBefore(this, description, scope, order));
}
try {
s.evaluate();
- } catch (Throwable t) {
- th = t;
- SneakyThrow.sneakyThrow(t);
- } finally {
if (isOnRavenwood()) {
- RavenwoodAwareTestRunnerHook.onAfter(this, description, scope, order, th);
+ RavenwoodAwareTestRunnerHook.onAfter(this, description, scope, order, null);
+ }
+ } catch (Throwable t) {
+ boolean shouldThrow = true;
+ if (isOnRavenwood()) {
+ shouldThrow = RavenwoodAwareTestRunnerHook.onAfter(
+ this, description, scope, order, t);
+ }
+ if (shouldThrow) {
+ SneakyThrow.sneakyThrow(t);
}
}
}
diff --git a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodClassRule.java b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodClassRule.java
index 6c8d96a..85297fe 100644
--- a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodClassRule.java
+++ b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodClassRule.java
@@ -16,37 +16,20 @@
package android.platform.test.ravenwood;
-import static android.platform.test.ravenwood.RavenwoodRule.ENABLE_PROBE_IGNORED;
-import static android.platform.test.ravenwood.RavenwoodRule.IS_ON_RAVENWOOD;
-import static android.platform.test.ravenwood.RavenwoodRule.shouldEnableOnRavenwood;
-import static android.platform.test.ravenwood.RavenwoodRule.shouldStillIgnoreInProbeIgnoreMode;
-
-import android.platform.test.annotations.DisabledOnRavenwood;
-import android.platform.test.annotations.EnabledOnRavenwood;
-
-import org.junit.Assume;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
/**
- * {@code @ClassRule} that respects Ravenwood-specific class annotations. This rule has no effect
- * when tests are run on non-Ravenwood test environments.
+ * No longer needed.
*
- * By default, all tests are executed on Ravenwood, but annotations such as
- * {@link DisabledOnRavenwood} and {@link EnabledOnRavenwood} can be used at both the method
- * and class level to "ignore" tests that may not be ready.
+ * @deprecated this class used to be used to handle the class level annotation, which
+ * is now done by the test runner, so this class is not needed.
*/
+@Deprecated
public class RavenwoodClassRule implements TestRule {
@Override
public Statement apply(Statement base, Description description) {
- if (!IS_ON_RAVENWOOD) {
- // No check on a real device.
- } else if (ENABLE_PROBE_IGNORED) {
- Assume.assumeFalse(shouldStillIgnoreInProbeIgnoreMode(description));
- } else {
- Assume.assumeTrue(shouldEnableOnRavenwood(description));
- }
return base;
}
}
diff --git a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
index 75faafb..d569896 100644
--- a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
+++ b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
@@ -20,22 +20,20 @@
import static android.os.Process.SYSTEM_UID;
import static android.os.UserHandle.SYSTEM;
-import static org.junit.Assert.fail;
+import static com.android.ravenwood.common.RavenwoodCommonUtils.log;
+import android.annotation.Nullable;
import android.app.Instrumentation;
import android.content.Context;
import android.platform.test.annotations.DisabledOnRavenwood;
import android.platform.test.annotations.EnabledOnRavenwood;
-import android.platform.test.annotations.IgnoreUnderRavenwood;
import com.android.ravenwood.common.RavenwoodCommonUtils;
-import org.junit.Assume;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -56,18 +54,18 @@
* before a test class is fully initialized.
*/
public class RavenwoodRule implements TestRule {
+ private static final String TAG = "RavenwoodRule";
+
static final boolean IS_ON_RAVENWOOD = RavenwoodCommonUtils.isOnRavenwood();
/**
- * When probing is enabled, all tests will be unconditionally run on Ravenwood to detect
+ * When this flag is enabled, all tests will be unconditionally run on Ravenwood to detect
* cases where a test is able to pass despite being marked as {@link DisabledOnRavenwood}.
*
* This is typically helpful for internal maintainers discovering tests that had previously
* been ignored, but now have enough Ravenwood-supported functionality to be enabled.
- *
- * TODO: Rename it to a more descriptive name.
*/
- static final boolean ENABLE_PROBE_IGNORED = "1".equals(
+ private static final boolean RUN_DISABLED_TESTS = "1".equals(
System.getenv("RAVENWOOD_RUN_DISABLED_TESTS"));
/**
@@ -92,23 +90,17 @@
*
* Because we use a regex-find, setting "." would disable all tests.
*/
- private static final Pattern REALLY_DISABLE_PATTERN = Pattern.compile(
- Objects.requireNonNullElse(System.getenv("RAVENWOOD_REALLY_DISABLE"), ""));
+ private static final Pattern REALLY_DISABLED_PATTERN = Pattern.compile(
+ Objects.requireNonNullElse(System.getenv("RAVENWOOD_REALLY_DISABLED"), ""));
- private static final boolean ENABLE_REALLY_DISABLE_PATTERN =
- !REALLY_DISABLE_PATTERN.pattern().isEmpty();
-
- /**
- * If true, enable optional validation on running tests.
- */
- private static final boolean ENABLE_OPTIONAL_VALIDATION = "1".equals(
- System.getenv("RAVENWOOD_OPTIONAL_VALIDATION"));
+ private static final boolean HAS_REALLY_DISABLE_PATTERN =
+ !REALLY_DISABLED_PATTERN.pattern().isEmpty();
static {
- if (ENABLE_PROBE_IGNORED) {
- System.out.println("$RAVENWOOD_RUN_DISABLED_TESTS enabled: force running all tests");
- if (ENABLE_REALLY_DISABLE_PATTERN) {
- System.out.println("$RAVENWOOD_REALLY_DISABLE=" + REALLY_DISABLE_PATTERN.pattern());
+ if (RUN_DISABLED_TESTS) {
+ log(TAG, "$RAVENWOOD_RUN_DISABLED_TESTS enabled: force running all tests");
+ if (HAS_REALLY_DISABLE_PATTERN) {
+ log(TAG, "$RAVENWOOD_REALLY_DISABLED=" + REALLY_DISABLED_PATTERN.pattern());
}
}
}
@@ -275,103 +267,18 @@
"Instrumentation is only available during @Test execution");
}
- /**
- * Determine if the given {@link Description} should be enabled when running on the
- * Ravenwood test environment.
- *
- * A more specific method-level annotation always takes precedence over any class-level
- * annotation, and an {@link EnabledOnRavenwood} annotation always takes precedence over
- * an {@link DisabledOnRavenwood} annotation.
- */
- public static boolean shouldEnableOnRavenwood(Description description) {
- // First, consult any method-level annotations
- if (description.isTest()) {
- // Stopgap for http://g/ravenwood/EPAD-N5ntxM
- if (description.getMethodName().endsWith("$noRavenwood")) {
- return false;
- }
- if (description.getAnnotation(EnabledOnRavenwood.class) != null) {
- return true;
- }
- if (description.getAnnotation(DisabledOnRavenwood.class) != null) {
- return false;
- }
- if (description.getAnnotation(IgnoreUnderRavenwood.class) != null) {
- return false;
- }
- }
-
- // Otherwise, consult any class-level annotations
- return shouldRunCassOnRavenwood(description.getTestClass());
- }
-
- public static boolean shouldRunCassOnRavenwood(Class<?> clazz) {
- if (clazz != null) {
- if (clazz.getAnnotation(EnabledOnRavenwood.class) != null) {
- return true;
- }
- if (clazz.getAnnotation(DisabledOnRavenwood.class) != null) {
- return false;
- }
- if (clazz.getAnnotation(IgnoreUnderRavenwood.class) != null) {
- return false;
- }
- }
- return true;
- }
-
- static boolean shouldStillIgnoreInProbeIgnoreMode(Description description) {
- if (!ENABLE_REALLY_DISABLE_PATTERN) {
- return false;
- }
-
- final var fullname = description.getTestClass().getName()
- + (description.isTest() ? "#" + description.getMethodName() : "");
-
- if (REALLY_DISABLE_PATTERN.matcher(fullname).find()) {
- System.out.println("Still ignoring " + fullname);
- return true;
- }
- return false;
- }
@Override
public Statement apply(Statement base, Description description) {
- // No special treatment when running outside Ravenwood; run tests as-is
- if (!IS_ON_RAVENWOOD) {
- return base;
- }
-
- if (ENABLE_PROBE_IGNORED) {
- return applyProbeIgnored(base, description);
- } else {
- return applyDefault(base, description);
- }
- }
-
- private void commonPrologue(Statement base, Description description) throws IOException {
- RavenwoodRuleImpl.logTestRunner("started", description);
- RavenwoodRuleImpl.validate(base, description, ENABLE_OPTIONAL_VALIDATION);
- RavenwoodRuleImpl.init(RavenwoodRule.this);
- }
-
- /**
- * Run the given {@link Statement} with no special treatment.
- */
- private Statement applyDefault(Statement base, Description description) {
+ // TODO: Here, we're calling init() / reset() once for each rule.
+ // That means if a test class has multiple rules -- even if they refer to the same
+ // rule instance -- we're calling them multiple times. We need to fix it.
return new Statement() {
@Override
public void evaluate() throws Throwable {
- Assume.assumeTrue(shouldEnableOnRavenwood(description));
-
- commonPrologue(base, description);
+ RavenwoodRuleImpl.init(RavenwoodRule.this);
try {
base.evaluate();
-
- RavenwoodRuleImpl.logTestRunner("finished", description);
- } catch (Throwable t) {
- RavenwoodRuleImpl.logTestRunner("failed", description);
- throw t;
} finally {
RavenwoodRuleImpl.reset(RavenwoodRule.this);
}
@@ -380,44 +287,6 @@
}
/**
- * Run the given {@link Statement} with probing enabled. All tests will be unconditionally
- * run on Ravenwood to detect cases where a test is able to pass despite being marked as
- * {@code IgnoreUnderRavenwood}.
- */
- private Statement applyProbeIgnored(Statement base, Description description) {
- return new Statement() {
- @Override
- public void evaluate() throws Throwable {
- Assume.assumeFalse(shouldStillIgnoreInProbeIgnoreMode(description));
-
- commonPrologue(base, description);
- try {
- base.evaluate();
- } catch (Throwable t) {
- // If the test isn't included, eat the exception and report the
- // assumption failure that test authors expect; otherwise throw
- Assume.assumeTrue(shouldEnableOnRavenwood(description));
- throw t;
- } finally {
- RavenwoodRuleImpl.logTestRunner("finished", description);
- RavenwoodRuleImpl.reset(RavenwoodRule.this);
- }
-
- if (!shouldEnableOnRavenwood(description)) {
- fail("Test wasn't included under Ravenwood, but it actually "
- + "passed under Ravenwood; consider updating annotations");
- }
- }
- };
- }
-
- public static class _$RavenwoodPrivate {
- public static boolean isOptionalValidationEnabled() {
- return ENABLE_OPTIONAL_VALIDATION;
- }
- }
-
- /**
* Returns the "real" result from {@link System#currentTimeMillis()}.
*
* Currently, it's the same thing as calling {@link System#currentTimeMillis()},
@@ -427,4 +296,47 @@
public long realCurrentTimeMillis() {
return System.currentTimeMillis();
}
+
+ // Below are internal to ravenwood. Don't use them from normal tests...
+
+ public static class RavenwoodPrivate {
+ private RavenwoodPrivate() {
+ }
+
+ private volatile Boolean mRunDisabledTestsOverride = null;
+
+ private volatile Pattern mReallyDisabledPattern = null;
+
+ public boolean isRunningDisabledTests() {
+ if (mRunDisabledTestsOverride != null) {
+ return mRunDisabledTestsOverride;
+ }
+ return RUN_DISABLED_TESTS;
+ }
+
+ public Pattern getReallyDisabledPattern() {
+ if (mReallyDisabledPattern != null) {
+ return mReallyDisabledPattern;
+ }
+ return REALLY_DISABLED_PATTERN;
+ }
+
+ public void overrideRunDisabledTest(boolean runDisabledTests,
+ @Nullable String reallyDisabledPattern) {
+ mRunDisabledTestsOverride = runDisabledTests;
+ mReallyDisabledPattern =
+ reallyDisabledPattern == null ? null : Pattern.compile(reallyDisabledPattern);
+ }
+
+ public void resetRunDisabledTest() {
+ mRunDisabledTestsOverride = null;
+ mReallyDisabledPattern = null;
+ }
+ }
+
+ private static final RavenwoodPrivate sRavenwoodPrivate = new RavenwoodPrivate();
+
+ public static RavenwoodPrivate private$ravenwood() {
+ return sRavenwoodPrivate;
+ }
}
diff --git a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
index 6b80e0c..1e4889c 100644
--- a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
+++ b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
@@ -23,27 +23,51 @@
import org.junit.runners.model.TestClass;
/**
- * Provide hook points created by {@link RavenwoodAwareTestRunner}.
+ * Provide hook points created by {@link RavenwoodAwareTestRunner}. This is a version
+ * that's used on a device side test.
+ *
+ * All methods are no-op in real device tests.
+ *
+ * TODO: Use some kind of factory to provide different implementation for the device test
+ * and the ravenwood test.
*/
public class RavenwoodAwareTestRunnerHook {
private RavenwoodAwareTestRunnerHook() {
}
/**
- * Called when a runner starts, befre the inner runner gets a chance to run.
+ * Called when a runner starts, before the inner runner gets a chance to run.
*/
public static void onRunnerInitializing(Runner runner, TestClass testClass) {
- // No-op on a real device.
}
+ /**
+ * Called when a whole test class is skipped.
+ */
+ public static void onClassSkipped(Description description) {
+ }
+
+ /**
+ * Called before a test / class.
+ *
+ * Return false if it should be skipped.
+ */
public static boolean onBefore(RavenwoodAwareTestRunner runner, Description description,
Scope scope, Order order) {
- // No-op on a real device.
return true;
}
- public static void onAfter(RavenwoodAwareTestRunner runner, Description description,
+ /**
+ * Called after a test / class.
+ *
+ * Return false if the exception should be ignored.
+ */
+ public static boolean onAfter(RavenwoodAwareTestRunner runner, Description description,
Scope scope, Order order, Throwable th) {
- // No-op on a real device.
+ return true;
+ }
+
+ public static boolean shouldRunClassOnRavenwood(Class<?> clazz) {
+ return true;
}
}
diff --git a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
index 483b98a..a470626 100644
--- a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
+++ b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
@@ -17,7 +17,6 @@
package android.platform.test.ravenwood;
import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
public class RavenwoodRuleImpl {
public static void init(RavenwoodRule rule) {
@@ -32,10 +31,6 @@
// No-op when running on a real device
}
- public static void validate(Statement base, Description description,
- boolean enableOptionalValidation) {
- }
-
public static long realCurrentTimeMillis() {
return System.currentTimeMillis();
}