Skip presubmit for flaky tests
Skip presubmit for tests that have been reported to be >0.5% flaky in
presubmit, or in mainline-presubmit. Tests that only meet that criteria
in mainline-presubmit are only marked as @SkipMainlinePresubmit.
Ensure that the tests are still run in postsubmit so they can be moved
back to presubmit after getting fixed.
Test: TH
Bug: 284534251
Bug: 292174275
Bug: 292173706
Bug: 292174832
Bug: 289879947
Bug: 286701510
Bug: 293239598
Bug: 279846955
Bug: 277567450
Bug: 293239828
Bug: 293240365
Change-Id: I986a1079709057d3210016c50b632fe0f3bff1e2
diff --git a/TEST_MAPPING b/TEST_MAPPING
index 1e110f5..7e70e6c 100644
--- a/TEST_MAPPING
+++ b/TEST_MAPPING
@@ -126,6 +126,15 @@
{
"name": "FrameworksNetDeflakeTest"
},
+ // Postsubmit on virtual devices to monitor flakiness of @SkipPresubmit methods
+ {
+ "name": "CtsNetTestCases",
+ "options": [
+ {
+ "exclude-annotation": "androidx.test.filters.RequiresDevice"
+ }
+ ]
+ },
{
"name": "FrameworksNetTests"
}
@@ -138,6 +147,9 @@
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
+ "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+ },
+ {
"exclude-annotation": "androidx.test.filters.RequiresDevice"
}
]
@@ -149,6 +161,9 @@
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
+ "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+ },
+ {
"exclude-annotation": "androidx.test.filters.RequiresDevice"
}
]
@@ -160,6 +175,9 @@
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
+ "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+ },
+ {
"exclude-annotation": "androidx.test.filters.RequiresDevice"
}
]
@@ -171,6 +189,9 @@
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
+ "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+ },
+ {
"exclude-annotation": "androidx.test.filters.RequiresDevice"
}
]
@@ -184,6 +205,9 @@
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
+ "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+ },
+ {
"exclude-annotation": "androidx.test.filters.RequiresDevice"
},
{
@@ -205,6 +229,9 @@
"exclude-annotation": "com.android.testutils.SkipPresubmit"
},
{
+ "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+ },
+ {
"exclude-annotation": "androidx.test.filters.RequiresDevice"
},
{
@@ -262,6 +289,15 @@
"exclude-annotation": "com.android.testutils.NetworkStackModuleTest"
}
]
+ },
+ // Postsubmit on virtual devices to monitor flakiness of @SkipMainlinePresubmit methods
+ {
+ "name": "CtsNetTestCasesLatestSdk[CaptivePortalLoginGoogle.apk+NetworkStackGoogle.apk+com.google.android.resolv.apex+com.google.android.tethering.apex]",
+ "options": [
+ {
+ "exclude-annotation": "androidx.test.filters.RequiresDevice"
+ }
+ ]
}
],
"imports": [
diff --git a/tests/cts/net/src/android/net/cts/CaptivePortalTest.kt b/tests/cts/net/src/android/net/cts/CaptivePortalTest.kt
index dc22369..1d79806 100644
--- a/tests/cts/net/src/android/net/cts/CaptivePortalTest.kt
+++ b/tests/cts/net/src/android/net/cts/CaptivePortalTest.kt
@@ -49,6 +49,7 @@
import com.android.net.module.util.NetworkStackConstants.TEST_CAPTIVE_PORTAL_HTTP_URL
import com.android.testutils.DeviceConfigRule
import com.android.testutils.RecorderCallback.CallbackEntry.CapabilitiesChanged
+import com.android.testutils.SkipMainlinePresubmit
import com.android.testutils.TestHttpServer
import com.android.testutils.TestHttpServer.Request
import com.android.testutils.TestableNetworkCallback
@@ -137,6 +138,7 @@
}
@Test
+ @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
fun testCaptivePortalIsNotDefaultNetwork() {
assumeTrue(pm.hasSystemFeature(FEATURE_TELEPHONY))
assumeTrue(pm.hasSystemFeature(FEATURE_WIFI))
diff --git a/tests/cts/net/src/android/net/cts/ConnectivityManagerTest.java b/tests/cts/net/src/android/net/cts/ConnectivityManagerTest.java
index a0508e1..c2dc45b 100644
--- a/tests/cts/net/src/android/net/cts/ConnectivityManagerTest.java
+++ b/tests/cts/net/src/android/net/cts/ConnectivityManagerTest.java
@@ -197,6 +197,8 @@
import com.android.testutils.DeviceInfoUtils;
import com.android.testutils.DumpTestUtils;
import com.android.testutils.RecorderCallback.CallbackEntry;
+import com.android.testutils.SkipMainlinePresubmit;
+import com.android.testutils.SkipPresubmit;
import com.android.testutils.TestHttpServer;
import com.android.testutils.TestNetworkTracker;
import com.android.testutils.TestableNetworkCallback;
@@ -1017,6 +1019,7 @@
@AppModeFull(reason = "WRITE_SECURE_SETTINGS permission can't be granted to instant apps")
@Test @IgnoreUpTo(Build.VERSION_CODES.Q)
+ @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testIsPrivateDnsBroken() throws InterruptedException {
final String invalidPrivateDnsServer = "invalidhostname.example.com";
final String goodPrivateDnsServer = "dns.google";
@@ -1124,6 +1127,7 @@
*/
@AppModeFull(reason = "Cannot get WifiManager in instant app mode")
@Test
+ @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testRegisterNetworkCallback_withPendingIntent() {
assumeTrue(mPackageManager.hasSystemFeature(FEATURE_WIFI));
@@ -1267,6 +1271,7 @@
@AppModeFull(reason = "Cannot get WifiManager in instant app mode")
@Test
+ @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testRegisterNetworkRequest_identicalPendingIntents() throws Exception {
runIdenticalPendingIntentsRequestTest(false /* useListen */);
}
@@ -2129,6 +2134,7 @@
*/
@AppModeFull(reason = "NETWORK_AIRPLANE_MODE permission can't be granted to instant apps")
@Test
+ @SkipPresubmit(reason = "Out of SLO flakiness")
public void testSetAirplaneMode() throws Exception{
// Starting from T, wifi supports airplane mode enhancement which may not disconnect wifi
// when airplane mode is on. The actual behavior that the device will have could only be
@@ -2738,6 +2744,7 @@
*/
@AppModeFull(reason = "Instant apps cannot create test networks")
@Test
+ @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testSetOemNetworkPreferenceForTestOnlyPref() throws Exception {
// Cannot use @IgnoreUpTo(Build.VERSION_CODES.R) because this test also requires API 31
// shims, and @IgnoreUpTo does not check that.
@@ -2892,6 +2899,7 @@
@AppModeFull(reason = "WRITE_DEVICE_CONFIG permission can't be granted to instant apps")
@Test
+ @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testRejectPartialConnectivity_TearDownNetwork() throws Exception {
assumeTrue(TestUtils.shouldTestSApis());
assumeTrue("testAcceptPartialConnectivity_validatedNetwork cannot execute"
@@ -3197,6 +3205,7 @@
@AppModeFull(reason = "Cannot get WifiManager in instant app mode")
@Test
+ @SkipPresubmit(reason = "Out of SLO flakiness")
public void testMobileDataPreferredUids() throws Exception {
assumeTrue(TestUtils.shouldTestSApis());
final boolean canRunTest = mPackageManager.hasSystemFeature(FEATURE_WIFI)
diff --git a/tests/cts/net/src/android/net/cts/Ikev2VpnTest.java b/tests/cts/net/src/android/net/cts/Ikev2VpnTest.java
index 805dd65..81834a9 100644
--- a/tests/cts/net/src/android/net/cts/Ikev2VpnTest.java
+++ b/tests/cts/net/src/android/net/cts/Ikev2VpnTest.java
@@ -71,6 +71,8 @@
import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo;
import com.android.testutils.DevSdkIgnoreRunner;
import com.android.testutils.RecorderCallback.CallbackEntry;
+import com.android.testutils.SkipMainlinePresubmit;
+import com.android.testutils.SkipPresubmit;
import com.android.testutils.TestableNetworkCallback;
import org.bouncycastle.x509.X509V1CertificateGenerator;
@@ -640,6 +642,7 @@
}
@Test
+ @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testStartStopVpnProfileV4() throws Exception {
doTestStartStopVpnProfile(false /* testIpv6Only */, false /* requiresValidation */,
false /* testSessionKey */, false /* testIkeTunConnParams */);
@@ -653,12 +656,14 @@
}
@Test
+ @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testStartStopVpnProfileV6() throws Exception {
doTestStartStopVpnProfile(true /* testIpv6Only */, false /* requiresValidation */,
false /* testSessionKey */, false /* testIkeTunConnParams */);
}
@Test @IgnoreUpTo(SC_V2)
+ @SkipPresubmit(reason = "Out of SLO flakiness")
public void testStartStopVpnProfileV6WithValidation() throws Exception {
assumeTrue(TestUtils.shouldTestTApis());
doTestStartStopVpnProfile(true /* testIpv6Only */, true /* requiresValidation */,
diff --git a/tests/cts/net/src/android/net/cts/IpSecManagerTest.java b/tests/cts/net/src/android/net/cts/IpSecManagerTest.java
index cc0a5df..fe86a90 100644
--- a/tests/cts/net/src/android/net/cts/IpSecManagerTest.java
+++ b/tests/cts/net/src/android/net/cts/IpSecManagerTest.java
@@ -81,6 +81,7 @@
import com.android.modules.utils.build.SdkLevel;
import com.android.testutils.DevSdkIgnoreRule;
import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo;
+import com.android.testutils.SkipMainlinePresubmit;
import org.junit.Rule;
import org.junit.Test;
@@ -716,6 +717,7 @@
}
@Test
+ @SkipMainlinePresubmit(reason = "Out of SLO flakiness")
public void testIkeOverUdpEncapSocket() throws Exception {
// IPv6 not supported for UDP-encap-ESP
InetAddress local = InetAddress.getByName(IPV4_LOOPBACK);