commit 8db52a59874ff1e9871b1f35ea0617827abd201c
author Remi NGUYEN VAN <reminv@google.com> Wed Jul 26 16:15:03 2023 +0900
committer Remi NGUYEN VAN <reminv@google.com> Thu Jul 27 02:36:38 2023 +0000
tree 7d79f4431099c1af0a12bc8563da3c81c9f8fa8a
parent b8fc7552597529bb881374a0cda02cf5176cfd44

Skip presubmit for flaky tests

Skip presubmit for tests that have been reported to be >0.5% flaky in
presubmit, or in mainline-presubmit. Tests that only meet that criteria
in mainline-presubmit are only marked as @SkipMainlinePresubmit.

Ensure that the tests are still run in postsubmit so they can be moved
back to presubmit after getting fixed.

Test: TH
Bug: 284534251
Bug: 292174275
Bug: 292173706
Bug: 292174832
Bug: 289879947
Bug: 286701510
Bug: 293239598
Bug: 279846955
Bug: 277567450
Bug: 293239828
Bug: 293240365
Change-Id: I986a1079709057d3210016c50b632fe0f3bff1e2

TEST_MAPPING

diff --git a/TEST_MAPPING b/TEST_MAPPING
index 1e110f5..7e70e6c 100644
--- a/TEST_MAPPING
+++ b/TEST_MAPPING
@@ -126,6 +126,15 @@
     {
       "name": "FrameworksNetDeflakeTest"
     },
+    // Postsubmit on virtual devices to monitor flakiness of @SkipPresubmit methods
+    {
+      "name": "CtsNetTestCases",
+      "options": [
+        {
+          "exclude-annotation": "androidx.test.filters.RequiresDevice"
+        }
+      ]
+    },
     {
       "name": "FrameworksNetTests"
     }
@@ -138,6 +147,9 @@
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
         {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
+        {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
       ]
@@ -149,6 +161,9 @@
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
         {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
+        {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
       ]
@@ -160,6 +175,9 @@
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
         {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
+        {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
       ]
@@ -171,6 +189,9 @@
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
         {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
+        {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         }
       ]
@@ -184,6 +205,9 @@
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
         {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
+        {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         },
         {
@@ -205,6 +229,9 @@
           "exclude-annotation": "com.android.testutils.SkipPresubmit"
         },
         {
+          "exclude-annotation": "com.android.testutils.SkipMainlinePresubmit"
+        },
+        {
           "exclude-annotation": "androidx.test.filters.RequiresDevice"
         },
         {
@@ -262,6 +289,15 @@
           "exclude-annotation": "com.android.testutils.NetworkStackModuleTest"
         }
       ]
+    },
+    // Postsubmit on virtual devices to monitor flakiness of @SkipMainlinePresubmit methods
+    {
+      "name": "CtsNetTestCasesLatestSdk[CaptivePortalLoginGoogle.apk+NetworkStackGoogle.apk+com.google.android.resolv.apex+com.google.android.tethering.apex]",
+      "options": [
+        {
+          "exclude-annotation": "androidx.test.filters.RequiresDevice"
+        }
+      ]
     }
   ],
   "imports": [