patch 8.1.0529: flaky test sometimes fails in different ways
Problem: Flaky test sometimes fails in different ways.
Solution: When the second run gives a different error, try running the test
again, up to five times.
diff --git a/src/testdir/runtest.vim b/src/testdir/runtest.vim
index cacfa75..2c22498 100644
--- a/src/testdir/runtest.vim
+++ b/src/testdir/runtest.vim
@@ -314,28 +314,47 @@
for s:test in sort(s:tests)
" Silence, please!
set belloff=all
+ let prev_error = ''
+ let total_errors = []
+ let run_nr = 1
call RunTheTest(s:test)
+ " Repeat a flaky test. Give up when:
+ " - it fails again with the same message
+ " - it fails five times (with a different mesage)
if len(v:errors) > 0 && index(s:flaky, s:test) >= 0
- call add(s:messages, 'Found errors in ' . s:test . ':')
- call extend(s:messages, v:errors)
- call add(s:messages, 'Flaky test failed, running it again')
- let first_run = v:errors
+ while 1
+ call add(s:messages, 'Found errors in ' . s:test . ':')
+ call extend(s:messages, v:errors)
- " Flakiness is often caused by the system being very busy. Sleep a couple
- " of seconds to have a higher chance of succeeding the second time.
- sleep 2
+ call add(total_errors, 'Run ' . run_nr . ':')
+ call extend(total_errors, v:errors)
- let v:errors = []
- call RunTheTest(s:test)
- if len(v:errors) > 0
- let second_run = v:errors
- let v:errors = ['First run:']
- call extend(v:errors, first_run)
- call add(v:errors, 'Second run:')
- call extend(v:errors, second_run)
- endif
+ if run_nr == 5 || prev_error == v:errors[0]
+ call add(total_errors, 'Flaky test failed too often, giving up')
+ let v:errors = total_errors
+ break
+ endif
+
+ call add(s:messages, 'Flaky test failed, running it again')
+
+ " Flakiness is often caused by the system being very busy. Sleep a
+ " couple of seconds to have a higher chance of succeeding the second
+ " time.
+ sleep 2
+
+ let prev_error = v:errors[0]
+ let v:errors = []
+ let run_nr += 1
+
+ call RunTheTest(s:test)
+
+ if len(v:errors) == 0
+ " Test passed on rerun.
+ break
+ endif
+ endwhile
endif
call AfterTheTest()
diff --git a/src/version.c b/src/version.c
index e72adfc..ebea70f 100644
--- a/src/version.c
+++ b/src/version.c
@@ -793,6 +793,8 @@
static int included_patches[] =
{ /* Add new patch number below this line */
/**/
+ 529,
+/**/
528,
/**/
527,