patch 9.1.0820: tests: Mac OS tests are too flaky
Problem: tests: Mac OS tests are too flaky
Solution: Increase max test timeout to 25 minutes,
allow up to 10 retries on Mac OS runners,
refactor runtest.vim (Milly).
closes: #15940
Co-authored-by: K.Takata <kentkt@csc.jp>
Signed-off-by: Milly <milly.ca@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>
diff --git a/src/testdir/runtest.vim b/src/testdir/runtest.vim
index 7ac4845..ded3197 100644
--- a/src/testdir/runtest.vim
+++ b/src/testdir/runtest.vim
@@ -190,10 +190,6 @@
return lnum - top - 1
endfunc
-if has('reltime')
- let g:func_start = reltime()
-endif
-
" Get the list of swap files in the current directory.
func s:GetSwapFileList()
let save_dir = &directory
@@ -603,6 +599,16 @@
" A test can set g:test_is_flaky to retry running the test.
let g:test_is_flaky = 0
+ " A test can set g:max_run_nr to change the max retry count.
+ let g:max_run_nr = 5
+ if has('mac')
+ let g:max_run_nr = 10
+ endif
+
+ " By default, give up if the same error occurs. A test can set
+ " g:giveup_same_error to 0 to not give up on the same error and keep trying.
+ let g:giveup_same_error = 1
+
let starttime = strftime("%H:%M:%S")
call RunTheTest(g:testfunc)
@@ -618,10 +624,15 @@
call extend(s:messages, v:errors)
let endtime = strftime("%H:%M:%S")
- call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}:')
+ if has('reltime')
+ let suffix = $' in{reltimestr(reltime(g:func_start))} seconds'
+ else
+ let suffix = ''
+ endif
+ call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}{suffix}:')
call extend(total_errors, v:errors)
- if g:run_nr >= 5 || prev_error == v:errors[0]
+ if g:run_nr >= g:max_run_nr || g:giveup_same_error && prev_error == v:errors[0]
call add(total_errors, 'Flaky test failed too often, giving up')
let v:errors = total_errors
break
@@ -632,7 +643,8 @@
" Flakiness is often caused by the system being very busy. Sleep a
" couple of seconds to have a higher chance of succeeding the second
" time.
- sleep 2
+ let delay = g:run_nr * 2
+ exe 'sleep' delay
let prev_error = v:errors[0]
let v:errors = []