patch 9.1.0820: tests: Mac OS tests are too flaky
Problem: tests: Mac OS tests are too flaky
Solution: Increase max test timeout to 25 minutes,
allow up to 10 retries on Mac OS runners,
refactor runtest.vim (Milly).
closes: #15940
Co-authored-by: K.Takata <kentkt@csc.jp>
Signed-off-by: Milly <milly.ca@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>
diff --git a/src/testdir/runtest.vim b/src/testdir/runtest.vim
index 7ac4845..ded3197 100644
--- a/src/testdir/runtest.vim
+++ b/src/testdir/runtest.vim
@@ -190,10 +190,6 @@
return lnum - top - 1
endfunc
-if has('reltime')
- let g:func_start = reltime()
-endif
-
" Get the list of swap files in the current directory.
func s:GetSwapFileList()
let save_dir = &directory
@@ -603,6 +599,16 @@
" A test can set g:test_is_flaky to retry running the test.
let g:test_is_flaky = 0
+ " A test can set g:max_run_nr to change the max retry count.
+ let g:max_run_nr = 5
+ if has('mac')
+ let g:max_run_nr = 10
+ endif
+
+ " By default, give up if the same error occurs. A test can set
+ " g:giveup_same_error to 0 to not give up on the same error and keep trying.
+ let g:giveup_same_error = 1
+
let starttime = strftime("%H:%M:%S")
call RunTheTest(g:testfunc)
@@ -618,10 +624,15 @@
call extend(s:messages, v:errors)
let endtime = strftime("%H:%M:%S")
- call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}:')
+ if has('reltime')
+ let suffix = $' in{reltimestr(reltime(g:func_start))} seconds'
+ else
+ let suffix = ''
+ endif
+ call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}{suffix}:')
call extend(total_errors, v:errors)
- if g:run_nr >= 5 || prev_error == v:errors[0]
+ if g:run_nr >= g:max_run_nr || g:giveup_same_error && prev_error == v:errors[0]
call add(total_errors, 'Flaky test failed too often, giving up')
let v:errors = total_errors
break
@@ -632,7 +643,8 @@
" Flakiness is often caused by the system being very busy. Sleep a
" couple of seconds to have a higher chance of succeeding the second
" time.
- sleep 2
+ let delay = g:run_nr * 2
+ exe 'sleep' delay
let prev_error = v:errors[0]
let v:errors = []
diff --git a/src/testdir/screendump.vim b/src/testdir/screendump.vim
index e7cda2e..84b4864 100644
--- a/src/testdir/screendump.vim
+++ b/src/testdir/screendump.vim
@@ -56,6 +56,7 @@
" Starting a terminal to make a screendump is always considered flaky.
let g:test_is_flaky = 1
+ let g:giveup_same_error = 0
" wait for the pending updates to be handled.
call TermWait(a:buf)
@@ -83,41 +84,55 @@
sleep 50m
call delete(testfile)
call term_dumpwrite(a:buf, testfile, a:options)
+
+ if refdump->empty()
+ let msg = 'See new dump file: call term_dumpload("testdir/' .. testfile .. '")'
+ call assert_report(msg)
+ " no point in retrying
+ let g:run_nr = 10
+ return 1
+ endif
+
let testdump = ReadAndFilter(testfile, filter)
if refdump == testdump
call delete(testfile)
if did_mkdir
call delete('failed', 'd')
endif
+ if i > 0
+ call remove(v:errors, -1)
+ endif
break
endif
- if i == max_loops
- " Leave the failed dump around for inspection.
- if filereadable(reference)
- let msg = 'See dump file difference: call term_dumpdiff("testdir/' .. testfile .. '", "testdir/' .. reference .. '")'
- if a:0 == 1
- let msg = a:1 . ': ' . msg
- endif
- if len(testdump) != len(refdump)
- let msg = msg . '; line count is ' . len(testdump) . ' instead of ' . len(refdump)
- endif
- else
- let msg = 'See new dump file: call term_dumpload("testdir/' .. testfile .. '")'
- " no point in retrying
- let g:run_nr = 10
+
+ " Leave the failed dump around for inspection.
+ let msg = 'See dump file difference: call term_dumpdiff("testdir/' .. testfile .. '", "testdir/' .. reference .. '")'
+ if a:0 == 1
+ let msg = a:1 . ': ' . msg
+ endif
+ if len(testdump) != len(refdump)
+ let msg = msg . '; line count is ' . len(testdump) . ' instead of ' . len(refdump)
+ endif
+ for j in range(len(refdump))
+ if j >= len(testdump)
+ break
endif
- for i in range(len(refdump))
- if i >= len(testdump)
- break
- endif
- if testdump[i] != refdump[i]
- let msg = msg . '; difference in line ' . (i + 1) . ': "' . testdump[i] . '"'
- endif
- endfor
- call assert_report(msg)
+ if testdump[j] != refdump[j]
+ let msg = msg . '; difference in line ' . (j + 1) . ': "' . testdump[j] . '"'
+ endif
+ endfor
+
+ " Always add the last error so that it is displayed on timeout.
+ " See TestTimeout() in runtest.vim.
+ if i > 0
+ call remove(v:errors, -1)
+ endif
+ call assert_report(msg)
+
+ let i += 1
+ if i >= max_loops
return 1
endif
- let i += 1
endwhile
return 0
endfunc
diff --git a/src/testdir/test_channel.vim b/src/testdir/test_channel.vim
index 78c834d..2d40eff 100644
--- a/src/testdir/test_channel.vim
+++ b/src/testdir/test_channel.vim
@@ -2762,6 +2762,8 @@
endfunc
func Test_channel_lsp_mode()
+ " The channel lsp mode test is flaky and gives the same error.
+ let g:giveup_same_error = 0
call RunServer('test_channel_lsp.py', 'LspTests', [])
endfunc
diff --git a/src/version.c b/src/version.c
index 838f25a..e042965 100644
--- a/src/version.c
+++ b/src/version.c
@@ -705,6 +705,8 @@
static int included_patches[] =
{ /* Add new patch number below this line */
/**/
+ 820,
+/**/
819,
/**/
818,