blob: 7822da91503db26b99e9f42e6bc48c12936c013e [file] [log] [blame]
Bram Moolenaar43345542015-11-29 17:35:35 +01001" This script is sourced while editing the .vim file with the tests.
2" When the script is successful the .res file will be created.
3" Errors are appended to the test.log file.
4"
Bram Moolenaarbefb3662016-02-20 14:41:40 +01005" To execute only specific test functions, add a second argument. It will be
Bram Moolenaare219f732019-11-30 15:34:08 +01006" matched against the names of the Test_ function. E.g.:
Bram Moolenaarbefb3662016-02-20 14:41:40 +01007" ../vim -u NONE -S runtest.vim test_channel.vim open_delay
8" The output can be found in the "messages" file.
9"
Bram Moolenaarce436de2020-03-21 15:17:20 +010010" If the environment variable $TEST_FILTER is set then only test functions
11" matching this pattern are executed. E.g. for sh/bash:
12" export TEST_FILTER=Test_channel
13" For csh:
14" setenv TEST_FILTER Test_channel
15"
Bram Moolenaardae453f2021-08-07 17:20:16 +020016" If the environment variable $TEST_SKIP_PAT is set then test functions
17" matching this pattern will be skipped. It's the opposite of $TEST_FILTER.
18"
Bram Moolenaar6ca6ca42020-07-27 19:47:07 +020019" While working on a test you can make $TEST_NO_RETRY non-empty to not retry:
20" export TEST_NO_RETRY=yes
21"
Bram Moolenaarce436de2020-03-21 15:17:20 +010022" To ignore failure for tests that are known to fail in a certain environment,
23" set $TEST_MAY_FAIL to a comma separated list of function names. E.g. for
24" sh/bash:
25" export TEST_MAY_FAIL=Test_channel_one,Test_channel_other
26" The failure report will then not be included in the test.log file and
27" "make test" will not fail.
28"
Bram Moolenaar43345542015-11-29 17:35:35 +010029" The test script may contain anything, only functions that start with
30" "Test_" are special. These will be invoked and should contain assert
31" functions. See test_assert.vim for an example.
32"
33" It is possible to source other files that contain "Test_" functions. This
34" can speed up testing, since Vim does not need to restart. But be careful
35" that the tests do not interfere with each other.
36"
37" If an error cannot be detected properly with an assert function add the
38" error to the v:errors list:
39" call add(v:errors, 'test foo failed: Cannot find xyz')
40"
41" If preparation for each Test_ function is needed, define a SetUp function.
42" It will be called before each Test_ function.
43"
44" If cleanup after each Test_ function is needed, define a TearDown function.
45" It will be called after each Test_ function.
Bram Moolenaar00af60b2016-02-13 14:06:14 +010046"
47" When debugging a test it can be useful to add messages to v:errors:
Bram Moolenaar8ad16da2019-01-06 15:29:57 +010048" call add(v:errors, "this happened")
Bram Moolenaar00af60b2016-02-13 14:06:14 +010049
Bram Moolenaar43345542015-11-29 17:35:35 +010050
51" Without the +eval feature we can't run these tests, bail out.
Bram Moolenaarb96a32e2020-08-13 18:59:55 +020052silent! while 0
53 qa!
54silent! endwhile
Bram Moolenaar43345542015-11-29 17:35:35 +010055
Bram Moolenaar18aa13d2020-07-11 13:09:36 +020056" In the GUI we can always change the screen size.
57if has('gui_running')
matveytad3b6a32024-12-08 10:26:51 +010058 if has('gui_gtk')
Aliaksei Budaveic4eb1cb2025-06-08 15:52:42 +020059 " Use e.g. SetUp() and TearDown() to change "&guifont" when needed;
60 " otherwise, keep the following value to match current screendumps.
matveytad3b6a32024-12-08 10:26:51 +010061 set guifont=Monospace\ 10
62 endif
Aliaksei Budaveic4eb1cb2025-06-08 15:52:42 +020063
64 func s:SetDefaultOptionsForGUIBuilds()
65 set columns=80 lines=25
66 endfunc
67else
68 func s:SetDefaultOptionsForGUIBuilds()
69 endfunc
Bram Moolenaar18aa13d2020-07-11 13:09:36 +020070endif
71
Bram Moolenaar43345542015-11-29 17:35:35 +010072" Check that the screen size is at least 24 x 80 characters.
Bram Moolenaar94722c52023-01-28 19:19:03 +000073if &lines < 24 || &columns < 80
Bram Moolenaar0b5dc642019-08-11 22:56:15 +020074 let error = 'Screen size too small! Tests require at least 24 lines with 80 characters, got ' .. &lines .. ' lines with ' .. &columns .. ' characters'
Bram Moolenaar43345542015-11-29 17:35:35 +010075 echoerr error
76 split test.log
77 $put =error
Bram Moolenaar45aa07d2019-06-15 18:20:38 +020078 write
79 split messages
Bram Moolenaar0b5dc642019-08-11 22:56:15 +020080 call append(line('$'), '')
81 call append(line('$'), 'From ' . expand('%') . ':')
Bram Moolenaar45aa07d2019-06-15 18:20:38 +020082 call append(line('$'), error)
83 write
84 qa!
Bram Moolenaar43345542015-11-29 17:35:35 +010085endif
86
Bram Moolenaar75ee5442019-06-06 18:05:25 +020087if has('reltime')
Bram Moolenaarb9093d52022-09-23 19:42:31 +010088 let s:run_start_time = reltime()
89
90 if !filereadable('starttime')
91 " first test, store the overall test starting time
92 let s:test_start_time = localtime()
93 call writefile([string(s:test_start_time)], 'starttime')
94 else
95 " second or later test, read the overall test starting time
96 let s:test_start_time = readfile('starttime')[0]->str2nr()
97 endif
Bram Moolenaar75ee5442019-06-06 18:05:25 +020098endif
99
Bakudankun29f3a452021-12-11 12:28:08 +0000100" Always use forward slashes.
101set shellslash
102
Bram Moolenaar89b10422016-07-12 22:51:22 +0200103" Common with all tests on all systems.
Christian Brabandteb380b92025-07-07 20:53:55 +0200104source util/setup.vim
Bram Moolenaar89b10422016-07-12 22:51:22 +0200105
Bram Moolenaar7c2beb42023-07-08 00:25:56 +0100106" Needed for RunningWithValgrind().
Christian Brabandteb380b92025-07-07 20:53:55 +0200107source util/shared.vim
Bram Moolenaar7c2beb42023-07-08 00:25:56 +0100108
Christian Brabandtb0905e22025-07-07 20:39:29 +0200109" Needed for the various Check commands
Christian Brabandteb380b92025-07-07 20:53:55 +0200110source util/check.vim
Christian Brabandtb0905e22025-07-07 20:39:29 +0200111
Bram Moolenaarc0662462015-12-30 15:49:05 +0100112" For consistency run all tests with 'nocompatible' set.
113" This also enables use of line continuation.
114set nocp viminfo+=nviminfo
115
Bram Moolenaar30276f22019-01-24 17:59:39 +0100116" Use utf-8 by default, instead of whatever the system default happens to be.
Bram Moolenaared79d1e2019-02-22 14:38:58 +0100117" Individual tests can overrule this at the top of the file and use
118" g:orig_encoding if needed.
119let g:orig_encoding = &encoding
Bram Moolenaar30276f22019-01-24 17:59:39 +0100120set encoding=utf-8
Bram Moolenaarac105ed2016-07-21 20:33:32 +0200121
Bram Moolenaard8f27b32018-10-07 15:42:07 +0200122" REDIR_TEST_TO_NULL has a very permissive SwapExists autocommand which is for
123" the test_name.vim file itself. Replace it here with a more restrictive one,
124" so we still catch mistakes.
Christian Brabandt4b2c8042021-11-03 22:31:44 +0000125if has("win32")
126 " replace any '/' directory separators by '\\'
127 let s:test_script_fname = substitute(expand('%'), '/', '\\', 'g')
128else
129 let s:test_script_fname = expand('%')
130endif
Bram Moolenaarabc81302023-06-04 16:55:27 +0100131
Bram Moolenaard8f27b32018-10-07 15:42:07 +0200132au! SwapExists * call HandleSwapExists()
133func HandleSwapExists()
Bram Moolenaar8ce4b7e2020-08-07 18:12:18 +0200134 if exists('g:ignoreSwapExists')
Bram Moolenaarabc81302023-06-04 16:55:27 +0100135 if type(g:ignoreSwapExists) == v:t_string
136 let v:swapchoice = g:ignoreSwapExists
137 endif
Bram Moolenaar8ce4b7e2020-08-07 18:12:18 +0200138 return
139 endif
Bram Moolenaarb073da82019-07-13 14:47:26 +0200140 " Ignore finding a swap file for the test script (the user might be
Bram Moolenaard8f27b32018-10-07 15:42:07 +0200141 " editing it and do ":make test_name") and the output file.
Bram Moolenaarb073da82019-07-13 14:47:26 +0200142 " Report finding another swap file and chose 'q' to avoid getting stuck.
Bram Moolenaard8f27b32018-10-07 15:42:07 +0200143 if expand('<afile>') == 'messages' || expand('<afile>') =~ s:test_script_fname
144 let v:swapchoice = 'e'
Bram Moolenaarb073da82019-07-13 14:47:26 +0200145 else
146 call assert_report('Unexpected swap file: ' .. v:swapname)
147 let v:swapchoice = 'q'
Bram Moolenaard8f27b32018-10-07 15:42:07 +0200148 endif
149endfunc
150
Bram Moolenaar7a073542017-02-01 23:17:36 +0100151" Avoid stopping at the "hit enter" prompt
152set nomore
153
Bram Moolenaarc0662462015-12-30 15:49:05 +0100154" Output all messages in English.
155lang mess C
156
Bram Moolenaar10e1d012020-07-18 22:03:11 +0200157" suppress menu translation
158if has('gui_running') && exists('did_install_default_menus')
159 source $VIMRUNTIME/delmenu.vim
160 set langmenu=none
161 source $VIMRUNTIME/menu.vim
162endif
163
Bram Moolenaar28fb79d2016-01-09 22:28:33 +0100164let s:srcdir = expand('%:p:h:h')
165
Bram Moolenaar2690b5a2020-07-22 18:14:58 +0200166if has('win32')
167 " avoid prompt that is long or contains a line break
168 let $PROMPT = '$P$G'
Bram Moolenaar45df2a02020-07-29 15:03:01 +0200169 " On MS-Windows t_md and t_me are Vim specific escape sequences.
170 let s:t_bold = "\x1b[1m"
171 let s:t_normal = "\x1b[m"
172else
173 let s:t_bold = &t_md
174 let s:t_normal = &t_me
Bram Moolenaar2690b5a2020-07-22 18:14:58 +0200175endif
176
Bram Moolenaar4b2ce122020-11-21 21:41:41 +0100177if has('mac')
Dominique Pelle923dce22021-11-21 11:36:04 +0000178 " In macOS, when starting a shell in a terminal, a bash deprecation warning
Bram Moolenaar4b2ce122020-11-21 21:41:41 +0100179 " message is displayed. This breaks the terminal test. Disable the warning
180 " message.
181 let $BASH_SILENCE_DEPRECATION_WARNING = 1
182endif
183
Bram Moolenaarc216a7a2022-12-05 13:50:55 +0000184
Bram Moolenaar8e8df252016-05-25 21:23:21 +0200185" Prepare for calling test_garbagecollect_now().
zeertzjq7a734b72025-07-05 17:02:04 +0200186" Also avoids some delays in Insert mode completion.
Bram Moolenaarebf7dfa2016-04-14 12:46:51 +0200187let v:testing = 1
188
Bram Moolenaarfa4873c2022-06-30 22:13:59 +0100189" By default, copy each buffer line into allocated memory, so that valgrind can
190" detect accessing memory before and after it.
191call test_override('alloc_lines', 1)
192
Bram Moolenaar28fb79d2016-01-09 22:28:33 +0100193" Support function: get the alloc ID by name.
194function GetAllocId(name)
195 exe 'split ' . s:srcdir . '/alloc.h'
Bram Moolenaar065ee9a2016-01-15 20:53:38 +0100196 let top = search('typedef enum')
197 if top == 0
198 call add(v:errors, 'typedef not found in alloc.h')
199 endif
Bram Moolenaar28fb79d2016-01-09 22:28:33 +0100200 let lnum = search('aid_' . a:name . ',')
201 if lnum == 0
202 call add(v:errors, 'Alloc ID ' . a:name . ' not defined')
203 endif
204 close
Bram Moolenaar065ee9a2016-01-15 20:53:38 +0100205 return lnum - top - 1
Bram Moolenaar28fb79d2016-01-09 22:28:33 +0100206endfunc
207
Bram Moolenaarc216a7a2022-12-05 13:50:55 +0000208" Get the list of swap files in the current directory.
209func s:GetSwapFileList()
210 let save_dir = &directory
211 let &directory = '.'
212 let files = swapfilelist()
213 let &directory = save_dir
214
215 " remove a match with runtest.vim
216 let idx = indexof(files, 'v:val =~ "runtest.vim."')
217 if idx >= 0
218 call remove(files, idx)
219 endif
220
221 return files
222endfunc
223
Bram Moolenaar6572a902022-12-06 14:21:09 +0000224" A previous (failed) test run may have left swap files behind. Delete them
225" before running tests again, they might interfere.
226for name in s:GetSwapFileList()
227 call delete(name)
228endfor
zeertzjqa36acb72023-10-21 11:50:26 +0200229unlet! name
Bram Moolenaar6572a902022-12-06 14:21:09 +0000230
231
Bram Moolenaar3bcd0dd2022-09-23 20:25:55 +0100232" Invoked when a test takes too much time.
233func TestTimeout(id)
234 split test.log
235 call append(line('$'), '')
Bram Moolenaar7c2beb42023-07-08 00:25:56 +0100236
237 let text = 'Test timed out: ' .. g:testfunc
238 if g:timeout_start > 0
239 let text ..= strftime(' after %s seconds', localtime() - g:timeout_start)
240 endif
241 call append(line('$'), text)
Bram Moolenaar3bcd0dd2022-09-23 20:25:55 +0100242 write
Bram Moolenaar7c2beb42023-07-08 00:25:56 +0100243 call add(v:errors, text)
Bram Moolenaar3bcd0dd2022-09-23 20:25:55 +0100244
245 cquit! 42
246endfunc
Bram Moolenaar7c2beb42023-07-08 00:25:56 +0100247let g:timeout_start = 0
Bram Moolenaar3bcd0dd2022-09-23 20:25:55 +0100248
Bram Moolenaar42205552017-03-18 19:42:22 +0100249func RunTheTest(test)
Bram Moolenaardaaa3d92022-09-22 15:13:00 +0100250 let prefix = ''
Bram Moolenaar75ee5442019-06-06 18:05:25 +0200251 if has('reltime')
Bram Moolenaarb9093d52022-09-23 19:42:31 +0100252 let prefix = strftime('%M:%S', localtime() - s:test_start_time) .. ' '
Bram Moolenaardaaa3d92022-09-22 15:13:00 +0100253 let g:func_start = reltime()
Bram Moolenaar75ee5442019-06-06 18:05:25 +0200254 endif
Bram Moolenaardaaa3d92022-09-22 15:13:00 +0100255 echoconsole prefix .. 'Executing ' .. a:test
Bram Moolenaare5f2a072017-02-01 22:31:49 +0100256
Bram Moolenaar3bcd0dd2022-09-23 20:25:55 +0100257 if has('timers')
Christian Brabandtec7a4e42025-02-09 17:16:36 +0100258 " No test should take longer than 45 seconds. If it takes longer we
Bram Moolenaar3bcd0dd2022-09-23 20:25:55 +0100259 " assume we are stuck and need to break out.
Bram Moolenaar7c2beb42023-07-08 00:25:56 +0100260 let test_timeout_timer =
Christian Brabandtec7a4e42025-02-09 17:16:36 +0100261 \ timer_start(RunningWithValgrind() ? 90000 : 45000, 'TestTimeout')
Bram Moolenaar7c2beb42023-07-08 00:25:56 +0100262 let g:timeout_start = localtime()
Bram Moolenaar3bcd0dd2022-09-23 20:25:55 +0100263 endif
264
Bram Moolenaare5f2a072017-02-01 22:31:49 +0100265 " Avoid stopping at the "hit enter" prompt
266 set nomore
267
268 " Avoid a three second wait when a message is about to be overwritten by the
269 " mode message.
270 set noshowmode
271
Bram Moolenaarfa4873c2022-06-30 22:13:59 +0100272 " Clear any overrides, except "alloc_lines".
Bram Moolenaareb992cb2017-03-09 18:20:16 +0100273 call test_override('ALL', 0)
274
Bram Moolenaarcf1ba352017-10-27 00:55:04 +0200275 " Some tests wipe out buffers. To be consistent, always wipe out all
276 " buffers.
277 %bwipe!
278
Yee Cheng Chine70587d2025-02-13 20:55:45 +0100279 " Clear all children notifications in case there are stale ones left
280 let g:child_notification = 0
281
Bram Moolenaar209d3872017-11-16 21:52:51 +0100282 " The test may change the current directory. Save and restore the
283 " directory after executing the test.
284 let save_cwd = getcwd()
285
Aliaksei Budaveic4eb1cb2025-06-08 15:52:42 +0200286 " Permit "SetUp()" implementations to override default settings.
287 call s:SetDefaultOptionsForGUIBuilds()
288
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100289 if exists("*SetUp")
Bram Moolenaarcc28e2d2016-11-17 17:56:13 +0100290 try
291 call SetUp()
292 catch
293 call add(v:errors, 'Caught exception in SetUp() before ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint)
294 endtry
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100295 endif
296
zeertzjqa0e1f062023-10-18 11:50:37 +0200297 let skipped = v:false
298
Bram Moolenaar8bea1712022-06-15 20:49:35 +0100299 au VimLeavePre * call EarlyExit(g:testfunc)
Bram Moolenaarf204e052017-10-26 17:14:01 +0200300 if a:test =~ 'Test_nocatch_'
301 " Function handles errors itself. This avoids skipping commands after the
302 " error.
Bram Moolenaar776b9542021-03-10 22:27:48 +0100303 let g:skipped_reason = ''
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100304 exe 'call ' . a:test
Bram Moolenaar776b9542021-03-10 22:27:48 +0100305 if g:skipped_reason != ''
306 call add(s:messages, ' Skipped')
307 call add(s:skipped, 'SKIPPED ' . a:test . ': ' . g:skipped_reason)
zeertzjqa0e1f062023-10-18 11:50:37 +0200308 let skipped = v:true
Bram Moolenaar776b9542021-03-10 22:27:48 +0100309 endif
Bram Moolenaarf204e052017-10-26 17:14:01 +0200310 else
311 try
312 exe 'call ' . a:test
313 catch /^\cskipped/
314 call add(s:messages, ' Skipped')
315 call add(s:skipped, 'SKIPPED ' . a:test . ': ' . substitute(v:exception, '^\S*\s\+', '', ''))
zeertzjqa0e1f062023-10-18 11:50:37 +0200316 let skipped = v:true
Bram Moolenaarf204e052017-10-26 17:14:01 +0200317 catch
318 call add(v:errors, 'Caught exception in ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint)
319 endtry
320 endif
Bram Moolenaar8bea1712022-06-15 20:49:35 +0100321 au! VimLeavePre
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100322
Bram Moolenaara22c56a2022-09-20 15:10:31 +0100323 if a:test =~ '_terminal_'
324 " Terminal tests sometimes hang, give extra information
325 echoconsole 'After executing ' .. a:test
326 endif
327
Bram Moolenaar8ad16da2019-01-06 15:29:57 +0100328 " In case 'insertmode' was set and something went wrong, make sure it is
329 " reset to avoid trouble with anything else.
330 set noinsertmode
331
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100332 if exists("*TearDown")
Bram Moolenaarcc28e2d2016-11-17 17:56:13 +0100333 try
334 call TearDown()
335 catch
336 call add(v:errors, 'Caught exception in TearDown() after ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint)
337 endtry
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100338 endif
Bram Moolenaar7cba71d2016-08-02 23:04:49 +0200339
Bram Moolenaar3bcd0dd2022-09-23 20:25:55 +0100340 if has('timers')
341 call timer_stop(test_timeout_timer)
Bram Moolenaar7c2beb42023-07-08 00:25:56 +0100342 let g:timeout_start = 0
Bram Moolenaar3bcd0dd2022-09-23 20:25:55 +0100343 endif
344
Bram Moolenaar0b5dc642019-08-11 22:56:15 +0200345 " Clear any autocommands and put back the catch-all for SwapExists.
Bram Moolenaarcf1ba352017-10-27 00:55:04 +0200346 au!
Bram Moolenaard8f27b32018-10-07 15:42:07 +0200347 au SwapExists * call HandleSwapExists()
Bram Moolenaarcf1ba352017-10-27 00:55:04 +0200348
Bram Moolenaaref6b9792020-05-13 16:34:15 +0200349 " Check for and close any stray popup windows.
Bram Moolenaar05ad5ff2019-11-30 22:48:27 +0100350 if has('popupwin')
Bram Moolenaarf05a1e52022-08-02 11:48:53 +0100351 call assert_equal([], popup_list(), 'Popup is still present')
Bram Moolenaar03a9f842020-05-13 13:40:16 +0200352 call popup_clear(1)
Bram Moolenaarae943152019-06-16 22:54:14 +0200353 endif
354
Bram Moolenaar2d12c252022-06-13 21:42:45 +0100355 if filereadable('guidialogfile')
Bram Moolenaar217ea512022-06-14 17:13:59 +0100356 call add(v:errors, "Unexpected dialog: " .. readfile('guidialogfile')->join('<NL>'))
Bram Moolenaar2d12c252022-06-13 21:42:45 +0100357 call delete('guidialogfile')
358 endif
359
Bram Moolenaarce11de82017-10-26 22:00:00 +0200360 " Close any extra tab pages and windows and make the current one not modified.
361 while tabpagenr('$') > 1
Bram Moolenaarbdf931c2020-10-01 22:37:40 +0200362 let winid = win_getid()
Bram Moolenaarcf1ba352017-10-27 00:55:04 +0200363 quit!
Bram Moolenaarbdf931c2020-10-01 22:37:40 +0200364 if winid == win_getid()
365 echoerr 'Could not quit window'
366 break
367 endif
Bram Moolenaarce11de82017-10-26 22:00:00 +0200368 endwhile
369
Bram Moolenaar358308d2016-08-24 21:21:26 +0200370 while 1
371 let wincount = winnr('$')
372 if wincount == 1
373 break
374 endif
Bram Moolenaar7cba71d2016-08-02 23:04:49 +0200375 bwipe!
Bram Moolenaar358308d2016-08-24 21:21:26 +0200376 if wincount == winnr('$')
377 " Did not manage to close a window.
378 only!
379 break
380 endif
Bram Moolenaar7cba71d2016-08-02 23:04:49 +0200381 endwhile
Bram Moolenaar209d3872017-11-16 21:52:51 +0100382
383 exe 'cd ' . save_cwd
Bram Moolenaar640d4f02019-06-10 17:43:46 +0200384
Bram Moolenaara22c56a2022-09-20 15:10:31 +0100385 if a:test =~ '_terminal_'
386 " Terminal tests sometimes hang, give extra information
387 echoconsole 'Finished ' . a:test
388 endif
389
Bram Moolenaar640d4f02019-06-10 17:43:46 +0200390 let message = 'Executed ' . a:test
391 if has('reltime')
Bram Moolenaar8d943792020-06-21 20:39:37 +0200392 let message ..= repeat(' ', 50 - len(message))
Bram Moolenaardaaa3d92022-09-22 15:13:00 +0100393 let time = reltime(g:func_start)
Bram Moolenaar73e28dc2022-09-17 21:08:33 +0100394 if reltimefloat(time) > 0.1
Bram Moolenaar45df2a02020-07-29 15:03:01 +0200395 let message = s:t_bold .. message
Bram Moolenaar8d943792020-06-21 20:39:37 +0200396 endif
397 let message ..= ' in ' .. reltimestr(time) .. ' seconds'
Bram Moolenaar73e28dc2022-09-17 21:08:33 +0100398 if reltimefloat(time) > 0.1
Bram Moolenaar45df2a02020-07-29 15:03:01 +0200399 let message ..= s:t_normal
Bram Moolenaar8d943792020-06-21 20:39:37 +0200400 endif
Bram Moolenaar640d4f02019-06-10 17:43:46 +0200401 endif
402 call add(s:messages, message)
403 let s:done += 1
Bram Moolenaarc216a7a2022-12-05 13:50:55 +0000404
Bram Moolenaare5eae822022-12-08 16:30:16 +0000405 " close any split windows
406 while winnr('$') > 1
Bram Moolenaar7c2beb42023-07-08 00:25:56 +0100407 noswapfile bwipe!
Bram Moolenaare5eae822022-12-08 16:30:16 +0000408 endwhile
409
Bram Moolenaar23526d22022-12-05 15:50:41 +0000410 " May be editing some buffer, wipe it out. Then we may end up in another
411 " buffer, continue until we end up in an empty no-name buffer without a swap
412 " file.
413 while bufname() != '' || execute('swapname') !~ 'No swap file'
Bram Moolenaarfa2533c2022-12-05 20:58:04 +0000414 let bn = bufnr()
415
416 noswapfile bwipe!
417
418 if bn == bufnr()
419 " avoid getting stuck in the same buffer
420 break
421 endif
Bram Moolenaar23526d22022-12-05 15:50:41 +0000422 endwhile
423
zeertzjqa0e1f062023-10-18 11:50:37 +0200424 if !skipped
425 " Check if the test has left any swap files behind. Delete them before
426 " running tests again, they might interfere.
427 let swapfiles = s:GetSwapFileList()
428 if len(swapfiles) > 0
429 call add(s:messages, "Found swap files: " .. string(swapfiles))
430 for name in swapfiles
431 call delete(name)
432 endfor
433 endif
Bram Moolenaarc216a7a2022-12-05 13:50:55 +0000434 endif
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100435endfunc
Bram Moolenaar28fb79d2016-01-09 22:28:33 +0100436
Christian Brabandt84bc00e2023-07-13 11:45:54 +0200437function Delete_Xtest_Files()
438 for file in glob('X*', v:false, v:true)
439 if file ==? 'XfakeHOME'
440 " Clean up files created by setup.vim
441 call delete('XfakeHOME', 'rf')
442 continue
443 endif
444 " call add(v:errors, file .. " exists when it shouldn't, trying to delete it!")
445 call delete(file)
446 if !empty(glob(file, v:false, v:true))
447 " call add(v:errors, file .. " still exists after trying to delete it!")
448 if has('unix')
449 call system('rm -rf ' .. file)
450 endif
451 endif
452 endfor
453endfunc
454
Bram Moolenaarce436de2020-03-21 15:17:20 +0100455func AfterTheTest(func_name)
Bram Moolenaar42205552017-03-18 19:42:22 +0100456 if len(v:errors) > 0
Bram Moolenaarce436de2020-03-21 15:17:20 +0100457 if match(s:may_fail_list, '^' .. a:func_name) >= 0
458 let s:fail_expected += 1
Bram Moolenaarbfe13cc2020-04-12 17:53:12 +0200459 call add(s:errors_expected, 'Found errors in ' . g:testfunc . ':')
Bram Moolenaarce436de2020-03-21 15:17:20 +0100460 call extend(s:errors_expected, v:errors)
461 else
462 let s:fail += 1
Bram Moolenaarbfe13cc2020-04-12 17:53:12 +0200463 call add(s:errors, 'Found errors in ' . g:testfunc . ':')
Bram Moolenaarce436de2020-03-21 15:17:20 +0100464 call extend(s:errors, v:errors)
465 endif
Bram Moolenaar42205552017-03-18 19:42:22 +0100466 let v:errors = []
467 endif
468endfunc
469
Bram Moolenaar89036762018-06-12 14:58:39 +0200470func EarlyExit(test)
471 " It's OK for the test we use to test the quit detection.
472 if a:test != 'Test_zz_quit_detected()'
Bram Moolenaar1c67f3a2021-12-30 13:32:09 +0000473 call add(v:errors, v:errmsg)
Bram Moolenaar89036762018-06-12 14:58:39 +0200474 call add(v:errors, 'Test caused Vim to exit: ' . a:test)
475 endif
476
477 call FinishTesting()
478endfunc
479
Bram Moolenaar42205552017-03-18 19:42:22 +0100480" This function can be called by a test if it wants to abort testing.
481func FinishTesting()
Bram Moolenaarce436de2020-03-21 15:17:20 +0100482 call AfterTheTest('')
Christian Brabandt84bc00e2023-07-13 11:45:54 +0200483 call Delete_Xtest_Files()
Bram Moolenaar42205552017-03-18 19:42:22 +0100484
485 " Don't write viminfo on exit.
486 set viminfo=
487
Bram Moolenaarce436de2020-03-21 15:17:20 +0100488 if s:fail == 0 && s:fail_expected == 0
Bram Moolenaar42205552017-03-18 19:42:22 +0100489 " Success, create the .res file so that make knows it's done.
490 exe 'split ' . fnamemodify(g:testname, ':r') . '.res'
491 write
492 endif
493
494 if len(s:errors) > 0
495 " Append errors to test.log
496 split test.log
497 call append(line('$'), '')
498 call append(line('$'), 'From ' . g:testname . ':')
499 call append(line('$'), s:errors)
500 write
501 endif
502
Bram Moolenaar29f9ed22018-04-10 19:20:31 +0200503 if s:done == 0
Bram Moolenaar7b666c72019-09-27 21:25:00 +0200504 if s:filtered > 0
Bram Moolenaardae453f2021-08-07 17:20:16 +0200505 if $TEST_FILTER != ''
506 let message = "NO tests match $TEST_FILTER: '" .. $TEST_FILTER .. "'"
507 else
508 let message = "ALL tests match $TEST_SKIP_PAT: '" .. $TEST_SKIP_PAT .. "'"
509 endif
Bram Moolenaar7b666c72019-09-27 21:25:00 +0200510 else
511 let message = 'NO tests executed'
512 endif
Bram Moolenaar29f9ed22018-04-10 19:20:31 +0200513 else
Bram Moolenaar7b666c72019-09-27 21:25:00 +0200514 if s:filtered > 0
Bram Moolenaardae453f2021-08-07 17:20:16 +0200515 call add(s:messages, "Filtered " .. s:filtered .. " tests with $TEST_FILTER and $TEST_SKIP_PAT")
Bram Moolenaar7b666c72019-09-27 21:25:00 +0200516 endif
Bram Moolenaar29f9ed22018-04-10 19:20:31 +0200517 let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test')
518 endif
Bram Moolenaar7b666c72019-09-27 21:25:00 +0200519 if s:done > 0 && has('reltime')
Bram Moolenaar45df2a02020-07-29 15:03:01 +0200520 let message = s:t_bold .. message .. repeat(' ', 40 - len(message))
Bram Moolenaarb9093d52022-09-23 19:42:31 +0100521 let message ..= ' in ' .. reltimestr(reltime(s:run_start_time)) .. ' seconds'
Bram Moolenaar45df2a02020-07-29 15:03:01 +0200522 let message ..= s:t_normal
Bram Moolenaar75ee5442019-06-06 18:05:25 +0200523 endif
Bram Moolenaar42205552017-03-18 19:42:22 +0100524 echo message
525 call add(s:messages, message)
526 if s:fail > 0
527 let message = s:fail . ' FAILED:'
528 echo message
529 call add(s:messages, message)
530 call extend(s:messages, s:errors)
531 endif
Bram Moolenaarce436de2020-03-21 15:17:20 +0100532 if s:fail_expected > 0
533 let message = s:fail_expected . ' FAILED (matching $TEST_MAY_FAIL):'
534 echo message
535 call add(s:messages, message)
536 call extend(s:messages, s:errors_expected)
537 endif
Bram Moolenaar42205552017-03-18 19:42:22 +0100538
539 " Add SKIPPED messages
540 call extend(s:messages, s:skipped)
541
Christian Brabandt075ab5a2024-10-03 16:38:52 +0200542 " Append messages to the file "messages", but remove ANSI Escape sequences
Bram Moolenaar42205552017-03-18 19:42:22 +0100543 split messages
544 call append(line('$'), '')
545 call append(line('$'), 'From ' . g:testname . ':')
Christian Brabandtbcb06c92025-07-07 20:03:03 +0200546 call append(line('$'), s:messages->map({_, val -> substitute(val, '\%x1b[[|]\(\d\?\|\d\+\)[hm]', '', 'g')}))
Bram Moolenaar42205552017-03-18 19:42:22 +0100547 write
548
549 qall!
550endfunc
551
Bram Moolenaar43345542015-11-29 17:35:35 +0100552" Source the test script. First grab the file name, in case the script
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100553" navigates away. g:testname can be used by the tests.
554let g:testname = expand('%')
555let s:done = 0
556let s:fail = 0
Bram Moolenaarce436de2020-03-21 15:17:20 +0100557let s:fail_expected = 0
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100558let s:errors = []
Bram Moolenaarce436de2020-03-21 15:17:20 +0100559let s:errors_expected = []
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100560let s:messages = []
Bram Moolenaardac19472016-09-03 22:35:40 +0200561let s:skipped = []
Bram Moolenaarb544f3c2017-02-23 19:03:28 +0100562if expand('%') =~ 'test_vimscript.vim'
Bram Moolenaarce436de2020-03-21 15:17:20 +0100563 " this test has intentional errors, don't use try/catch.
Bram Moolenaar4686b322015-12-28 14:44:10 +0100564 source %
Bram Moolenaara2cce862016-01-02 19:50:04 +0100565else
566 try
567 source %
Bram Moolenaar9c0cec62019-06-06 13:38:15 +0200568 catch /^\cskipped/
569 call add(s:messages, ' Skipped')
570 call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '', ''))
Bram Moolenaara2cce862016-01-02 19:50:04 +0100571 catch
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100572 let s:fail += 1
573 call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint)
Bram Moolenaara2cce862016-01-02 19:50:04 +0100574 endtry
575endif
Bram Moolenaar43345542015-11-29 17:35:35 +0100576
Bram Moolenaar7e0be3e2022-03-20 13:40:41 +0000577" Delete the .res file, it may change behavior for completion
578call delete(fnamemodify(g:testname, ':r') .. '.res')
579
Bram Moolenaar43345542015-11-29 17:35:35 +0100580" Locate Test_ functions and execute them.
581redir @q
Bram Moolenaar93bf5582016-02-18 22:25:47 +0100582silent function /^Test_
Bram Moolenaar43345542015-11-29 17:35:35 +0100583redir END
Bram Moolenaar61a6d4e2020-03-01 23:32:25 +0100584let s:tests = split(substitute(@q, '\(function\|def\) \(\k*()\)', '\2', 'g'))
Bram Moolenaar43345542015-11-29 17:35:35 +0100585
Bram Moolenaarbefb3662016-02-20 14:41:40 +0100586" If there is an extra argument filter the function names against it.
587if argc() > 1
588 let s:tests = filter(s:tests, 'v:val =~ argv(1)')
589endif
590
Bram Moolenaara7f6c3c2019-09-27 15:34:16 +0200591" If the environment variable $TEST_FILTER is set then filter the function
592" names against it.
Bram Moolenaar7b666c72019-09-27 21:25:00 +0200593let s:filtered = 0
Bram Moolenaara7f6c3c2019-09-27 15:34:16 +0200594if $TEST_FILTER != ''
Bram Moolenaar7b666c72019-09-27 21:25:00 +0200595 let s:filtered = len(s:tests)
Bram Moolenaara7f6c3c2019-09-27 15:34:16 +0200596 let s:tests = filter(s:tests, 'v:val =~ $TEST_FILTER')
Bram Moolenaar7b666c72019-09-27 21:25:00 +0200597 let s:filtered -= len(s:tests)
Bram Moolenaara7f6c3c2019-09-27 15:34:16 +0200598endif
599
Bram Moolenaarce436de2020-03-21 15:17:20 +0100600let s:may_fail_list = []
601if $TEST_MAY_FAIL != ''
Bram Moolenaarbfe13cc2020-04-12 17:53:12 +0200602 " Split the list at commas and add () to make it match g:testfunc.
Bram Moolenaarce436de2020-03-21 15:17:20 +0100603 let s:may_fail_list = split($TEST_MAY_FAIL, ',')->map({i, v -> v .. '()'})
604endif
605
Bram Moolenaarcfc0a352016-01-09 20:23:00 +0100606" Execute the tests in alphabetical order.
Bram Moolenaarbfe13cc2020-04-12 17:53:12 +0200607for g:testfunc in sort(s:tests)
Bram Moolenaardae453f2021-08-07 17:20:16 +0200608 if $TEST_SKIP_PAT != '' && g:testfunc =~ $TEST_SKIP_PAT
609 call add(s:messages, g:testfunc .. ' matches $TEST_SKIP_PAT')
610 let s:filtered += 1
611 continue
612 endif
613
Bram Moolenaar4a6fcf82017-10-12 21:29:22 +0200614 " Silence, please!
615 set belloff=all
Bram Moolenaarf77af0e2018-11-16 16:52:16 +0100616 let prev_error = ''
617 let total_errors = []
Bram Moolenaar3ed9efc2020-03-26 16:50:57 +0100618 let g:run_nr = 1
Bram Moolenaar4a6fcf82017-10-12 21:29:22 +0200619
Bram Moolenaarbfe13cc2020-04-12 17:53:12 +0200620 " A test can set g:test_is_flaky to retry running the test.
621 let g:test_is_flaky = 0
Bram Moolenaar3cdcb092020-03-18 19:18:10 +0100622
Drew Vogelea67ba72025-05-07 22:05:17 +0200623 let g:check_screendump_called = v:false
624
Millybaab7c02024-10-28 21:56:14 +0100625 " A test can set g:max_run_nr to change the max retry count.
626 let g:max_run_nr = 5
627 if has('mac')
628 let g:max_run_nr = 10
629 endif
630
631 " By default, give up if the same error occurs. A test can set
632 " g:giveup_same_error to 0 to not give up on the same error and keep trying.
633 let g:giveup_same_error = 1
634
Bram Moolenaar5fbbec12022-09-03 22:08:11 +0100635 let starttime = strftime("%H:%M:%S")
Bram Moolenaarbfe13cc2020-04-12 17:53:12 +0200636 call RunTheTest(g:testfunc)
Bram Moolenaar43345542015-11-29 17:35:35 +0100637
Bram Moolenaarf77af0e2018-11-16 16:52:16 +0100638 " Repeat a flaky test. Give up when:
Bram Moolenaar6ca6ca42020-07-27 19:47:07 +0200639 " - $TEST_NO_RETRY is not empty
Bram Moolenaarf77af0e2018-11-16 16:52:16 +0100640 " - it fails again with the same message
Bram Moolenaar1bc353b2019-09-01 14:45:28 +0200641 " - it fails five times (with a different message)
Bram Moolenaardbc0d212018-11-16 18:22:45 +0100642 if len(v:errors) > 0
Bram Moolenaar622b3562020-07-27 20:02:41 +0200643 \ && $TEST_NO_RETRY == ''
Bram Moolenaarf08b0eb2021-10-16 13:00:14 +0100644 \ && g:test_is_flaky
Bram Moolenaarf77af0e2018-11-16 16:52:16 +0100645 while 1
Bram Moolenaar06d32a02022-09-03 13:58:47 +0100646 call add(s:messages, 'Found errors in ' .. g:testfunc .. ':')
Bram Moolenaarf77af0e2018-11-16 16:52:16 +0100647 call extend(s:messages, v:errors)
Bram Moolenaar15e737f2017-03-18 21:22:47 +0100648
Bram Moolenaar65258d32022-09-09 15:09:59 +0100649 let endtime = strftime("%H:%M:%S")
Millybaab7c02024-10-28 21:56:14 +0100650 if has('reltime')
651 let suffix = $' in{reltimestr(reltime(g:func_start))} seconds'
652 else
653 let suffix = ''
654 endif
655 call add(total_errors, $'Run {g:run_nr}, {starttime} - {endtime}{suffix}:')
Bram Moolenaarf77af0e2018-11-16 16:52:16 +0100656 call extend(total_errors, v:errors)
Bram Moolenaar55058602017-11-21 15:14:51 +0100657
Millybaab7c02024-10-28 21:56:14 +0100658 if g:run_nr >= g:max_run_nr || g:giveup_same_error && prev_error == v:errors[0]
Bram Moolenaarf77af0e2018-11-16 16:52:16 +0100659 call add(total_errors, 'Flaky test failed too often, giving up')
660 let v:errors = total_errors
661 break
662 endif
663
664 call add(s:messages, 'Flaky test failed, running it again')
665
666 " Flakiness is often caused by the system being very busy. Sleep a
667 " couple of seconds to have a higher chance of succeeding the second
668 " time.
Millybaab7c02024-10-28 21:56:14 +0100669 let delay = g:run_nr * 2
670 exe 'sleep' delay
Bram Moolenaarf77af0e2018-11-16 16:52:16 +0100671
672 let prev_error = v:errors[0]
673 let v:errors = []
Bram Moolenaar3ed9efc2020-03-26 16:50:57 +0100674 let g:run_nr += 1
Bram Moolenaarf77af0e2018-11-16 16:52:16 +0100675
Bram Moolenaar5fbbec12022-09-03 22:08:11 +0100676 let starttime = strftime("%H:%M:%S")
Bram Moolenaarbfe13cc2020-04-12 17:53:12 +0200677 call RunTheTest(g:testfunc)
Bram Moolenaarf77af0e2018-11-16 16:52:16 +0100678
679 if len(v:errors) == 0
680 " Test passed on rerun.
681 break
682 endif
683 endwhile
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100684 endif
Bram Moolenaar43345542015-11-29 17:35:35 +0100685
Bram Moolenaarbfe13cc2020-04-12 17:53:12 +0200686 call AfterTheTest(g:testfunc)
Bram Moolenaar43345542015-11-29 17:35:35 +0100687endfor
688
Bram Moolenaar42205552017-03-18 19:42:22 +0100689call FinishTesting()
Bram Moolenaarcc28e2d2016-11-17 17:56:13 +0100690
691" vim: shiftwidth=2 sts=2 expandtab