blob: c4cb8473245d9221d72588abea08643e302aca2a [file] [log] [blame]
Bram Moolenaar43345542015-11-29 17:35:35 +01001" This script is sourced while editing the .vim file with the tests.
2" When the script is successful the .res file will be created.
3" Errors are appended to the test.log file.
4"
Bram Moolenaarbefb3662016-02-20 14:41:40 +01005" To execute only specific test functions, add a second argument. It will be
6" matched against the names of the Test_ funtion. E.g.:
7" ../vim -u NONE -S runtest.vim test_channel.vim open_delay
8" The output can be found in the "messages" file.
9"
Bram Moolenaar43345542015-11-29 17:35:35 +010010" The test script may contain anything, only functions that start with
11" "Test_" are special. These will be invoked and should contain assert
12" functions. See test_assert.vim for an example.
13"
14" It is possible to source other files that contain "Test_" functions. This
15" can speed up testing, since Vim does not need to restart. But be careful
16" that the tests do not interfere with each other.
17"
18" If an error cannot be detected properly with an assert function add the
19" error to the v:errors list:
20" call add(v:errors, 'test foo failed: Cannot find xyz')
21"
22" If preparation for each Test_ function is needed, define a SetUp function.
23" It will be called before each Test_ function.
24"
25" If cleanup after each Test_ function is needed, define a TearDown function.
26" It will be called after each Test_ function.
Bram Moolenaar00af60b2016-02-13 14:06:14 +010027"
28" When debugging a test it can be useful to add messages to v:errors:
29" call add(v:errors, "this happened")
30
Bram Moolenaar43345542015-11-29 17:35:35 +010031
32" Without the +eval feature we can't run these tests, bail out.
Bram Moolenaar4686b322015-12-28 14:44:10 +010033so small.vim
Bram Moolenaar43345542015-11-29 17:35:35 +010034
35" Check that the screen size is at least 24 x 80 characters.
36if &lines < 24 || &columns < 80
37 let error = 'Screen size too small! Tests require at least 24 lines with 80 characters'
38 echoerr error
39 split test.log
40 $put =error
41 w
42 cquit
43endif
44
Bram Moolenaar89b10422016-07-12 22:51:22 +020045" Common with all tests on all systems.
46source setup.vim
47
Bram Moolenaarc0662462015-12-30 15:49:05 +010048" For consistency run all tests with 'nocompatible' set.
49" This also enables use of line continuation.
50set nocp viminfo+=nviminfo
51
Bram Moolenaarac105ed2016-07-21 20:33:32 +020052" Use utf-8 or latin1 be default, instead of whatever the system default
53" happens to be. Individual tests can overrule this at the top of the file.
54if has('multi_byte')
55 set encoding=utf-8
56else
57 set encoding=latin1
58endif
59
Bram Moolenaarc0662462015-12-30 15:49:05 +010060" Avoid stopping at the "hit enter" prompt
61set nomore
62
63" Output all messages in English.
64lang mess C
65
Bram Moolenaarf60b7962016-01-16 22:47:23 +010066" Always use forward slashes.
67set shellslash
68
Bram Moolenaar28fb79d2016-01-09 22:28:33 +010069let s:srcdir = expand('%:p:h:h')
70
Bram Moolenaar8e8df252016-05-25 21:23:21 +020071" Prepare for calling test_garbagecollect_now().
Bram Moolenaarebf7dfa2016-04-14 12:46:51 +020072let v:testing = 1
73
Bram Moolenaar28fb79d2016-01-09 22:28:33 +010074" Support function: get the alloc ID by name.
75function GetAllocId(name)
76 exe 'split ' . s:srcdir . '/alloc.h'
Bram Moolenaar065ee9a2016-01-15 20:53:38 +010077 let top = search('typedef enum')
78 if top == 0
79 call add(v:errors, 'typedef not found in alloc.h')
80 endif
Bram Moolenaar28fb79d2016-01-09 22:28:33 +010081 let lnum = search('aid_' . a:name . ',')
82 if lnum == 0
83 call add(v:errors, 'Alloc ID ' . a:name . ' not defined')
84 endif
85 close
Bram Moolenaar065ee9a2016-01-15 20:53:38 +010086 return lnum - top - 1
Bram Moolenaar28fb79d2016-01-09 22:28:33 +010087endfunc
88
Bram Moolenaarb5760a12016-03-03 13:10:44 +010089function RunTheTest(test)
90 echo 'Executing ' . a:test
91 if exists("*SetUp")
92 call SetUp()
93 endif
94
95 call add(s:messages, 'Executing ' . a:test)
96 let s:done += 1
97 try
98 exe 'call ' . a:test
Bram Moolenaardac19472016-09-03 22:35:40 +020099 catch /^\cskipped/
100 call add(s:messages, ' Skipped')
101 call add(s:skipped, 'SKIPPED ' . a:test . ': ' . substitute(v:exception, '^\S*\s\+', '', ''))
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100102 catch
103 call add(v:errors, 'Caught exception in ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint)
104 endtry
105
106 if exists("*TearDown")
107 call TearDown()
108 endif
Bram Moolenaar7cba71d2016-08-02 23:04:49 +0200109
110 " Close any extra windows and make the current one not modified.
Bram Moolenaar358308d2016-08-24 21:21:26 +0200111 while 1
112 let wincount = winnr('$')
113 if wincount == 1
114 break
115 endif
Bram Moolenaar7cba71d2016-08-02 23:04:49 +0200116 bwipe!
Bram Moolenaar358308d2016-08-24 21:21:26 +0200117 if wincount == winnr('$')
118 " Did not manage to close a window.
119 only!
120 break
121 endif
Bram Moolenaar7cba71d2016-08-02 23:04:49 +0200122 endwhile
123 set nomodified
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100124endfunc
Bram Moolenaar28fb79d2016-01-09 22:28:33 +0100125
Bram Moolenaar43345542015-11-29 17:35:35 +0100126" Source the test script. First grab the file name, in case the script
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100127" navigates away. g:testname can be used by the tests.
128let g:testname = expand('%')
129let s:done = 0
130let s:fail = 0
131let s:errors = []
132let s:messages = []
Bram Moolenaardac19472016-09-03 22:35:40 +0200133let s:skipped = []
Bram Moolenaara2cce862016-01-02 19:50:04 +0100134if expand('%') =~ 'test_viml.vim'
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100135 " this test has intentional s:errors, don't use try/catch.
Bram Moolenaar4686b322015-12-28 14:44:10 +0100136 source %
Bram Moolenaara2cce862016-01-02 19:50:04 +0100137else
138 try
139 source %
140 catch
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100141 let s:fail += 1
142 call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint)
Bram Moolenaara2cce862016-01-02 19:50:04 +0100143 endtry
144endif
Bram Moolenaar43345542015-11-29 17:35:35 +0100145
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100146" Names of flaky tests.
Bram Moolenaare1c8c7a2016-09-11 16:48:50 +0200147let s:flaky = [
148 \ 'Test_reltime()',
149 \ 'Test_nb_basic()',
Bram Moolenaaredeb8462016-09-12 13:32:02 +0200150 \ 'Test_communicate()',
Bram Moolenaarc79d6aa2016-09-25 22:27:37 +0200151 \ 'Test_pipe_through_sort_all()',
Bram Moolenaare1c8c7a2016-09-11 16:48:50 +0200152 \ 'Test_pipe_through_sort_some()'
153 \ ]
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100154
Bram Moolenaar43345542015-11-29 17:35:35 +0100155" Locate Test_ functions and execute them.
Bram Moolenaara99b9042016-01-17 17:10:59 +0100156set nomore
Bram Moolenaar43345542015-11-29 17:35:35 +0100157redir @q
Bram Moolenaar93bf5582016-02-18 22:25:47 +0100158silent function /^Test_
Bram Moolenaar43345542015-11-29 17:35:35 +0100159redir END
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100160let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g'))
Bram Moolenaar43345542015-11-29 17:35:35 +0100161
Bram Moolenaarbefb3662016-02-20 14:41:40 +0100162" If there is an extra argument filter the function names against it.
163if argc() > 1
164 let s:tests = filter(s:tests, 'v:val =~ argv(1)')
165endif
166
Bram Moolenaarcfc0a352016-01-09 20:23:00 +0100167" Execute the tests in alphabetical order.
Bram Moolenaar93bf5582016-02-18 22:25:47 +0100168for s:test in sort(s:tests)
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100169 call RunTheTest(s:test)
Bram Moolenaar43345542015-11-29 17:35:35 +0100170
Bram Moolenaarb5760a12016-03-03 13:10:44 +0100171 if len(v:errors) > 0 && index(s:flaky, s:test) >= 0
172 call add(s:messages, 'Flaky test failed, running it again')
173 let v:errors = []
174 call RunTheTest(s:test)
175 endif
Bram Moolenaar43345542015-11-29 17:35:35 +0100176
177 if len(v:errors) > 0
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100178 let s:fail += 1
179 call add(s:errors, 'Found errors in ' . s:test . ':')
180 call extend(s:errors, v:errors)
Bram Moolenaar43345542015-11-29 17:35:35 +0100181 let v:errors = []
182 endif
Bram Moolenaar43345542015-11-29 17:35:35 +0100183endfor
184
Bram Moolenaarfc4ad612016-07-09 15:38:32 +0200185" Don't write viminfo on exit.
186set viminfo=
187
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100188if s:fail == 0
Bram Moolenaar43345542015-11-29 17:35:35 +0100189 " Success, create the .res file so that make knows it's done.
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100190 exe 'split ' . fnamemodify(g:testname, ':r') . '.res'
Bram Moolenaar43345542015-11-29 17:35:35 +0100191 write
192endif
193
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100194if len(s:errors) > 0
Bram Moolenaar43345542015-11-29 17:35:35 +0100195 " Append errors to test.log
196 split test.log
197 call append(line('$'), '')
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100198 call append(line('$'), 'From ' . g:testname . ':')
199 call append(line('$'), s:errors)
Bram Moolenaar43345542015-11-29 17:35:35 +0100200 write
201endif
202
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100203let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test')
Bram Moolenaar096c8bb2015-12-29 14:26:57 +0100204echo message
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100205call add(s:messages, message)
206if s:fail > 0
207 let message = s:fail . ' FAILED:'
Bram Moolenaar096c8bb2015-12-29 14:26:57 +0100208 echo message
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100209 call add(s:messages, message)
210 call extend(s:messages, s:errors)
Bram Moolenaar43345542015-11-29 17:35:35 +0100211endif
212
Bram Moolenaardac19472016-09-03 22:35:40 +0200213" Add SKIPPED messages
214call extend(s:messages, s:skipped)
215
216" Append messages to the file "messages"
Bram Moolenaar096c8bb2015-12-29 14:26:57 +0100217split messages
218call append(line('$'), '')
Bram Moolenaar00af60b2016-02-13 14:06:14 +0100219call append(line('$'), 'From ' . g:testname . ':')
220call append(line('$'), s:messages)
Bram Moolenaar096c8bb2015-12-29 14:26:57 +0100221write
222
Bram Moolenaar43345542015-11-29 17:35:35 +0100223qall!