Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 1 | " This script is sourced while editing the .vim file with the tests. |
| 2 | " When the script is successful the .res file will be created. |
| 3 | " Errors are appended to the test.log file. |
| 4 | " |
Bram Moolenaar | befb366 | 2016-02-20 14:41:40 +0100 | [diff] [blame] | 5 | " To execute only specific test functions, add a second argument. It will be |
| 6 | " matched against the names of the Test_ funtion. E.g.: |
| 7 | " ../vim -u NONE -S runtest.vim test_channel.vim open_delay |
| 8 | " The output can be found in the "messages" file. |
| 9 | " |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 10 | " The test script may contain anything, only functions that start with |
| 11 | " "Test_" are special. These will be invoked and should contain assert |
| 12 | " functions. See test_assert.vim for an example. |
| 13 | " |
| 14 | " It is possible to source other files that contain "Test_" functions. This |
| 15 | " can speed up testing, since Vim does not need to restart. But be careful |
| 16 | " that the tests do not interfere with each other. |
| 17 | " |
| 18 | " If an error cannot be detected properly with an assert function add the |
| 19 | " error to the v:errors list: |
| 20 | " call add(v:errors, 'test foo failed: Cannot find xyz') |
| 21 | " |
| 22 | " If preparation for each Test_ function is needed, define a SetUp function. |
| 23 | " It will be called before each Test_ function. |
| 24 | " |
| 25 | " If cleanup after each Test_ function is needed, define a TearDown function. |
| 26 | " It will be called after each Test_ function. |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 27 | " |
| 28 | " When debugging a test it can be useful to add messages to v:errors: |
| 29 | " call add(v:errors, "this happened") |
| 30 | |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 31 | |
| 32 | " Without the +eval feature we can't run these tests, bail out. |
Bram Moolenaar | 4686b32 | 2015-12-28 14:44:10 +0100 | [diff] [blame] | 33 | so small.vim |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 34 | |
| 35 | " Check that the screen size is at least 24 x 80 characters. |
| 36 | if &lines < 24 || &columns < 80 |
| 37 | let error = 'Screen size too small! Tests require at least 24 lines with 80 characters' |
| 38 | echoerr error |
| 39 | split test.log |
| 40 | $put =error |
| 41 | w |
| 42 | cquit |
| 43 | endif |
| 44 | |
Bram Moolenaar | 89b1042 | 2016-07-12 22:51:22 +0200 | [diff] [blame] | 45 | " Common with all tests on all systems. |
| 46 | source setup.vim |
| 47 | |
Bram Moolenaar | c066246 | 2015-12-30 15:49:05 +0100 | [diff] [blame] | 48 | " For consistency run all tests with 'nocompatible' set. |
| 49 | " This also enables use of line continuation. |
| 50 | set nocp viminfo+=nviminfo |
| 51 | |
Bram Moolenaar | ac105ed | 2016-07-21 20:33:32 +0200 | [diff] [blame] | 52 | " Use utf-8 or latin1 be default, instead of whatever the system default |
| 53 | " happens to be. Individual tests can overrule this at the top of the file. |
| 54 | if has('multi_byte') |
| 55 | set encoding=utf-8 |
| 56 | else |
| 57 | set encoding=latin1 |
| 58 | endif |
| 59 | |
Bram Moolenaar | c066246 | 2015-12-30 15:49:05 +0100 | [diff] [blame] | 60 | " Avoid stopping at the "hit enter" prompt |
| 61 | set nomore |
| 62 | |
| 63 | " Output all messages in English. |
| 64 | lang mess C |
| 65 | |
Bram Moolenaar | f60b796 | 2016-01-16 22:47:23 +0100 | [diff] [blame] | 66 | " Always use forward slashes. |
| 67 | set shellslash |
| 68 | |
Bram Moolenaar | 28fb79d | 2016-01-09 22:28:33 +0100 | [diff] [blame] | 69 | let s:srcdir = expand('%:p:h:h') |
| 70 | |
Bram Moolenaar | 8e8df25 | 2016-05-25 21:23:21 +0200 | [diff] [blame] | 71 | " Prepare for calling test_garbagecollect_now(). |
Bram Moolenaar | ebf7dfa | 2016-04-14 12:46:51 +0200 | [diff] [blame] | 72 | let v:testing = 1 |
| 73 | |
Bram Moolenaar | 28fb79d | 2016-01-09 22:28:33 +0100 | [diff] [blame] | 74 | " Support function: get the alloc ID by name. |
| 75 | function GetAllocId(name) |
| 76 | exe 'split ' . s:srcdir . '/alloc.h' |
Bram Moolenaar | 065ee9a | 2016-01-15 20:53:38 +0100 | [diff] [blame] | 77 | let top = search('typedef enum') |
| 78 | if top == 0 |
| 79 | call add(v:errors, 'typedef not found in alloc.h') |
| 80 | endif |
Bram Moolenaar | 28fb79d | 2016-01-09 22:28:33 +0100 | [diff] [blame] | 81 | let lnum = search('aid_' . a:name . ',') |
| 82 | if lnum == 0 |
| 83 | call add(v:errors, 'Alloc ID ' . a:name . ' not defined') |
| 84 | endif |
| 85 | close |
Bram Moolenaar | 065ee9a | 2016-01-15 20:53:38 +0100 | [diff] [blame] | 86 | return lnum - top - 1 |
Bram Moolenaar | 28fb79d | 2016-01-09 22:28:33 +0100 | [diff] [blame] | 87 | endfunc |
| 88 | |
Bram Moolenaar | b5760a1 | 2016-03-03 13:10:44 +0100 | [diff] [blame] | 89 | function RunTheTest(test) |
| 90 | echo 'Executing ' . a:test |
| 91 | if exists("*SetUp") |
| 92 | call SetUp() |
| 93 | endif |
| 94 | |
| 95 | call add(s:messages, 'Executing ' . a:test) |
| 96 | let s:done += 1 |
| 97 | try |
| 98 | exe 'call ' . a:test |
Bram Moolenaar | dac1947 | 2016-09-03 22:35:40 +0200 | [diff] [blame] | 99 | catch /^\cskipped/ |
| 100 | call add(s:messages, ' Skipped') |
| 101 | call add(s:skipped, 'SKIPPED ' . a:test . ': ' . substitute(v:exception, '^\S*\s\+', '', '')) |
Bram Moolenaar | b5760a1 | 2016-03-03 13:10:44 +0100 | [diff] [blame] | 102 | catch |
| 103 | call add(v:errors, 'Caught exception in ' . a:test . ': ' . v:exception . ' @ ' . v:throwpoint) |
| 104 | endtry |
| 105 | |
| 106 | if exists("*TearDown") |
| 107 | call TearDown() |
| 108 | endif |
Bram Moolenaar | 7cba71d | 2016-08-02 23:04:49 +0200 | [diff] [blame] | 109 | |
| 110 | " Close any extra windows and make the current one not modified. |
Bram Moolenaar | 358308d | 2016-08-24 21:21:26 +0200 | [diff] [blame] | 111 | while 1 |
| 112 | let wincount = winnr('$') |
| 113 | if wincount == 1 |
| 114 | break |
| 115 | endif |
Bram Moolenaar | 7cba71d | 2016-08-02 23:04:49 +0200 | [diff] [blame] | 116 | bwipe! |
Bram Moolenaar | 358308d | 2016-08-24 21:21:26 +0200 | [diff] [blame] | 117 | if wincount == winnr('$') |
| 118 | " Did not manage to close a window. |
| 119 | only! |
| 120 | break |
| 121 | endif |
Bram Moolenaar | 7cba71d | 2016-08-02 23:04:49 +0200 | [diff] [blame] | 122 | endwhile |
| 123 | set nomodified |
Bram Moolenaar | b5760a1 | 2016-03-03 13:10:44 +0100 | [diff] [blame] | 124 | endfunc |
Bram Moolenaar | 28fb79d | 2016-01-09 22:28:33 +0100 | [diff] [blame] | 125 | |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 126 | " Source the test script. First grab the file name, in case the script |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 127 | " navigates away. g:testname can be used by the tests. |
| 128 | let g:testname = expand('%') |
| 129 | let s:done = 0 |
| 130 | let s:fail = 0 |
| 131 | let s:errors = [] |
| 132 | let s:messages = [] |
Bram Moolenaar | dac1947 | 2016-09-03 22:35:40 +0200 | [diff] [blame] | 133 | let s:skipped = [] |
Bram Moolenaar | a2cce86 | 2016-01-02 19:50:04 +0100 | [diff] [blame] | 134 | if expand('%') =~ 'test_viml.vim' |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 135 | " this test has intentional s:errors, don't use try/catch. |
Bram Moolenaar | 4686b32 | 2015-12-28 14:44:10 +0100 | [diff] [blame] | 136 | source % |
Bram Moolenaar | a2cce86 | 2016-01-02 19:50:04 +0100 | [diff] [blame] | 137 | else |
| 138 | try |
| 139 | source % |
| 140 | catch |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 141 | let s:fail += 1 |
| 142 | call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint) |
Bram Moolenaar | a2cce86 | 2016-01-02 19:50:04 +0100 | [diff] [blame] | 143 | endtry |
| 144 | endif |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 145 | |
Bram Moolenaar | b5760a1 | 2016-03-03 13:10:44 +0100 | [diff] [blame] | 146 | " Names of flaky tests. |
Bram Moolenaar | e1c8c7a | 2016-09-11 16:48:50 +0200 | [diff] [blame] | 147 | let s:flaky = [ |
| 148 | \ 'Test_reltime()', |
| 149 | \ 'Test_nb_basic()', |
Bram Moolenaar | edeb846 | 2016-09-12 13:32:02 +0200 | [diff] [blame] | 150 | \ 'Test_communicate()', |
Bram Moolenaar | c79d6aa | 2016-09-25 22:27:37 +0200 | [diff] [blame] | 151 | \ 'Test_pipe_through_sort_all()', |
Bram Moolenaar | e1c8c7a | 2016-09-11 16:48:50 +0200 | [diff] [blame] | 152 | \ 'Test_pipe_through_sort_some()' |
| 153 | \ ] |
Bram Moolenaar | b5760a1 | 2016-03-03 13:10:44 +0100 | [diff] [blame] | 154 | |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 155 | " Locate Test_ functions and execute them. |
Bram Moolenaar | a99b904 | 2016-01-17 17:10:59 +0100 | [diff] [blame] | 156 | set nomore |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 157 | redir @q |
Bram Moolenaar | 93bf558 | 2016-02-18 22:25:47 +0100 | [diff] [blame] | 158 | silent function /^Test_ |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 159 | redir END |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 160 | let s:tests = split(substitute(@q, 'function \(\k*()\)', '\1', 'g')) |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 161 | |
Bram Moolenaar | befb366 | 2016-02-20 14:41:40 +0100 | [diff] [blame] | 162 | " If there is an extra argument filter the function names against it. |
| 163 | if argc() > 1 |
| 164 | let s:tests = filter(s:tests, 'v:val =~ argv(1)') |
| 165 | endif |
| 166 | |
Bram Moolenaar | cfc0a35 | 2016-01-09 20:23:00 +0100 | [diff] [blame] | 167 | " Execute the tests in alphabetical order. |
Bram Moolenaar | 93bf558 | 2016-02-18 22:25:47 +0100 | [diff] [blame] | 168 | for s:test in sort(s:tests) |
Bram Moolenaar | b5760a1 | 2016-03-03 13:10:44 +0100 | [diff] [blame] | 169 | call RunTheTest(s:test) |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 170 | |
Bram Moolenaar | b5760a1 | 2016-03-03 13:10:44 +0100 | [diff] [blame] | 171 | if len(v:errors) > 0 && index(s:flaky, s:test) >= 0 |
| 172 | call add(s:messages, 'Flaky test failed, running it again') |
| 173 | let v:errors = [] |
| 174 | call RunTheTest(s:test) |
| 175 | endif |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 176 | |
| 177 | if len(v:errors) > 0 |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 178 | let s:fail += 1 |
| 179 | call add(s:errors, 'Found errors in ' . s:test . ':') |
| 180 | call extend(s:errors, v:errors) |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 181 | let v:errors = [] |
| 182 | endif |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 183 | endfor |
| 184 | |
Bram Moolenaar | fc4ad61 | 2016-07-09 15:38:32 +0200 | [diff] [blame] | 185 | " Don't write viminfo on exit. |
| 186 | set viminfo= |
| 187 | |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 188 | if s:fail == 0 |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 189 | " Success, create the .res file so that make knows it's done. |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 190 | exe 'split ' . fnamemodify(g:testname, ':r') . '.res' |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 191 | write |
| 192 | endif |
| 193 | |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 194 | if len(s:errors) > 0 |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 195 | " Append errors to test.log |
| 196 | split test.log |
| 197 | call append(line('$'), '') |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 198 | call append(line('$'), 'From ' . g:testname . ':') |
| 199 | call append(line('$'), s:errors) |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 200 | write |
| 201 | endif |
| 202 | |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 203 | let message = 'Executed ' . s:done . (s:done > 1 ? ' tests' : ' test') |
Bram Moolenaar | 096c8bb | 2015-12-29 14:26:57 +0100 | [diff] [blame] | 204 | echo message |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 205 | call add(s:messages, message) |
| 206 | if s:fail > 0 |
| 207 | let message = s:fail . ' FAILED:' |
Bram Moolenaar | 096c8bb | 2015-12-29 14:26:57 +0100 | [diff] [blame] | 208 | echo message |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 209 | call add(s:messages, message) |
| 210 | call extend(s:messages, s:errors) |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 211 | endif |
| 212 | |
Bram Moolenaar | dac1947 | 2016-09-03 22:35:40 +0200 | [diff] [blame] | 213 | " Add SKIPPED messages |
| 214 | call extend(s:messages, s:skipped) |
| 215 | |
| 216 | " Append messages to the file "messages" |
Bram Moolenaar | 096c8bb | 2015-12-29 14:26:57 +0100 | [diff] [blame] | 217 | split messages |
| 218 | call append(line('$'), '') |
Bram Moolenaar | 00af60b | 2016-02-13 14:06:14 +0100 | [diff] [blame] | 219 | call append(line('$'), 'From ' . g:testname . ':') |
| 220 | call append(line('$'), s:messages) |
Bram Moolenaar | 096c8bb | 2015-12-29 14:26:57 +0100 | [diff] [blame] | 221 | write |
| 222 | |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 223 | qall! |