Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 1 | # |
Bram Moolenaar | 6ee1016 | 2007-07-26 20:58:42 +0000 | [diff] [blame] | 2 | # Makefile to run all tests for Vim |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 3 | # |
| 4 | |
Bram Moolenaar | 3f12a24 | 2015-12-05 20:56:57 +0100 | [diff] [blame] | 5 | # Use console or GUI. |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 6 | VIMPROG = ../vim |
Bram Moolenaar | 29f9ed2 | 2018-04-10 19:20:31 +0200 | [diff] [blame] | 7 | XXDPROG = ../xxd/xxd |
Bram Moolenaar | 3f12a24 | 2015-12-05 20:56:57 +0100 | [diff] [blame] | 8 | # VIMPROG = ../gvim |
| 9 | |
Bram Moolenaar | efa304d | 2014-04-01 14:08:28 +0200 | [diff] [blame] | 10 | SCRIPTSOURCE = ../../runtime |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 11 | |
Bram Moolenaar | d8f27b3 | 2018-10-07 15:42:07 +0200 | [diff] [blame] | 12 | # Comment out this line to see the verbose output of tests. |
| 13 | # |
| 14 | # Catches SwapExists to avoid hanging at the ATTENTION prompt. |
Bram Moolenaar | 133c737 | 2019-06-06 14:38:06 +0200 | [diff] [blame] | 15 | REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null |
Bram Moolenaar | e76d7a6 | 2018-09-21 16:37:25 +0200 | [diff] [blame] | 16 | |
Bram Moolenaar | 0c77b7b | 2009-06-24 16:08:18 +0000 | [diff] [blame] | 17 | # Uncomment this line to use valgrind for memory leaks and extra warnings. |
| 18 | # The output goes into a file "valgrind.testN" |
| 19 | # Vim should be compiled with EXITFREE to avoid false warnings. |
| 20 | # This will make testing about 10 times as slow. |
Bram Moolenaar | c854898 | 2019-02-18 17:40:18 +0100 | [diff] [blame] | 21 | # VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=25 --log-file=valgrind.$* |
Bram Moolenaar | 4d2fffc | 2007-08-14 15:29:16 +0000 | [diff] [blame] | 22 | |
Bram Moolenaar | 40bbcee | 2015-12-28 22:24:41 +0100 | [diff] [blame] | 23 | default: nongui |
Bram Moolenaar | 680eeca | 2010-10-20 17:44:42 +0200 | [diff] [blame] | 24 | |
Bram Moolenaar | 7b6156f | 2015-12-28 16:01:26 +0100 | [diff] [blame] | 25 | # The list of tests is common to all systems. |
Bram Moolenaar | ec50401 | 2019-01-11 17:30:16 +0100 | [diff] [blame] | 26 | # This defines NEW_TESTS, NEW_TESTS_RES, SCRIPTS_ALL, SCRIPTS_MORE* and |
| 27 | # SCRIPTS_GUI. |
Bram Moolenaar | 7b6156f | 2015-12-28 16:01:26 +0100 | [diff] [blame] | 28 | include Make_all.mak |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 29 | |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 30 | |
Bram Moolenaar | 7b6156f | 2015-12-28 16:01:26 +0100 | [diff] [blame] | 31 | SCRIPTS = $(SCRIPTS_ALL) \ |
| 32 | $(SCRIPTS_MORE1) \ |
| 33 | $(SCRIPTS_MORE2) \ |
Bram Moolenaar | 7b6156f | 2015-12-28 16:01:26 +0100 | [diff] [blame] | 34 | $(SCRIPTS_MORE4) |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 35 | |
Bram Moolenaar | 46fd6bf | 2019-01-11 19:19:44 +0100 | [diff] [blame] | 36 | # Explicit dependencies. |
| 37 | test49.out: test49.vim |
| 38 | |
| 39 | test_options.res test_alot.res: opt_test.vim |
| 40 | |
Bram Moolenaar | ad48e6c | 2020-04-21 22:19:45 +0200 | [diff] [blame] | 41 | SCRIPTS_BENCH = test_bench_regexp.res |
Bram Moolenaar | fda3729 | 2014-11-05 14:27:36 +0100 | [diff] [blame] | 42 | |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 43 | .SUFFIXES: .in .out .res .vim |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 44 | |
Bram Moolenaar | a608243 | 2015-12-28 16:26:45 +0100 | [diff] [blame] | 45 | nongui: nolog $(SCRIPTS_FIRST) $(SCRIPTS) newtests report |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 46 | |
Bram Moolenaar | a608243 | 2015-12-28 16:26:45 +0100 | [diff] [blame] | 47 | gui: nolog $(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) newtests report |
Bram Moolenaar | f2e53da | 2008-09-10 16:26:04 +0000 | [diff] [blame] | 48 | |
Bram Moolenaar | fda3729 | 2014-11-05 14:27:36 +0100 | [diff] [blame] | 49 | benchmark: $(SCRIPTS_BENCH) |
| 50 | |
Bram Moolenaar | f2e53da | 2008-09-10 16:26:04 +0000 | [diff] [blame] | 51 | report: |
Bram Moolenaar | 18250e2 | 2019-06-06 14:28:54 +0200 | [diff] [blame] | 52 | # without the +eval feature test_result.log is a copy of test.log |
| 53 | @/bin/sh -c "if test -f test.log; \ |
| 54 | then cp test.log test_result.log; \ |
| 55 | else echo No failures reported > test_result.log; \ |
| 56 | fi" |
Bram Moolenaar | 7d23204 | 2019-11-05 22:37:20 +0100 | [diff] [blame] | 57 | $(VIMPROG) -u NONE $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL) |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 58 | @echo |
Bram Moolenaar | f2e53da | 2008-09-10 16:26:04 +0000 | [diff] [blame] | 59 | @echo 'Test results:' |
Bram Moolenaar | 9c0cec6 | 2019-06-06 13:38:15 +0200 | [diff] [blame] | 60 | @cat test_result.log |
Bram Moolenaar | f2e53da | 2008-09-10 16:26:04 +0000 | [diff] [blame] | 61 | @/bin/sh -c "if test -f test.log; \ |
Bram Moolenaar | 9c0cec6 | 2019-06-06 13:38:15 +0200 | [diff] [blame] | 62 | then echo TEST FAILURE; exit 1; \ |
Bram Moolenaar | f2e53da | 2008-09-10 16:26:04 +0000 | [diff] [blame] | 63 | else echo ALL DONE; \ |
| 64 | fi" |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 65 | |
Bram Moolenaar | ec50401 | 2019-01-11 17:30:16 +0100 | [diff] [blame] | 66 | $(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) $(NEW_TESTS_RES): $(VIMPROG) |
Bram Moolenaar | a608243 | 2015-12-28 16:26:45 +0100 | [diff] [blame] | 67 | |
| 68 | # Must run test1 first to create small.vim. |
Bram Moolenaar | ec50401 | 2019-01-11 17:30:16 +0100 | [diff] [blame] | 69 | $(SCRIPTS) $(SCRIPTS_GUI) $(NEW_TESTS_RES): $(SCRIPTS_FIRST) |
| 70 | |
| 71 | |
| 72 | # Execute an individual new style test, e.g.: |
| 73 | # make test_largefile |
| 74 | $(NEW_TESTS): |
| 75 | rm -f $@.res test.log messages |
Bram Moolenaar | 85683ec | 2020-02-20 22:35:02 +0100 | [diff] [blame] | 76 | @MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile $@.res VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE) |
Bram Moolenaar | ba08930 | 2019-10-19 18:56:58 +0200 | [diff] [blame] | 77 | @cat messages |
Bram Moolenaar | 4e0bf84 | 2019-06-07 22:40:25 +0200 | [diff] [blame] | 78 | @if test -f test.log; then \ |
| 79 | exit 1; \ |
| 80 | fi |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 81 | |
Bram Moolenaar | ec5929d | 2020-04-07 20:53:39 +0200 | [diff] [blame] | 82 | # Run only tests specific for Vim9 script |
| 83 | test_vim9: |
| 84 | rm -f test_vim9_*.res test.log messages |
| 85 | @MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile $(TEST_VIM9_RES) VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE) |
| 86 | @cat messages |
| 87 | @MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile report VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE) |
| 88 | @if test -f test.log; then \ |
| 89 | exit 1; \ |
| 90 | fi |
| 91 | |
Bram Moolenaar | 60ec5f3 | 2012-12-05 18:21:32 +0100 | [diff] [blame] | 92 | RM_ON_RUN = test.out X* viminfo |
Bram Moolenaar | 1ce9a15 | 2018-07-14 21:48:46 +0200 | [diff] [blame] | 93 | RM_ON_START = tiny.vim small.vim mbyte.vim mzscheme.vim test.ok benchmark.out |
Bram Moolenaar | 41a8260 | 2019-07-14 21:54:26 +0200 | [diff] [blame] | 94 | RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE) $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in |
Bram Moolenaar | 60ec5f3 | 2012-12-05 18:21:32 +0100 | [diff] [blame] | 95 | |
Bram Moolenaar | d9b0d83 | 2019-09-14 15:46:32 +0200 | [diff] [blame] | 96 | # Delete files that may interfere with running tests. This includes some files |
| 97 | # that may result from working on the tests, not only from running them. |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 98 | clean: |
Bram Moolenaar | 4f5b0e5 | 2019-07-05 20:43:52 +0200 | [diff] [blame] | 99 | -rm -rf *.out *.failed *.res *.rej *.orig XfakeHOME Xdir1 Xfind |
Bram Moolenaar | 8ed75cb | 2019-06-09 14:39:13 +0200 | [diff] [blame] | 100 | -rm -f opt_test.vim test.log test_result.log messages |
| 101 | -rm -f $(RM_ON_RUN) $(RM_ON_START) |
| 102 | -rm -f valgrind.* |
Bram Moolenaar | 95a467e | 2020-04-23 14:41:46 +0200 | [diff] [blame^] | 103 | -rm -f asan.* |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 104 | |
| 105 | test1.out: test1.in |
Bram Moolenaar | 6848c8b | 2013-09-22 15:03:38 +0200 | [diff] [blame] | 106 | -rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize |
Bram Moolenaar | e76d7a6 | 2018-09-21 16:37:25 +0200 | [diff] [blame] | 107 | $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL) |
Bram Moolenaar | f7dc2b5 | 2014-03-12 15:50:22 +0100 | [diff] [blame] | 108 | @/bin/sh -c "if test -f wrongtermsize; \ |
Bram Moolenaar | 6848c8b | 2013-09-22 15:03:38 +0200 | [diff] [blame] | 109 | then echo; \ |
| 110 | echo test1 FAILED - terminal size must be 80x24 or larger; \ |
| 111 | echo; exit 1; \ |
| 112 | elif diff test.out $*.ok; \ |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 113 | then mv -f test.out $*.out; \ |
| 114 | else echo; \ |
| 115 | echo test1 FAILED - Something basic is wrong; \ |
| 116 | echo; exit 1; fi" |
| 117 | -rm -rf X* viminfo |
| 118 | |
| 119 | .in.out: |
Bram Moolenaar | 60ec5f3 | 2012-12-05 18:21:32 +0100 | [diff] [blame] | 120 | -rm -rf $*.failed test.ok $(RM_ON_RUN) |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 121 | cp $*.ok test.ok |
Bram Moolenaar | 3f99152 | 2013-04-24 12:56:19 +0200 | [diff] [blame] | 122 | # Sleep a moment to avoid that the xterm title is messed up. |
| 123 | # 200 msec is sufficient, but only modern sleep supports a fraction of |
| 124 | # a second, fall back to a second if it fails. |
| 125 | @-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1" |
Bram Moolenaar | e76d7a6 | 2018-09-21 16:37:25 +0200 | [diff] [blame] | 126 | $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL) |
Bram Moolenaar | 60ec5f3 | 2012-12-05 18:21:32 +0100 | [diff] [blame] | 127 | |
Bram Moolenaar | 89eaa41 | 2016-07-31 14:17:27 +0200 | [diff] [blame] | 128 | # For flaky tests retry one time. No tests at the moment. |
| 129 | #@/bin/sh -c "if test -f test.out -a $* = test61; then \ |
| 130 | # if diff test.out $*.ok; \ |
| 131 | # then echo flaky test ok first time; \ |
| 132 | # else rm -rf $*.failed $(RM_ON_RUN); \ |
| 133 | # $(RUN_VIM) $*.in; \ |
| 134 | # fi \ |
| 135 | # fi" |
Bram Moolenaar | 60ec5f3 | 2012-12-05 18:21:32 +0100 | [diff] [blame] | 136 | |
| 137 | # Check if the test.out file matches test.ok. |
Bram Moolenaar | fbc0d2e | 2013-05-19 19:40:29 +0200 | [diff] [blame] | 138 | @/bin/sh -c "if test -f test.out; then \ |
Bram Moolenaar | 5313dcb | 2005-02-22 08:56:13 +0000 | [diff] [blame] | 139 | if diff test.out $*.ok; \ |
| 140 | then mv -f test.out $*.out; \ |
| 141 | else echo $* FAILED >>test.log; mv -f test.out $*.failed; \ |
| 142 | fi \ |
| 143 | else echo $* NO OUTPUT >>test.log; \ |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 144 | fi" |
Bram Moolenaar | fbc0d2e | 2013-05-19 19:40:29 +0200 | [diff] [blame] | 145 | @/bin/sh -c "if test -f valgrind; then\ |
| 146 | mv -f valgrind valgrind.$*; \ |
| 147 | fi" |
Bram Moolenaar | b35c338 | 2012-10-06 19:10:35 +0200 | [diff] [blame] | 148 | -rm -rf X* test.ok viminfo |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 149 | |
Bram Moolenaar | 071d427 | 2004-06-13 20:20:40 +0000 | [diff] [blame] | 150 | nolog: |
Bram Moolenaar | 096c8bb | 2015-12-29 14:26:57 +0100 | [diff] [blame] | 151 | -rm -f test.log messages |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 152 | |
| 153 | |
| 154 | # New style of tests uses Vim script with assert calls. These are easier |
| 155 | # to write and a lot easier to read and debug. |
| 156 | # Limitation: Only works with the +eval feature. |
Bram Moolenaar | 41a8260 | 2019-07-14 21:54:26 +0200 | [diff] [blame] | 157 | RUN_VIMTEST = VIMRUNTIME=$(SCRIPTSOURCE) $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 158 | |
Bram Moolenaar | e7893a4 | 2016-01-06 21:23:56 +0100 | [diff] [blame] | 159 | newtests: newtestssilent |
Bram Moolenaar | 933bef7 | 2018-09-20 21:39:33 +0200 | [diff] [blame] | 160 | @/bin/sh -c "if test -f messages && grep -q 'SKIPPED\|FAILED' messages; then cat messages; fi" |
Bram Moolenaar | e7893a4 | 2016-01-06 21:23:56 +0100 | [diff] [blame] | 161 | |
Bram Moolenaar | ec50401 | 2019-01-11 17:30:16 +0100 | [diff] [blame] | 162 | newtestssilent: $(NEW_TESTS_RES) |
Bram Moolenaar | e7893a4 | 2016-01-06 21:23:56 +0100 | [diff] [blame] | 163 | |
Bram Moolenaar | 4334554 | 2015-11-29 17:35:35 +0100 | [diff] [blame] | 164 | |
Bram Moolenaar | 1417031 | 2019-02-21 21:50:46 +0100 | [diff] [blame] | 165 | .vim.res: |
Bram Moolenaar | 6318205 | 2017-10-07 20:03:23 +0200 | [diff] [blame] | 166 | @echo "$(VIMPROG)" > vimcmd |
| 167 | @echo "$(RUN_VIMTEST)" >> vimcmd |
Bram Moolenaar | e76d7a6 | 2018-09-21 16:37:25 +0200 | [diff] [blame] | 168 | $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL) |
Bram Moolenaar | 8be2fbb | 2017-02-23 19:32:47 +0100 | [diff] [blame] | 169 | @rm vimcmd |
| 170 | |
| 171 | test_gui.res: test_gui.vim |
Bram Moolenaar | 6318205 | 2017-10-07 20:03:23 +0200 | [diff] [blame] | 172 | @echo "$(VIMPROG)" > vimcmd |
| 173 | @echo "$(RUN_GVIMTEST)" >> vimcmd |
Bram Moolenaar | 8be2fbb | 2017-02-23 19:32:47 +0100 | [diff] [blame] | 174 | $(RUN_VIMTEST) -u NONE $(NO_INITS) -S runtest.vim $< |
| 175 | @rm vimcmd |
| 176 | |
| 177 | test_gui_init.res: test_gui_init.vim |
Bram Moolenaar | 6318205 | 2017-10-07 20:03:23 +0200 | [diff] [blame] | 178 | @echo "$(VIMPROG)" > vimcmd |
| 179 | @echo "$(RUN_GVIMTEST_WITH_GVIMRC)" >> vimcmd |
Bram Moolenaar | 8774845 | 2017-03-12 17:10:33 +0100 | [diff] [blame] | 180 | $(RUN_VIMTEST) -u gui_preinit.vim -U gui_init.vim $(NO_PLUGINS) -S runtest.vim $< |
Bram Moolenaar | 66459b7 | 2016-08-06 19:01:55 +0200 | [diff] [blame] | 181 | @rm vimcmd |
Bram Moolenaar | 65408f7 | 2017-03-07 21:31:27 +0100 | [diff] [blame] | 182 | |
Bram Moolenaar | d9b0d83 | 2019-09-14 15:46:32 +0200 | [diff] [blame] | 183 | opt_test.vim: ../optiondefs.h gen_opt_test.vim |
| 184 | $(VIMPROG) -u NONE -S gen_opt_test.vim --noplugin --not-a-term ../optiondefs.h |
Bram Moolenaar | 29f9ed2 | 2018-04-10 19:20:31 +0200 | [diff] [blame] | 185 | |
| 186 | test_xxd.res: |
| 187 | XXD=$(XXDPROG); export XXD; $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim test_xxd.vim |
Bram Moolenaar | ad48e6c | 2020-04-21 22:19:45 +0200 | [diff] [blame] | 188 | |
| 189 | test_bench_regexp.res: test_bench_regexp.vim |
| 190 | -rm -rf benchmark.out $(RM_ON_RUN) |
| 191 | # Sleep a moment to avoid that the xterm title is messed up. |
| 192 | # 200 msec is sufficient, but only modern sleep supports a fraction of |
| 193 | # a second, fall back to a second if it fails. |
| 194 | @-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1" |
| 195 | $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL) |
| 196 | @/bin/sh -c "if test -f benchmark.out; then cat benchmark.out; fi" |