blob: d3c1184e8c01c97bc9fef1162056d5379e9b7817 [file] [log] [blame]
Bram Moolenaar071d4272004-06-13 20:20:40 +00001#
Bram Moolenaar6ee10162007-07-26 20:58:42 +00002# Makefile to run all tests for Vim
Bram Moolenaar071d4272004-06-13 20:20:40 +00003#
4
Bram Moolenaar3f12a242015-12-05 20:56:57 +01005# Use console or GUI.
Bram Moolenaar071d4272004-06-13 20:20:40 +00006VIMPROG = ../vim
Bram Moolenaar29f9ed22018-04-10 19:20:31 +02007XXDPROG = ../xxd/xxd
Bram Moolenaar3f12a242015-12-05 20:56:57 +01008# VIMPROG = ../gvim
9
Bram Moolenaarefa304d2014-04-01 14:08:28 +020010SCRIPTSOURCE = ../../runtime
Bram Moolenaar071d4272004-06-13 20:20:40 +000011
Bram Moolenaard8f27b32018-10-07 15:42:07 +020012# Comment out this line to see the verbose output of tests.
13#
14# Catches SwapExists to avoid hanging at the ATTENTION prompt.
Bram Moolenaar133c7372019-06-06 14:38:06 +020015REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
Bram Moolenaare76d7a62018-09-21 16:37:25 +020016
Bram Moolenaar0c77b7b2009-06-24 16:08:18 +000017# Uncomment this line to use valgrind for memory leaks and extra warnings.
18# The output goes into a file "valgrind.testN"
19# Vim should be compiled with EXITFREE to avoid false warnings.
20# This will make testing about 10 times as slow.
Bram Moolenaarc8548982019-02-18 17:40:18 +010021# VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=25 --log-file=valgrind.$*
Bram Moolenaar4d2fffc2007-08-14 15:29:16 +000022
Bram Moolenaar40bbcee2015-12-28 22:24:41 +010023default: nongui
Bram Moolenaar680eeca2010-10-20 17:44:42 +020024
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010025# The list of tests is common to all systems.
Bram Moolenaarec504012019-01-11 17:30:16 +010026# This defines NEW_TESTS, NEW_TESTS_RES, SCRIPTS_ALL, SCRIPTS_MORE* and
27# SCRIPTS_GUI.
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010028include Make_all.mak
Bram Moolenaar071d4272004-06-13 20:20:40 +000029
Bram Moolenaar43345542015-11-29 17:35:35 +010030
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010031SCRIPTS = $(SCRIPTS_ALL) \
32 $(SCRIPTS_MORE1) \
33 $(SCRIPTS_MORE2) \
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010034 $(SCRIPTS_MORE4)
Bram Moolenaar071d4272004-06-13 20:20:40 +000035
Bram Moolenaar46fd6bf2019-01-11 19:19:44 +010036# Explicit dependencies.
37test49.out: test49.vim
38
39test_options.res test_alot.res: opt_test.vim
40
Bram Moolenaarfda37292014-11-05 14:27:36 +010041SCRIPTS_BENCH = bench_re_freeze.out
42
Bram Moolenaar43345542015-11-29 17:35:35 +010043.SUFFIXES: .in .out .res .vim
Bram Moolenaar071d4272004-06-13 20:20:40 +000044
Bram Moolenaara6082432015-12-28 16:26:45 +010045nongui: nolog $(SCRIPTS_FIRST) $(SCRIPTS) newtests report
Bram Moolenaar071d4272004-06-13 20:20:40 +000046
Bram Moolenaara6082432015-12-28 16:26:45 +010047gui: nolog $(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) newtests report
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000048
Bram Moolenaarfda37292014-11-05 14:27:36 +010049benchmark: $(SCRIPTS_BENCH)
50
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000051report:
Bram Moolenaar18250e22019-06-06 14:28:54 +020052 # without the +eval feature test_result.log is a copy of test.log
53 @/bin/sh -c "if test -f test.log; \
54 then cp test.log test_result.log; \
55 else echo No failures reported > test_result.log; \
56 fi"
Bram Moolenaar9c0cec62019-06-06 13:38:15 +020057 $(RUN_VIMTEST) $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL)
Bram Moolenaar071d4272004-06-13 20:20:40 +000058 @echo
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000059 @echo 'Test results:'
Bram Moolenaar9c0cec62019-06-06 13:38:15 +020060 @cat test_result.log
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000061 @/bin/sh -c "if test -f test.log; \
Bram Moolenaar9c0cec62019-06-06 13:38:15 +020062 then echo TEST FAILURE; exit 1; \
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000063 else echo ALL DONE; \
64 fi"
Bram Moolenaar071d4272004-06-13 20:20:40 +000065
Bram Moolenaarec504012019-01-11 17:30:16 +010066$(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) $(NEW_TESTS_RES): $(VIMPROG)
Bram Moolenaara6082432015-12-28 16:26:45 +010067
68# Must run test1 first to create small.vim.
Bram Moolenaarec504012019-01-11 17:30:16 +010069$(SCRIPTS) $(SCRIPTS_GUI) $(NEW_TESTS_RES): $(SCRIPTS_FIRST)
70
71
72# Execute an individual new style test, e.g.:
73# make test_largefile
74$(NEW_TESTS):
75 rm -f $@.res test.log messages
76 $(MAKE) -f Makefile $@.res
Bram Moolenaarec504012019-01-11 17:30:16 +010077 cat messages
Bram Moolenaar4e0bf842019-06-07 22:40:25 +020078 @if test -f test.log; then \
79 exit 1; \
80 fi
Bram Moolenaar071d4272004-06-13 20:20:40 +000081
Bram Moolenaar60ec5f32012-12-05 18:21:32 +010082RM_ON_RUN = test.out X* viminfo
Bram Moolenaar1ce9a152018-07-14 21:48:46 +020083RM_ON_START = tiny.vim small.vim mbyte.vim mzscheme.vim test.ok benchmark.out
Bram Moolenaar8be2fbb2017-02-23 19:32:47 +010084RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in
Bram Moolenaar60ec5f32012-12-05 18:21:32 +010085
Bram Moolenaar071d4272004-06-13 20:20:40 +000086clean:
Bram Moolenaar9c0cec62019-06-06 13:38:15 +020087 -rm -rf *.out *.failed *.res *.rej *.orig
Bram Moolenaar8ed75cb2019-06-09 14:39:13 +020088 -rm -f opt_test.vim test.log test_result.log messages
89 -rm -f $(RM_ON_RUN) $(RM_ON_START)
90 -rm -f valgrind.*
Bram Moolenaar071d4272004-06-13 20:20:40 +000091
92test1.out: test1.in
Bram Moolenaar6848c8b2013-09-22 15:03:38 +020093 -rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize
Bram Moolenaare76d7a62018-09-21 16:37:25 +020094 $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL)
Bram Moolenaarf7dc2b52014-03-12 15:50:22 +010095 @/bin/sh -c "if test -f wrongtermsize; \
Bram Moolenaar6848c8b2013-09-22 15:03:38 +020096 then echo; \
97 echo test1 FAILED - terminal size must be 80x24 or larger; \
98 echo; exit 1; \
99 elif diff test.out $*.ok; \
Bram Moolenaar071d4272004-06-13 20:20:40 +0000100 then mv -f test.out $*.out; \
101 else echo; \
102 echo test1 FAILED - Something basic is wrong; \
103 echo; exit 1; fi"
104 -rm -rf X* viminfo
105
106.in.out:
Bram Moolenaar60ec5f32012-12-05 18:21:32 +0100107 -rm -rf $*.failed test.ok $(RM_ON_RUN)
Bram Moolenaar071d4272004-06-13 20:20:40 +0000108 cp $*.ok test.ok
Bram Moolenaar3f991522013-04-24 12:56:19 +0200109 # Sleep a moment to avoid that the xterm title is messed up.
110 # 200 msec is sufficient, but only modern sleep supports a fraction of
111 # a second, fall back to a second if it fails.
112 @-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1"
Bram Moolenaare76d7a62018-09-21 16:37:25 +0200113 $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL)
Bram Moolenaar60ec5f32012-12-05 18:21:32 +0100114
Bram Moolenaar89eaa412016-07-31 14:17:27 +0200115 # For flaky tests retry one time. No tests at the moment.
116 #@/bin/sh -c "if test -f test.out -a $* = test61; then \
117 # if diff test.out $*.ok; \
118 # then echo flaky test ok first time; \
119 # else rm -rf $*.failed $(RM_ON_RUN); \
120 # $(RUN_VIM) $*.in; \
121 # fi \
122 # fi"
Bram Moolenaar60ec5f32012-12-05 18:21:32 +0100123
124 # Check if the test.out file matches test.ok.
Bram Moolenaarfbc0d2e2013-05-19 19:40:29 +0200125 @/bin/sh -c "if test -f test.out; then \
Bram Moolenaar5313dcb2005-02-22 08:56:13 +0000126 if diff test.out $*.ok; \
127 then mv -f test.out $*.out; \
128 else echo $* FAILED >>test.log; mv -f test.out $*.failed; \
129 fi \
130 else echo $* NO OUTPUT >>test.log; \
Bram Moolenaar071d4272004-06-13 20:20:40 +0000131 fi"
Bram Moolenaarfbc0d2e2013-05-19 19:40:29 +0200132 @/bin/sh -c "if test -f valgrind; then\
133 mv -f valgrind valgrind.$*; \
134 fi"
Bram Moolenaarb35c3382012-10-06 19:10:35 +0200135 -rm -rf X* test.ok viminfo
Bram Moolenaar071d4272004-06-13 20:20:40 +0000136
Bram Moolenaarfda37292014-11-05 14:27:36 +0100137bench_re_freeze.out: bench_re_freeze.vim
138 -rm -rf benchmark.out $(RM_ON_RUN)
139 # Sleep a moment to avoid that the xterm title is messed up.
140 # 200 msec is sufficient, but only modern sleep supports a fraction of
141 # a second, fall back to a second if it fails.
142 @-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1"
Bram Moolenaare76d7a62018-09-21 16:37:25 +0200143 $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL)
Bram Moolenaarfda37292014-11-05 14:27:36 +0100144 @/bin/sh -c "if test -f benchmark.out; then cat benchmark.out; fi"
145
Bram Moolenaar071d4272004-06-13 20:20:40 +0000146nolog:
Bram Moolenaar096c8bb2015-12-29 14:26:57 +0100147 -rm -f test.log messages
Bram Moolenaar43345542015-11-29 17:35:35 +0100148
149
150# New style of tests uses Vim script with assert calls. These are easier
151# to write and a lot easier to read and debug.
152# Limitation: Only works with the +eval feature.
Bram Moolenaar8be2fbb2017-02-23 19:32:47 +0100153RUN_VIMTEST = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim
Bram Moolenaar43345542015-11-29 17:35:35 +0100154
Bram Moolenaare7893a42016-01-06 21:23:56 +0100155newtests: newtestssilent
Bram Moolenaar933bef72018-09-20 21:39:33 +0200156 @/bin/sh -c "if test -f messages && grep -q 'SKIPPED\|FAILED' messages; then cat messages; fi"
Bram Moolenaare7893a42016-01-06 21:23:56 +0100157
Bram Moolenaarec504012019-01-11 17:30:16 +0100158newtestssilent: $(NEW_TESTS_RES)
Bram Moolenaare7893a42016-01-06 21:23:56 +0100159
Bram Moolenaar43345542015-11-29 17:35:35 +0100160
Bram Moolenaar14170312019-02-21 21:50:46 +0100161.vim.res:
Bram Moolenaar63182052017-10-07 20:03:23 +0200162 @echo "$(VIMPROG)" > vimcmd
163 @echo "$(RUN_VIMTEST)" >> vimcmd
Bram Moolenaare76d7a62018-09-21 16:37:25 +0200164 $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL)
Bram Moolenaar8be2fbb2017-02-23 19:32:47 +0100165 @rm vimcmd
166
167test_gui.res: test_gui.vim
Bram Moolenaar63182052017-10-07 20:03:23 +0200168 @echo "$(VIMPROG)" > vimcmd
169 @echo "$(RUN_GVIMTEST)" >> vimcmd
Bram Moolenaar8be2fbb2017-02-23 19:32:47 +0100170 $(RUN_VIMTEST) -u NONE $(NO_INITS) -S runtest.vim $<
171 @rm vimcmd
172
173test_gui_init.res: test_gui_init.vim
Bram Moolenaar63182052017-10-07 20:03:23 +0200174 @echo "$(VIMPROG)" > vimcmd
175 @echo "$(RUN_GVIMTEST_WITH_GVIMRC)" >> vimcmd
Bram Moolenaar87748452017-03-12 17:10:33 +0100176 $(RUN_VIMTEST) -u gui_preinit.vim -U gui_init.vim $(NO_PLUGINS) -S runtest.vim $<
Bram Moolenaar66459b72016-08-06 19:01:55 +0200177 @rm vimcmd
Bram Moolenaar65408f72017-03-07 21:31:27 +0100178
179opt_test.vim: ../option.c gen_opt_test.vim
Bram Moolenaarbb962262017-03-08 00:01:35 +0100180 $(VIMPROG) -u NONE -S gen_opt_test.vim --noplugin --not-a-term ../option.c
Bram Moolenaar29f9ed22018-04-10 19:20:31 +0200181
182test_xxd.res:
183 XXD=$(XXDPROG); export XXD; $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim test_xxd.vim