blob: 7f9ef0434266ab1ccfc4c57986cb6a248acc8e38 [file] [log] [blame]
Bram Moolenaar071d4272004-06-13 20:20:40 +00001#
Bram Moolenaar6ee10162007-07-26 20:58:42 +00002# Makefile to run all tests for Vim
Bram Moolenaar071d4272004-06-13 20:20:40 +00003#
4
Bram Moolenaar3f12a242015-12-05 20:56:57 +01005# Use console or GUI.
Bram Moolenaar071d4272004-06-13 20:20:40 +00006VIMPROG = ../vim
Bram Moolenaar29f9ed22018-04-10 19:20:31 +02007XXDPROG = ../xxd/xxd
Bram Moolenaar3f12a242015-12-05 20:56:57 +01008# VIMPROG = ../gvim
9
Bram Moolenaarefa304d2014-04-01 14:08:28 +020010SCRIPTSOURCE = ../../runtime
Bram Moolenaar071d4272004-06-13 20:20:40 +000011
Bram Moolenaard8f27b32018-10-07 15:42:07 +020012# Comment out this line to see the verbose output of tests.
13#
14# Catches SwapExists to avoid hanging at the ATTENTION prompt.
Bram Moolenaar133c7372019-06-06 14:38:06 +020015REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
Bram Moolenaare76d7a62018-09-21 16:37:25 +020016
Bram Moolenaar0c77b7b2009-06-24 16:08:18 +000017# Uncomment this line to use valgrind for memory leaks and extra warnings.
18# The output goes into a file "valgrind.testN"
19# Vim should be compiled with EXITFREE to avoid false warnings.
20# This will make testing about 10 times as slow.
Bram Moolenaarc8548982019-02-18 17:40:18 +010021# VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=25 --log-file=valgrind.$*
Bram Moolenaar4d2fffc2007-08-14 15:29:16 +000022
Bram Moolenaar40bbcee2015-12-28 22:24:41 +010023default: nongui
Bram Moolenaar680eeca2010-10-20 17:44:42 +020024
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010025# The list of tests is common to all systems.
Bram Moolenaarec504012019-01-11 17:30:16 +010026# This defines NEW_TESTS, NEW_TESTS_RES, SCRIPTS_ALL, SCRIPTS_MORE* and
27# SCRIPTS_GUI.
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010028include Make_all.mak
Bram Moolenaar071d4272004-06-13 20:20:40 +000029
Bram Moolenaar43345542015-11-29 17:35:35 +010030
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010031SCRIPTS = $(SCRIPTS_ALL) \
32 $(SCRIPTS_MORE1) \
33 $(SCRIPTS_MORE2) \
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010034 $(SCRIPTS_MORE4)
Bram Moolenaar071d4272004-06-13 20:20:40 +000035
Bram Moolenaar46fd6bf2019-01-11 19:19:44 +010036# Explicit dependencies.
37test49.out: test49.vim
38
39test_options.res test_alot.res: opt_test.vim
40
Bram Moolenaarad48e6c2020-04-21 22:19:45 +020041SCRIPTS_BENCH = test_bench_regexp.res
Bram Moolenaarfda37292014-11-05 14:27:36 +010042
Bram Moolenaar43345542015-11-29 17:35:35 +010043.SUFFIXES: .in .out .res .vim
Bram Moolenaar071d4272004-06-13 20:20:40 +000044
Bram Moolenaara6082432015-12-28 16:26:45 +010045nongui: nolog $(SCRIPTS_FIRST) $(SCRIPTS) newtests report
Bram Moolenaar071d4272004-06-13 20:20:40 +000046
Bram Moolenaara6082432015-12-28 16:26:45 +010047gui: nolog $(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) newtests report
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000048
Bram Moolenaarfda37292014-11-05 14:27:36 +010049benchmark: $(SCRIPTS_BENCH)
50
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000051report:
Bram Moolenaar18250e22019-06-06 14:28:54 +020052 # without the +eval feature test_result.log is a copy of test.log
53 @/bin/sh -c "if test -f test.log; \
54 then cp test.log test_result.log; \
55 else echo No failures reported > test_result.log; \
56 fi"
Bram Moolenaar7d232042019-11-05 22:37:20 +010057 $(VIMPROG) -u NONE $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL)
Bram Moolenaar071d4272004-06-13 20:20:40 +000058 @echo
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000059 @echo 'Test results:'
Bram Moolenaar9c0cec62019-06-06 13:38:15 +020060 @cat test_result.log
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000061 @/bin/sh -c "if test -f test.log; \
Bram Moolenaar9c0cec62019-06-06 13:38:15 +020062 then echo TEST FAILURE; exit 1; \
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000063 else echo ALL DONE; \
64 fi"
Bram Moolenaar071d4272004-06-13 20:20:40 +000065
Bram Moolenaarec504012019-01-11 17:30:16 +010066$(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) $(NEW_TESTS_RES): $(VIMPROG)
Bram Moolenaara6082432015-12-28 16:26:45 +010067
68# Must run test1 first to create small.vim.
Bram Moolenaarec504012019-01-11 17:30:16 +010069$(SCRIPTS) $(SCRIPTS_GUI) $(NEW_TESTS_RES): $(SCRIPTS_FIRST)
70
71
72# Execute an individual new style test, e.g.:
73# make test_largefile
74$(NEW_TESTS):
75 rm -f $@.res test.log messages
Bram Moolenaar85683ec2020-02-20 22:35:02 +010076 @MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile $@.res VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE)
Bram Moolenaarba089302019-10-19 18:56:58 +020077 @cat messages
Bram Moolenaar4e0bf842019-06-07 22:40:25 +020078 @if test -f test.log; then \
79 exit 1; \
80 fi
Bram Moolenaar071d4272004-06-13 20:20:40 +000081
Bram Moolenaarec5929d2020-04-07 20:53:39 +020082# Run only tests specific for Vim9 script
83test_vim9:
84 rm -f test_vim9_*.res test.log messages
85 @MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile $(TEST_VIM9_RES) VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE)
86 @cat messages
87 @MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile report VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE)
88 @if test -f test.log; then \
89 exit 1; \
90 fi
91
Bram Moolenaar60ec5f32012-12-05 18:21:32 +010092RM_ON_RUN = test.out X* viminfo
Bram Moolenaar1ce9a152018-07-14 21:48:46 +020093RM_ON_START = tiny.vim small.vim mbyte.vim mzscheme.vim test.ok benchmark.out
Bram Moolenaar41a82602019-07-14 21:54:26 +020094RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE) $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in
Bram Moolenaar60ec5f32012-12-05 18:21:32 +010095
Bram Moolenaard9b0d832019-09-14 15:46:32 +020096# Delete files that may interfere with running tests. This includes some files
97# that may result from working on the tests, not only from running them.
Bram Moolenaar071d4272004-06-13 20:20:40 +000098clean:
Bram Moolenaar4f5b0e52019-07-05 20:43:52 +020099 -rm -rf *.out *.failed *.res *.rej *.orig XfakeHOME Xdir1 Xfind
Bram Moolenaar8ed75cb2019-06-09 14:39:13 +0200100 -rm -f opt_test.vim test.log test_result.log messages
101 -rm -f $(RM_ON_RUN) $(RM_ON_START)
102 -rm -f valgrind.*
Bram Moolenaar95a467e2020-04-23 14:41:46 +0200103 -rm -f asan.*
Bram Moolenaar071d4272004-06-13 20:20:40 +0000104
105test1.out: test1.in
Bram Moolenaar6848c8b2013-09-22 15:03:38 +0200106 -rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize
Bram Moolenaare76d7a62018-09-21 16:37:25 +0200107 $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL)
Bram Moolenaarf7dc2b52014-03-12 15:50:22 +0100108 @/bin/sh -c "if test -f wrongtermsize; \
Bram Moolenaar6848c8b2013-09-22 15:03:38 +0200109 then echo; \
110 echo test1 FAILED - terminal size must be 80x24 or larger; \
111 echo; exit 1; \
112 elif diff test.out $*.ok; \
Bram Moolenaar071d4272004-06-13 20:20:40 +0000113 then mv -f test.out $*.out; \
114 else echo; \
115 echo test1 FAILED - Something basic is wrong; \
116 echo; exit 1; fi"
117 -rm -rf X* viminfo
118
119.in.out:
Bram Moolenaar60ec5f32012-12-05 18:21:32 +0100120 -rm -rf $*.failed test.ok $(RM_ON_RUN)
Bram Moolenaar071d4272004-06-13 20:20:40 +0000121 cp $*.ok test.ok
Bram Moolenaar3f991522013-04-24 12:56:19 +0200122 # Sleep a moment to avoid that the xterm title is messed up.
123 # 200 msec is sufficient, but only modern sleep supports a fraction of
124 # a second, fall back to a second if it fails.
125 @-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1"
Bram Moolenaare76d7a62018-09-21 16:37:25 +0200126 $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL)
Bram Moolenaar60ec5f32012-12-05 18:21:32 +0100127
Bram Moolenaar89eaa412016-07-31 14:17:27 +0200128 # For flaky tests retry one time. No tests at the moment.
129 #@/bin/sh -c "if test -f test.out -a $* = test61; then \
130 # if diff test.out $*.ok; \
131 # then echo flaky test ok first time; \
132 # else rm -rf $*.failed $(RM_ON_RUN); \
133 # $(RUN_VIM) $*.in; \
134 # fi \
135 # fi"
Bram Moolenaar60ec5f32012-12-05 18:21:32 +0100136
137 # Check if the test.out file matches test.ok.
Bram Moolenaarfbc0d2e2013-05-19 19:40:29 +0200138 @/bin/sh -c "if test -f test.out; then \
Bram Moolenaar5313dcb2005-02-22 08:56:13 +0000139 if diff test.out $*.ok; \
140 then mv -f test.out $*.out; \
141 else echo $* FAILED >>test.log; mv -f test.out $*.failed; \
142 fi \
143 else echo $* NO OUTPUT >>test.log; \
Bram Moolenaar071d4272004-06-13 20:20:40 +0000144 fi"
Bram Moolenaarfbc0d2e2013-05-19 19:40:29 +0200145 @/bin/sh -c "if test -f valgrind; then\
146 mv -f valgrind valgrind.$*; \
147 fi"
Bram Moolenaarb35c3382012-10-06 19:10:35 +0200148 -rm -rf X* test.ok viminfo
Bram Moolenaar071d4272004-06-13 20:20:40 +0000149
Bram Moolenaar071d4272004-06-13 20:20:40 +0000150nolog:
Bram Moolenaar096c8bb2015-12-29 14:26:57 +0100151 -rm -f test.log messages
Bram Moolenaar43345542015-11-29 17:35:35 +0100152
153
154# New style of tests uses Vim script with assert calls. These are easier
155# to write and a lot easier to read and debug.
156# Limitation: Only works with the +eval feature.
Bram Moolenaar41a82602019-07-14 21:54:26 +0200157RUN_VIMTEST = VIMRUNTIME=$(SCRIPTSOURCE) $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim
Bram Moolenaar43345542015-11-29 17:35:35 +0100158
Bram Moolenaare7893a42016-01-06 21:23:56 +0100159newtests: newtestssilent
Bram Moolenaar933bef72018-09-20 21:39:33 +0200160 @/bin/sh -c "if test -f messages && grep -q 'SKIPPED\|FAILED' messages; then cat messages; fi"
Bram Moolenaare7893a42016-01-06 21:23:56 +0100161
Bram Moolenaarec504012019-01-11 17:30:16 +0100162newtestssilent: $(NEW_TESTS_RES)
Bram Moolenaare7893a42016-01-06 21:23:56 +0100163
Bram Moolenaar43345542015-11-29 17:35:35 +0100164
Bram Moolenaar14170312019-02-21 21:50:46 +0100165.vim.res:
Bram Moolenaar63182052017-10-07 20:03:23 +0200166 @echo "$(VIMPROG)" > vimcmd
167 @echo "$(RUN_VIMTEST)" >> vimcmd
Bram Moolenaare76d7a62018-09-21 16:37:25 +0200168 $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL)
Bram Moolenaar8be2fbb2017-02-23 19:32:47 +0100169 @rm vimcmd
170
171test_gui.res: test_gui.vim
Bram Moolenaar63182052017-10-07 20:03:23 +0200172 @echo "$(VIMPROG)" > vimcmd
173 @echo "$(RUN_GVIMTEST)" >> vimcmd
Bram Moolenaar8be2fbb2017-02-23 19:32:47 +0100174 $(RUN_VIMTEST) -u NONE $(NO_INITS) -S runtest.vim $<
175 @rm vimcmd
176
177test_gui_init.res: test_gui_init.vim
Bram Moolenaar63182052017-10-07 20:03:23 +0200178 @echo "$(VIMPROG)" > vimcmd
179 @echo "$(RUN_GVIMTEST_WITH_GVIMRC)" >> vimcmd
Bram Moolenaar87748452017-03-12 17:10:33 +0100180 $(RUN_VIMTEST) -u gui_preinit.vim -U gui_init.vim $(NO_PLUGINS) -S runtest.vim $<
Bram Moolenaar66459b72016-08-06 19:01:55 +0200181 @rm vimcmd
Bram Moolenaar65408f72017-03-07 21:31:27 +0100182
Bram Moolenaard9b0d832019-09-14 15:46:32 +0200183opt_test.vim: ../optiondefs.h gen_opt_test.vim
184 $(VIMPROG) -u NONE -S gen_opt_test.vim --noplugin --not-a-term ../optiondefs.h
Bram Moolenaar29f9ed22018-04-10 19:20:31 +0200185
186test_xxd.res:
187 XXD=$(XXDPROG); export XXD; $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim test_xxd.vim
Bram Moolenaarad48e6c2020-04-21 22:19:45 +0200188
189test_bench_regexp.res: test_bench_regexp.vim
190 -rm -rf benchmark.out $(RM_ON_RUN)
191 # Sleep a moment to avoid that the xterm title is messed up.
192 # 200 msec is sufficient, but only modern sleep supports a fraction of
193 # a second, fall back to a second if it fails.
194 @-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1"
195 $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL)
196 @/bin/sh -c "if test -f benchmark.out; then cat benchmark.out; fi"