blob: a0520e0fb41058bb6a106f25caf0e2cb9be5d76f [file] [log] [blame]
Bram Moolenaar071d4272004-06-13 20:20:40 +00001#
Bram Moolenaar6ee10162007-07-26 20:58:42 +00002# Makefile to run all tests for Vim
Bram Moolenaar071d4272004-06-13 20:20:40 +00003#
4
Bram Moolenaar3f12a242015-12-05 20:56:57 +01005# Use console or GUI.
Bram Moolenaar071d4272004-06-13 20:20:40 +00006VIMPROG = ../vim
Bram Moolenaar29f9ed22018-04-10 19:20:31 +02007XXDPROG = ../xxd/xxd
Bram Moolenaar3f12a242015-12-05 20:56:57 +01008# VIMPROG = ../gvim
9
Bram Moolenaarefa304d2014-04-01 14:08:28 +020010SCRIPTSOURCE = ../../runtime
Bram Moolenaar071d4272004-06-13 20:20:40 +000011
Bram Moolenaard8f27b32018-10-07 15:42:07 +020012# Comment out this line to see the verbose output of tests.
13#
14# Catches SwapExists to avoid hanging at the ATTENTION prompt.
Bram Moolenaar133c7372019-06-06 14:38:06 +020015REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
Bram Moolenaare76d7a62018-09-21 16:37:25 +020016
Bram Moolenaar0c77b7b2009-06-24 16:08:18 +000017# Uncomment this line to use valgrind for memory leaks and extra warnings.
18# The output goes into a file "valgrind.testN"
19# Vim should be compiled with EXITFREE to avoid false warnings.
20# This will make testing about 10 times as slow.
Bram Moolenaarc8548982019-02-18 17:40:18 +010021# VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=25 --log-file=valgrind.$*
Bram Moolenaar4d2fffc2007-08-14 15:29:16 +000022
Bram Moolenaar40bbcee2015-12-28 22:24:41 +010023default: nongui
Bram Moolenaar680eeca2010-10-20 17:44:42 +020024
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010025# The list of tests is common to all systems.
Bram Moolenaarec504012019-01-11 17:30:16 +010026# This defines NEW_TESTS, NEW_TESTS_RES, SCRIPTS_ALL, SCRIPTS_MORE* and
27# SCRIPTS_GUI.
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010028include Make_all.mak
Bram Moolenaar071d4272004-06-13 20:20:40 +000029
Bram Moolenaar43345542015-11-29 17:35:35 +010030
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010031SCRIPTS = $(SCRIPTS_ALL) \
32 $(SCRIPTS_MORE1) \
33 $(SCRIPTS_MORE2) \
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010034 $(SCRIPTS_MORE4)
Bram Moolenaar071d4272004-06-13 20:20:40 +000035
Bram Moolenaar46fd6bf2019-01-11 19:19:44 +010036# Explicit dependencies.
37test49.out: test49.vim
38
39test_options.res test_alot.res: opt_test.vim
40
Bram Moolenaarfda37292014-11-05 14:27:36 +010041SCRIPTS_BENCH = bench_re_freeze.out
42
Bram Moolenaar43345542015-11-29 17:35:35 +010043.SUFFIXES: .in .out .res .vim
Bram Moolenaar071d4272004-06-13 20:20:40 +000044
Bram Moolenaara6082432015-12-28 16:26:45 +010045nongui: nolog $(SCRIPTS_FIRST) $(SCRIPTS) newtests report
Bram Moolenaar071d4272004-06-13 20:20:40 +000046
Bram Moolenaara6082432015-12-28 16:26:45 +010047gui: nolog $(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) newtests report
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000048
Bram Moolenaarfda37292014-11-05 14:27:36 +010049benchmark: $(SCRIPTS_BENCH)
50
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000051report:
Bram Moolenaar18250e22019-06-06 14:28:54 +020052 # without the +eval feature test_result.log is a copy of test.log
53 @/bin/sh -c "if test -f test.log; \
54 then cp test.log test_result.log; \
55 else echo No failures reported > test_result.log; \
56 fi"
Bram Moolenaar7d232042019-11-05 22:37:20 +010057 $(VIMPROG) -u NONE $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL)
Bram Moolenaar071d4272004-06-13 20:20:40 +000058 @echo
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000059 @echo 'Test results:'
Bram Moolenaar9c0cec62019-06-06 13:38:15 +020060 @cat test_result.log
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000061 @/bin/sh -c "if test -f test.log; \
Bram Moolenaar9c0cec62019-06-06 13:38:15 +020062 then echo TEST FAILURE; exit 1; \
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000063 else echo ALL DONE; \
64 fi"
Bram Moolenaar071d4272004-06-13 20:20:40 +000065
Bram Moolenaarec504012019-01-11 17:30:16 +010066$(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) $(NEW_TESTS_RES): $(VIMPROG)
Bram Moolenaara6082432015-12-28 16:26:45 +010067
68# Must run test1 first to create small.vim.
Bram Moolenaarec504012019-01-11 17:30:16 +010069$(SCRIPTS) $(SCRIPTS_GUI) $(NEW_TESTS_RES): $(SCRIPTS_FIRST)
70
71
72# Execute an individual new style test, e.g.:
73# make test_largefile
74$(NEW_TESTS):
75 rm -f $@.res test.log messages
Bram Moolenaarba089302019-10-19 18:56:58 +020076 @MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile $@.res
77 @cat messages
Bram Moolenaar4e0bf842019-06-07 22:40:25 +020078 @if test -f test.log; then \
79 exit 1; \
80 fi
Bram Moolenaar071d4272004-06-13 20:20:40 +000081
Bram Moolenaar60ec5f32012-12-05 18:21:32 +010082RM_ON_RUN = test.out X* viminfo
Bram Moolenaar1ce9a152018-07-14 21:48:46 +020083RM_ON_START = tiny.vim small.vim mbyte.vim mzscheme.vim test.ok benchmark.out
Bram Moolenaar41a82602019-07-14 21:54:26 +020084RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE) $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in
Bram Moolenaar60ec5f32012-12-05 18:21:32 +010085
Bram Moolenaard9b0d832019-09-14 15:46:32 +020086# Delete files that may interfere with running tests. This includes some files
87# that may result from working on the tests, not only from running them.
Bram Moolenaar071d4272004-06-13 20:20:40 +000088clean:
Bram Moolenaar4f5b0e52019-07-05 20:43:52 +020089 -rm -rf *.out *.failed *.res *.rej *.orig XfakeHOME Xdir1 Xfind
Bram Moolenaar8ed75cb2019-06-09 14:39:13 +020090 -rm -f opt_test.vim test.log test_result.log messages
91 -rm -f $(RM_ON_RUN) $(RM_ON_START)
92 -rm -f valgrind.*
Bram Moolenaar071d4272004-06-13 20:20:40 +000093
94test1.out: test1.in
Bram Moolenaar6848c8b2013-09-22 15:03:38 +020095 -rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize
Bram Moolenaare76d7a62018-09-21 16:37:25 +020096 $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL)
Bram Moolenaarf7dc2b52014-03-12 15:50:22 +010097 @/bin/sh -c "if test -f wrongtermsize; \
Bram Moolenaar6848c8b2013-09-22 15:03:38 +020098 then echo; \
99 echo test1 FAILED - terminal size must be 80x24 or larger; \
100 echo; exit 1; \
101 elif diff test.out $*.ok; \
Bram Moolenaar071d4272004-06-13 20:20:40 +0000102 then mv -f test.out $*.out; \
103 else echo; \
104 echo test1 FAILED - Something basic is wrong; \
105 echo; exit 1; fi"
106 -rm -rf X* viminfo
107
108.in.out:
Bram Moolenaar60ec5f32012-12-05 18:21:32 +0100109 -rm -rf $*.failed test.ok $(RM_ON_RUN)
Bram Moolenaar071d4272004-06-13 20:20:40 +0000110 cp $*.ok test.ok
Bram Moolenaar3f991522013-04-24 12:56:19 +0200111 # Sleep a moment to avoid that the xterm title is messed up.
112 # 200 msec is sufficient, but only modern sleep supports a fraction of
113 # a second, fall back to a second if it fails.
114 @-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1"
Bram Moolenaare76d7a62018-09-21 16:37:25 +0200115 $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL)
Bram Moolenaar60ec5f32012-12-05 18:21:32 +0100116
Bram Moolenaar89eaa412016-07-31 14:17:27 +0200117 # For flaky tests retry one time. No tests at the moment.
118 #@/bin/sh -c "if test -f test.out -a $* = test61; then \
119 # if diff test.out $*.ok; \
120 # then echo flaky test ok first time; \
121 # else rm -rf $*.failed $(RM_ON_RUN); \
122 # $(RUN_VIM) $*.in; \
123 # fi \
124 # fi"
Bram Moolenaar60ec5f32012-12-05 18:21:32 +0100125
126 # Check if the test.out file matches test.ok.
Bram Moolenaarfbc0d2e2013-05-19 19:40:29 +0200127 @/bin/sh -c "if test -f test.out; then \
Bram Moolenaar5313dcb2005-02-22 08:56:13 +0000128 if diff test.out $*.ok; \
129 then mv -f test.out $*.out; \
130 else echo $* FAILED >>test.log; mv -f test.out $*.failed; \
131 fi \
132 else echo $* NO OUTPUT >>test.log; \
Bram Moolenaar071d4272004-06-13 20:20:40 +0000133 fi"
Bram Moolenaarfbc0d2e2013-05-19 19:40:29 +0200134 @/bin/sh -c "if test -f valgrind; then\
135 mv -f valgrind valgrind.$*; \
136 fi"
Bram Moolenaarb35c3382012-10-06 19:10:35 +0200137 -rm -rf X* test.ok viminfo
Bram Moolenaar071d4272004-06-13 20:20:40 +0000138
Bram Moolenaarfda37292014-11-05 14:27:36 +0100139bench_re_freeze.out: bench_re_freeze.vim
140 -rm -rf benchmark.out $(RM_ON_RUN)
141 # Sleep a moment to avoid that the xterm title is messed up.
142 # 200 msec is sufficient, but only modern sleep supports a fraction of
143 # a second, fall back to a second if it fails.
144 @-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1"
Bram Moolenaare76d7a62018-09-21 16:37:25 +0200145 $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL)
Bram Moolenaarfda37292014-11-05 14:27:36 +0100146 @/bin/sh -c "if test -f benchmark.out; then cat benchmark.out; fi"
147
Bram Moolenaar071d4272004-06-13 20:20:40 +0000148nolog:
Bram Moolenaar096c8bb2015-12-29 14:26:57 +0100149 -rm -f test.log messages
Bram Moolenaar43345542015-11-29 17:35:35 +0100150
151
152# New style of tests uses Vim script with assert calls. These are easier
153# to write and a lot easier to read and debug.
154# Limitation: Only works with the +eval feature.
Bram Moolenaar41a82602019-07-14 21:54:26 +0200155RUN_VIMTEST = VIMRUNTIME=$(SCRIPTSOURCE) $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim
Bram Moolenaar43345542015-11-29 17:35:35 +0100156
Bram Moolenaare7893a42016-01-06 21:23:56 +0100157newtests: newtestssilent
Bram Moolenaar933bef72018-09-20 21:39:33 +0200158 @/bin/sh -c "if test -f messages && grep -q 'SKIPPED\|FAILED' messages; then cat messages; fi"
Bram Moolenaare7893a42016-01-06 21:23:56 +0100159
Bram Moolenaarec504012019-01-11 17:30:16 +0100160newtestssilent: $(NEW_TESTS_RES)
Bram Moolenaare7893a42016-01-06 21:23:56 +0100161
Bram Moolenaar43345542015-11-29 17:35:35 +0100162
Bram Moolenaar14170312019-02-21 21:50:46 +0100163.vim.res:
Bram Moolenaar63182052017-10-07 20:03:23 +0200164 @echo "$(VIMPROG)" > vimcmd
165 @echo "$(RUN_VIMTEST)" >> vimcmd
Bram Moolenaare76d7a62018-09-21 16:37:25 +0200166 $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL)
Bram Moolenaar8be2fbb2017-02-23 19:32:47 +0100167 @rm vimcmd
168
169test_gui.res: test_gui.vim
Bram Moolenaar63182052017-10-07 20:03:23 +0200170 @echo "$(VIMPROG)" > vimcmd
171 @echo "$(RUN_GVIMTEST)" >> vimcmd
Bram Moolenaar8be2fbb2017-02-23 19:32:47 +0100172 $(RUN_VIMTEST) -u NONE $(NO_INITS) -S runtest.vim $<
173 @rm vimcmd
174
175test_gui_init.res: test_gui_init.vim
Bram Moolenaar63182052017-10-07 20:03:23 +0200176 @echo "$(VIMPROG)" > vimcmd
177 @echo "$(RUN_GVIMTEST_WITH_GVIMRC)" >> vimcmd
Bram Moolenaar87748452017-03-12 17:10:33 +0100178 $(RUN_VIMTEST) -u gui_preinit.vim -U gui_init.vim $(NO_PLUGINS) -S runtest.vim $<
Bram Moolenaar66459b72016-08-06 19:01:55 +0200179 @rm vimcmd
Bram Moolenaar65408f72017-03-07 21:31:27 +0100180
Bram Moolenaard9b0d832019-09-14 15:46:32 +0200181opt_test.vim: ../optiondefs.h gen_opt_test.vim
182 $(VIMPROG) -u NONE -S gen_opt_test.vim --noplugin --not-a-term ../optiondefs.h
Bram Moolenaar29f9ed22018-04-10 19:20:31 +0200183
184test_xxd.res:
185 XXD=$(XXDPROG); export XXD; $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim test_xxd.vim