blob: 54a998961f1fd86351b2d6ad5db9c78c54009496 [file] [log] [blame]
Bram Moolenaar071d4272004-06-13 20:20:40 +00001#
Bram Moolenaar6ee10162007-07-26 20:58:42 +00002# Makefile to run all tests for Vim
Bram Moolenaar071d4272004-06-13 20:20:40 +00003#
4
Bram Moolenaar3f12a242015-12-05 20:56:57 +01005# Use console or GUI.
Bram Moolenaar071d4272004-06-13 20:20:40 +00006VIMPROG = ../vim
Bram Moolenaar29f9ed22018-04-10 19:20:31 +02007XXDPROG = ../xxd/xxd
Bram Moolenaar3f12a242015-12-05 20:56:57 +01008# VIMPROG = ../gvim
9
Bram Moolenaarefa304d2014-04-01 14:08:28 +020010SCRIPTSOURCE = ../../runtime
Bram Moolenaar071d4272004-06-13 20:20:40 +000011
Bram Moolenaard8f27b32018-10-07 15:42:07 +020012# Comment out this line to see the verbose output of tests.
13#
14# Catches SwapExists to avoid hanging at the ATTENTION prompt.
Bram Moolenaardaaa3d92022-09-22 15:13:00 +010015REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' | LC_ALL=C LANG=C LANGUAGE=C awk '/Executing Test_/{match($$0, "(took .*; now )?Executing Test_[^\\)]*\\)"); print substr($$0, RSTART, RLENGTH) "\r"; fflush()}'
Bram Moolenaare76d7a62018-09-21 16:37:25 +020016
Bram Moolenaar0c77b7b2009-06-24 16:08:18 +000017# Uncomment this line to use valgrind for memory leaks and extra warnings.
18# The output goes into a file "valgrind.testN"
19# Vim should be compiled with EXITFREE to avoid false warnings.
20# This will make testing about 10 times as slow.
shane.xb.qian5a8fad32022-05-16 11:14:09 +010021# VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=35 --log-file=valgrind.$*
Bram Moolenaar4d2fffc2007-08-14 15:29:16 +000022
Bram Moolenaardaaa3d92022-09-22 15:13:00 +010023# To execute one test, e.g. in gdb, use something like this:
24# run -f -u unix.vim --gui-dialog-file guidialog -U NONE --noplugin --not-a-term -S runtest.vim test_name.vim
25
Bram Moolenaar40bbcee2015-12-28 22:24:41 +010026default: nongui
Bram Moolenaar680eeca2010-10-20 17:44:42 +020027
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010028# The list of tests is common to all systems.
Bram Moolenaarb96a32e2020-08-13 18:59:55 +020029# This defines SCRIPTS_TINY_OUT, NEW_TESTS and NEW_TESTS_RES.
Bram Moolenaar7b6156f2015-12-28 16:01:26 +010030include Make_all.mak
Bram Moolenaar071d4272004-06-13 20:20:40 +000031
Bram Moolenaar46fd6bf2019-01-11 19:19:44 +010032# Explicit dependencies.
Bram Moolenaar46fd6bf2019-01-11 19:19:44 +010033test_options.res test_alot.res: opt_test.vim
34
Bram Moolenaar43345542015-11-29 17:35:35 +010035.SUFFIXES: .in .out .res .vim
Bram Moolenaar071d4272004-06-13 20:20:40 +000036
Bram Moolenaarb96a32e2020-08-13 18:59:55 +020037nongui: nolog tinytests newtests report
Bram Moolenaar071d4272004-06-13 20:20:40 +000038
Bram Moolenaarb96a32e2020-08-13 18:59:55 +020039gui: nolog tinytests newtests report
40
41tiny: nolog tinytests report
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000042
Bram Moolenaarfda37292014-11-05 14:27:36 +010043benchmark: $(SCRIPTS_BENCH)
44
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000045report:
K.Takatac5125992021-05-26 21:49:18 +020046 @# without the +eval feature test_result.log is a copy of test.log
Bram Moolenaar18250e22019-06-06 14:28:54 +020047 @/bin/sh -c "if test -f test.log; \
48 then cp test.log test_result.log; \
49 else echo No failures reported > test_result.log; \
50 fi"
Bram Moolenaar7d232042019-11-05 22:37:20 +010051 $(VIMPROG) -u NONE $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL)
Bram Moolenaar071d4272004-06-13 20:20:40 +000052 @echo
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000053 @echo 'Test results:'
Bram Moolenaar9c0cec62019-06-06 13:38:15 +020054 @cat test_result.log
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000055 @/bin/sh -c "if test -f test.log; \
Bram Moolenaar9c0cec62019-06-06 13:38:15 +020056 then echo TEST FAILURE; exit 1; \
Bram Moolenaarf2e53da2008-09-10 16:26:04 +000057 else echo ALL DONE; \
58 fi"
Bram Moolenaar071d4272004-06-13 20:20:40 +000059
Bram Moolenaarb96a32e2020-08-13 18:59:55 +020060$(SCRIPTS_TINY_OUT) $(NEW_TESTS_RES): $(VIMPROG)
Bram Moolenaarec504012019-01-11 17:30:16 +010061
62
63# Execute an individual new style test, e.g.:
64# make test_largefile
65$(NEW_TESTS):
66 rm -f $@.res test.log messages
Bram Moolenaar85683ec2020-02-20 22:35:02 +010067 @MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile $@.res VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE)
Bram Moolenaarba089302019-10-19 18:56:58 +020068 @cat messages
Bram Moolenaar4e0bf842019-06-07 22:40:25 +020069 @if test -f test.log; then \
70 exit 1; \
71 fi
Bram Moolenaar071d4272004-06-13 20:20:40 +000072
Bram Moolenaarec5929d2020-04-07 20:53:39 +020073# Run only tests specific for Vim9 script
74test_vim9:
75 rm -f test_vim9_*.res test.log messages
76 @MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile $(TEST_VIM9_RES) VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE)
77 @cat messages
78 @MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile report VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE)
79 @if test -f test.log; then \
80 exit 1; \
81 fi
82
Bram Moolenaar60ec5f32012-12-05 18:21:32 +010083RM_ON_RUN = test.out X* viminfo
Bram Moolenaarb96a32e2020-08-13 18:59:55 +020084RM_ON_START = test.ok benchmark.out
Bram Moolenaar41a82602019-07-14 21:54:26 +020085RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE) $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in
Bram Moolenaar60ec5f32012-12-05 18:21:32 +010086
Bram Moolenaard9b0d832019-09-14 15:46:32 +020087# Delete files that may interfere with running tests. This includes some files
88# that may result from working on the tests, not only from running them.
Bram Moolenaar071d4272004-06-13 20:20:40 +000089clean:
Bram Moolenaar4f5b0e52019-07-05 20:43:52 +020090 -rm -rf *.out *.failed *.res *.rej *.orig XfakeHOME Xdir1 Xfind
Bram Moolenaar8ed75cb2019-06-09 14:39:13 +020091 -rm -f opt_test.vim test.log test_result.log messages
shane.xb.qian5a8fad32022-05-16 11:14:09 +010092 -rm -rf $(RM_ON_RUN) $(RM_ON_START)
Bram Moolenaar8ed75cb2019-06-09 14:39:13 +020093 -rm -f valgrind.*
Bram Moolenaar95a467e2020-04-23 14:41:46 +020094 -rm -f asan.*
Bram Moolenaar2d12c252022-06-13 21:42:45 +010095 -rm -f guidialog guidialogfile
Bram Moolenaar071d4272004-06-13 20:20:40 +000096
Bram Moolenaar76381a52022-05-27 20:13:28 +010097# Delete the files produced by benchmarking, so they can run again.
98benchmarkclean:
99 rm -f $(SCRIPTS_BENCH)
100
Bram Moolenaar071d4272004-06-13 20:20:40 +0000101nolog:
Bram Moolenaarb96a32e2020-08-13 18:59:55 +0200102 -rm -f test.log test_result.log messages
103
104
105# Tiny tests. Works even without the +eval feature.
106tinytests: $(SCRIPTS_TINY_OUT)
107
108.in.out:
109 -rm -rf $*.failed test.ok $(RM_ON_RUN)
110 cp $*.ok test.ok
111 @# Sleep a moment to avoid that the xterm title is messed up.
112 @# 200 msec is sufficient, but only modern sleep supports a fraction of
113 @# a second, fall back to a second if it fails.
114 @-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1"
115 $(RUN_VIM) $*.in $(REDIR_TEST_TO_NULL)
116
117 @# Check if the test.out file matches test.ok.
118 @/bin/sh -c "if test -f test.out; then \
119 if diff test.out $*.ok; \
120 then mv -f test.out $*.out; \
121 else echo $* FAILED >>test.log; mv -f test.out $*.failed; \
122 fi \
123 else echo $* NO OUTPUT >>test.log; \
124 fi"
125 @/bin/sh -c "if test -f valgrind; then\
126 mv -f valgrind valgrind.$*; \
127 fi"
128 -rm -rf X* test.ok viminfo
Bram Moolenaar43345542015-11-29 17:35:35 +0100129
130
131# New style of tests uses Vim script with assert calls. These are easier
132# to write and a lot easier to read and debug.
133# Limitation: Only works with the +eval feature.
Bram Moolenaar2d12c252022-06-13 21:42:45 +0100134# Add --gui-dialog-file to avoid getting stuck in a dialog.
135RUN_VIMTEST = VIMRUNTIME=$(SCRIPTSOURCE) $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim --gui-dialog-file guidialog
Bram Moolenaar43345542015-11-29 17:35:35 +0100136
Bram Moolenaare7893a42016-01-06 21:23:56 +0100137newtests: newtestssilent
Bram Moolenaar6e3aeec2020-08-26 22:29:57 +0200138 @/bin/sh -c "if test -f messages; then cat messages; fi"
Bram Moolenaare7893a42016-01-06 21:23:56 +0100139
Bram Moolenaarec504012019-01-11 17:30:16 +0100140newtestssilent: $(NEW_TESTS_RES)
Bram Moolenaare7893a42016-01-06 21:23:56 +0100141
Bram Moolenaar43345542015-11-29 17:35:35 +0100142
Bram Moolenaar14170312019-02-21 21:50:46 +0100143.vim.res:
Bram Moolenaar63182052017-10-07 20:03:23 +0200144 @echo "$(VIMPROG)" > vimcmd
145 @echo "$(RUN_VIMTEST)" >> vimcmd
Bram Moolenaare76d7a62018-09-21 16:37:25 +0200146 $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL)
Bram Moolenaar8be2fbb2017-02-23 19:32:47 +0100147 @rm vimcmd
148
149test_gui.res: test_gui.vim
Bram Moolenaar63182052017-10-07 20:03:23 +0200150 @echo "$(VIMPROG)" > vimcmd
151 @echo "$(RUN_GVIMTEST)" >> vimcmd
Bram Moolenaar8be2fbb2017-02-23 19:32:47 +0100152 $(RUN_VIMTEST) -u NONE $(NO_INITS) -S runtest.vim $<
153 @rm vimcmd
154
155test_gui_init.res: test_gui_init.vim
Bram Moolenaar63182052017-10-07 20:03:23 +0200156 @echo "$(VIMPROG)" > vimcmd
157 @echo "$(RUN_GVIMTEST_WITH_GVIMRC)" >> vimcmd
Bram Moolenaar87748452017-03-12 17:10:33 +0100158 $(RUN_VIMTEST) -u gui_preinit.vim -U gui_init.vim $(NO_PLUGINS) -S runtest.vim $<
Bram Moolenaar66459b72016-08-06 19:01:55 +0200159 @rm vimcmd
Bram Moolenaar65408f72017-03-07 21:31:27 +0100160
Bram Moolenaard9b0d832019-09-14 15:46:32 +0200161opt_test.vim: ../optiondefs.h gen_opt_test.vim
162 $(VIMPROG) -u NONE -S gen_opt_test.vim --noplugin --not-a-term ../optiondefs.h
Bram Moolenaar29f9ed22018-04-10 19:20:31 +0200163
164test_xxd.res:
165 XXD=$(XXDPROG); export XXD; $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim test_xxd.vim
Bram Moolenaarad48e6c2020-04-21 22:19:45 +0200166
167test_bench_regexp.res: test_bench_regexp.vim
168 -rm -rf benchmark.out $(RM_ON_RUN)
Bram Moolenaarb96a32e2020-08-13 18:59:55 +0200169 @# Sleep a moment to avoid that the xterm title is messed up.
170 @# 200 msec is sufficient, but only modern sleep supports a fraction of
171 @# a second, fall back to a second if it fails.
Bram Moolenaarad48e6c2020-04-21 22:19:45 +0200172 @-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1"
173 $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL)
174 @/bin/sh -c "if test -f benchmark.out; then cat benchmark.out; fi"