diff --git a/src/testdir/Makefile b/src/testdir/Makefile
index 45c520d..0abf661 100644
--- a/src/testdir/Makefile
+++ b/src/testdir/Makefile
@@ -12,7 +12,7 @@
 # Comment out this line to see the verbose output of tests.
 #
 # Catches SwapExists to avoid hanging at the ATTENTION prompt.
-REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
+# REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
 
 # Uncomment this line to use valgrind for memory leaks and extra warnings.
 #   The output goes into a file "valgrind.testN"
@@ -49,10 +49,12 @@
 benchmark: $(SCRIPTS_BENCH)
 
 report:
+	$(RUN_VIMTEST) $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL)
 	@echo
 	@echo 'Test results:'
+	@cat test_result.log
 	@/bin/sh -c "if test -f test.log; \
-		then cat test.log; echo TEST FAILURE; exit 1; \
+		then echo TEST FAILURE; exit 1; \
 		else echo ALL DONE; \
 		fi"
 
@@ -77,7 +79,10 @@
 RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in
 
 clean:
-	-rm -rf *.out *.failed *.res *.rej *.orig opt_test.vim test.log messages $(RM_ON_RUN) $(RM_ON_START) valgrind.*
+	-rm -rf *.out *.failed *.res *.rej *.orig
+	-rm opt_test.vim test.log test_result.log messages
+	-rm $(RM_ON_RUN) $(RM_ON_START)
+	-rm valgrind.*
 
 test1.out: test1.in
 	-rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize
diff --git a/src/testdir/runtest.vim b/src/testdir/runtest.vim
index 2a1b5ea..02d4d20 100644
--- a/src/testdir/runtest.vim
+++ b/src/testdir/runtest.vim
@@ -268,6 +268,9 @@
 else
   try
     source %
+  catch /^\cskipped/
+    call add(s:messages, '    Skipped')
+    call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '',  ''))
   catch
     let s:fail += 1
     call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint)
diff --git a/src/testdir/summarize.vim b/src/testdir/summarize.vim
new file mode 100644
index 0000000..43dbce0
--- /dev/null
+++ b/src/testdir/summarize.vim
@@ -0,0 +1,60 @@
+if 1
+  " This is executed with the eval feature
+  set nocp
+  func Count(match, type)
+    if a:type ==# 'executed'
+      let g:executed += (a:match+0)
+    elseif a:type ==# 'failed'
+      let g:failed = a:match+0
+    elseif a:type ==# 'skipped'
+      let g:skipped += 1
+      call extend(g:skipped_output, ["\t".a:match])
+    endif
+  endfunc
+
+  let g:executed = 0
+  let g:skipped = 0
+  let g:failed = 0
+  let g:skipped_output = []
+  let g:failed_output = []
+  let output = [""]
+
+  try
+    " This uses the :s command to just fetch and process the output of the
+    " tests, it doesn't acutally replay anything
+    %s/^Executed\s\+\zs\d\+\ze\s\+tests/\=Count(submatch(0),'executed')/egn
+    %s/^SKIPPED \zs.*/\=Count(submatch(0), 'skipped')/egn
+    %s/^\(\d\+\)\s\+FAILED:/\=Count(submatch(1), 'failed')/egn
+
+    call extend(output, ["Skipped:"]) 
+    call extend(output, skipped_output)
+
+    call extend(output, [
+          \ "",
+          \ "-------------------------------",
+          \ printf("Executed: %5d Tests", g:executed),
+          \ printf(" Skipped: %5d Tests", g:skipped),
+          \ printf("  %s: %5d Tests", g:failed == 0 ? 'Failed' : 'FAILED', g:failed),
+          \ "",
+          \ ]) 
+    if filereadable('test.log')
+      " outputs and indents the failed test result
+      call extend(output, ["", "Failures: "])
+      let failed_output = filter(readfile('test.log'), { v,k -> !empty(k)})
+      call extend(output, map(failed_output, { v,k -> "\t".k}))
+      " Add a final newline
+      call extend(output, [""])
+    endif
+
+  catch  " Catch-all
+  finally
+    call writefile(output, 'test_result.log')  " overwrites an existing file
+    q!
+  endtry
+endif
+
+" This is executed without the eval feature
+%d
+r test.log
+w test_result.log
+q!
diff --git a/src/testdir/test_arabic.vim b/src/testdir/test_arabic.vim
index d0211fe..450c6f9 100644
--- a/src/testdir/test_arabic.vim
+++ b/src/testdir/test_arabic.vim
@@ -3,7 +3,7 @@
 " functional tests that check the shaping works with real text.
 
 if !has('arabic')
-  finish
+  throw 'Skipped: arabic feature missing'
 endif
 
 source view_util.vim
diff --git a/src/testdir/test_autochdir.vim b/src/testdir/test_autochdir.vim
index 90538fb..1f99858 100644
--- a/src/testdir/test_autochdir.vim
+++ b/src/testdir/test_autochdir.vim
@@ -1,7 +1,7 @@
 " Test 'autochdir' behavior
 
 if !exists("+autochdir")
-  finish
+  throw 'Skipped: autochdir feature missing'
 endif
 
 func Test_set_filename()
diff --git a/src/testdir/test_balloon.vim b/src/testdir/test_balloon.vim
index 39c2602..37fc57e 100644
--- a/src/testdir/test_balloon.vim
+++ b/src/testdir/test_balloon.vim
@@ -1,11 +1,15 @@
 " Tests for 'balloonevalterm'.
 
-" Tests that only work in the terminal.
-if has('balloon_eval_term') && !has('gui_running')
+if !has('balloon_eval_term')
+  throw 'Skipped: balloon_eval_term feature missing'
+endif
+
+" A few tests only work in the terminal.
+if !has('gui_running')
 
 source screendump.vim
 if !CanRunVimInTerminal()
-  finish
+  throw 'Skipped: cannot run Vim in a terminal window'
 endif
 
 let s:common_script =<< [CODE]