patch 8.1.1476: no statistics displayed after running tests

Problem:    No statistics displayed after running tests.
Solution:   Summarize the test results. (Christian Brabandt, closes #4391)
            Also make it possible to report a skipped file.

diff --git a/src/Makefile b/src/Makefile
index 1347499eb11afa89c4152723eb580949fb3f1f2e..65398d01841495650b8b84954e66ed5b0138389a 100644 (file)
--- a/src/Makefile
+++ b/src/Makefile
@@ -2117,7 +2117,8 @@ types.vim: $(TAGS_FILES)
 # TESTING
 #
 # Execute the test scripts and the unittests.
-test check: scripttests unittests test_libvterm
+# Do the scripttests first, so that the summary shows last.
+test check: unittests test_libvterm scripttests
 
 # Execute the test scripts.  Run these after compiling Vim, before installing.
 # This doesn't depend on $(VIMTARGET), because that won't work when configure

diff --git a/src/testdir/Makefile b/src/testdir/Makefile
index 45c520d2df2175a01a68192ecfc10eca06bd462e..0abf66137dc9a16d37cf113b56f51c10ea99b79a 100644 (file)
--- a/src/testdir/Makefile
+++ b/src/testdir/Makefile
@@ -12,7 +12,7 @@ SCRIPTSOURCE = ../../runtime
 # Comment out this line to see the verbose output of tests.
 #
 # Catches SwapExists to avoid hanging at the ATTENTION prompt.
-REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
+# REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
 
 # Uncomment this line to use valgrind for memory leaks and extra warnings.
 #   The output goes into a file "valgrind.testN"
@@ -49,10 +49,12 @@ gui:        nolog $(SCRIPTS_FIRST) $(SCRIPTS) $(SCRIPTS_GUI) newtests report
 benchmark: $(SCRIPTS_BENCH)
 
 report:
+       $(RUN_VIMTEST) $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL)
        @echo
        @echo 'Test results:'
+       @cat test_result.log
        @/bin/sh -c "if test -f test.log; \
-               then cat test.log; echo TEST FAILURE; exit 1; \
+               then echo TEST FAILURE; exit 1; \
                else echo ALL DONE; \
                fi"
 
@@ -77,7 +79,10 @@ RM_ON_START = tiny.vim small.vim mbyte.vim mzscheme.vim test.ok benchmark.out
 RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in
 
 clean:
-       -rm -rf *.out *.failed *.res *.rej *.orig opt_test.vim test.log messages $(RM_ON_RUN) $(RM_ON_START) valgrind.*
+       -rm -rf *.out *.failed *.res *.rej *.orig
+       -rm opt_test.vim test.log test_result.log messages
+       -rm $(RM_ON_RUN) $(RM_ON_START)
+       -rm valgrind.*
 
 test1.out: test1.in
        -rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize

diff --git a/src/testdir/runtest.vim b/src/testdir/runtest.vim
index 2a1b5ea5cc238927a14e8fc82b2efb3e5ba9ae6f..02d4d20e340a38491c221a6a50cb023031479c81 100644 (file)
--- a/src/testdir/runtest.vim
+++ b/src/testdir/runtest.vim
@@ -268,6 +268,9 @@ if expand('%') =~ 'test_vimscript.vim'
 else
   try
     source %
+  catch /^\cskipped/
+    call add(s:messages, '    Skipped')
+    call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '',  ''))
   catch
     let s:fail += 1
     call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint)

diff --git a/src/testdir/summarize.vim b/src/testdir/summarize.vim
new file mode 100644 (file)
index 0000000..43dbce0
--- /dev/null
+++ b/src/testdir/summarize.vim
@@ -0,0 +1,60 @@
+if 1
+  " This is executed with the eval feature
+  set nocp
+  func Count(match, type)
+    if a:type ==# 'executed'
+      let g:executed += (a:match+0)
+    elseif a:type ==# 'failed'
+      let g:failed = a:match+0
+    elseif a:type ==# 'skipped'
+      let g:skipped += 1
+      call extend(g:skipped_output, ["\t".a:match])
+    endif
+  endfunc
+
+  let g:executed = 0
+  let g:skipped = 0
+  let g:failed = 0
+  let g:skipped_output = []
+  let g:failed_output = []
+  let output = [""]
+
+  try
+    " This uses the :s command to just fetch and process the output of the
+    " tests, it doesn't acutally replay anything
+    %s/^Executed\s\+\zs\d\+\ze\s\+tests/\=Count(submatch(0),'executed')/egn
+    %s/^SKIPPED \zs.*/\=Count(submatch(0), 'skipped')/egn
+    %s/^\(\d\+\)\s\+FAILED:/\=Count(submatch(1), 'failed')/egn
+
+    call extend(output, ["Skipped:"]) 
+    call extend(output, skipped_output)
+
+    call extend(output, [
+          \ "",
+          \ "-------------------------------",
+          \ printf("Executed: %5d Tests", g:executed),
+          \ printf(" Skipped: %5d Tests", g:skipped),
+          \ printf("  %s: %5d Tests", g:failed == 0 ? 'Failed' : 'FAILED', g:failed),
+          \ "",
+          \ ]) 
+    if filereadable('test.log')
+      " outputs and indents the failed test result
+      call extend(output, ["", "Failures: "])
+      let failed_output = filter(readfile('test.log'), { v,k -> !empty(k)})
+      call extend(output, map(failed_output, { v,k -> "\t".k}))
+      " Add a final newline
+      call extend(output, [""])
+    endif
+
+  catch  " Catch-all
+  finally
+    call writefile(output, 'test_result.log')  " overwrites an existing file
+    q!
+  endtry
+endif
+
+" This is executed without the eval feature
+%d
+r test.log
+w test_result.log
+q!

diff --git a/src/testdir/test_arabic.vim b/src/testdir/test_arabic.vim
index d0211fee228befc9943cd9b192773ed66deddcac..450c6f98f5a63867038374c68b368b3ff1dd190b 100644 (file)
--- a/src/testdir/test_arabic.vim
+++ b/src/testdir/test_arabic.vim
@@ -3,7 +3,7 @@
 " functional tests that check the shaping works with real text.
 
 if !has('arabic')
-  finish
+  throw 'Skipped: arabic feature missing'
 endif
 
 source view_util.vim

diff --git a/src/testdir/test_autochdir.vim b/src/testdir/test_autochdir.vim
index 90538fbc0997e7ffa4ee9c10dfd7f8f36668cc8c..1f99858efbaf4fa7dd33ef11859a96b2f2ca7873 100644 (file)
--- a/src/testdir/test_autochdir.vim
+++ b/src/testdir/test_autochdir.vim
@@ -1,7 +1,7 @@
 " Test 'autochdir' behavior
 
 if !exists("+autochdir")
-  finish
+  throw 'Skipped: autochdir feature missing'
 endif
 
 func Test_set_filename()

diff --git a/src/testdir/test_balloon.vim b/src/testdir/test_balloon.vim
index 39c2602be6a33d9f6524a6ec1cb16a12c5ad9fd6..37fc57eb5c277fc5787f4cc06cb049f3f7a970e4 100644 (file)
--- a/src/testdir/test_balloon.vim
+++ b/src/testdir/test_balloon.vim
@@ -1,11 +1,15 @@
 " Tests for 'balloonevalterm'.
 
-" Tests that only work in the terminal.
-if has('balloon_eval_term') && !has('gui_running')
+if !has('balloon_eval_term')
+  throw 'Skipped: balloon_eval_term feature missing'
+endif
+
+" A few tests only work in the terminal.
+if !has('gui_running')
 
 source screendump.vim
 if !CanRunVimInTerminal()
-  finish
+  throw 'Skipped: cannot run Vim in a terminal window'
 endif
 
 let s:common_script =<< [CODE]

diff --git a/src/version.c b/src/version.c
index 8f707b0263a7cc8806374dbae8c231e6c368142b..d0d2c28fc82a01c1a3648625e3842db68596735c 100644 (file)
--- a/src/version.c
+++ b/src/version.c
@@ -767,6 +767,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    1476,
 /**/
     1475,
 /**/