patch 9.1.0681: tests: Analyzing failed screendumps is hard
Problem: tests: Analyzing failed screendumps is hard
Solution: Facilitate the viewing of rendered screendumps under src/
add some documentation on how to use the viewdumps.vim
script (Aliaksei Budavei)
With the submitted "viewdumps.vim" script, a few manual
steps in typical workflows (see below) can be automated.
The updated "README.txt" contains additional information.
============================================================
Reviewing LOCAL failed screendump tests can be arranged as
follows:
1) Run tests and generate screendumps:
------------------------------------------------------------
cd /path/to/fork/src/testdir
make
------------------------------------------------------------
2) Examine the screendumps from the "failed" directory:
------------------------------------------------------------
../vim -u NONE -S viewdumps.vim
------------------------------------------------------------
============================================================
Reviewing UPLOADED failed screendump tests can be arranged
as follows (it can be further locally scripted):
1) Fetch an artifact with failed screendumps from
"github.com/vim/vim/actions/runs/A_ID/artifacts/B_ID".
2) Extract the archived files:
------------------------------------------------------------
unzip /tmp/failed-tests.zip -d /tmp
------------------------------------------------------------
3) Set up the "dumps" directory. Create a symlink to
"/path/to/fork/dirs/dumps" in the extracted directories so
that term_dumpdiff() can be used. (The lookup algorithm
resolves "dumps" for every loaded filename. So, with
"/tmp/src/testdir/failed/*.dump" files passed as script
arguments, the algorithm will make the files in
"/tmp/src/testdir/dumps" queried.)
------------------------------------------------------------
cd /path/to/fork
ln -s $(pwd)/src/testdir/dumps /tmp/src/testdir/dumps
------------------------------------------------------------
4) Examine the extracted screendumps:
------------------------------------------------------------
./src/vim -u NONE -S src/testdir/viewdumps.vim \
/tmp/src/testdir/failed/*.dump
------------------------------------------------------------
5) Clean up:
------------------------------------------------------------
unlink /tmp/src/testdir/dumps
rm -rf /tmp/src
------------------------------------------------------------
============================================================
Reviewing SUBMITTED FOR PULL REQUEST screendump tests can be
arranged as follows (it can be further locally scripted):
1) List the fetched changeset and write the changed "dumps"
filenames to "/tmp/filelist":
------------------------------------------------------------
cd /path/to/fork
git switch prs/1234
git diff-index --relative=src/testdir/dumps/ \
--name-only prs/1234~1 > /tmp/filelist
------------------------------------------------------------
2) Reconcile relative filepaths, and copy next-to-be-updated
"dumps" files in the "failed" directory (note the missing
new screendumps, if any):
------------------------------------------------------------
git switch master
cd src/testdir/dumps
test -d ../failed || mkdir ../failed
cp -t ../failed $(cat /tmp/filelist)
------------------------------------------------------------
3) Remember about the introduced INVERTED relation between
"dumps" and "failed", i.e. the files to be committed are in
"dumps" already and their previous versions are in "failed";
therefore, copy the missing new screendumps from "dumps" to
"failed" (otherwise these won't be shown):
------------------------------------------------------------
git switch prs/1234
cp -t ../failed foo_10.dump foo_11.dump foo_12.dump
------------------------------------------------------------
4) Examine the screendumps from the "failed" directory (new
screendumps will be shown with no difference between their
versions):
------------------------------------------------------------
cd ..
../vim -u NONE -S viewdumps.vim
------------------------------------------------------------
closes: #15515
Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>
diff --git a/runtime/syntax/testdir/README.txt b/runtime/syntax/testdir/README.txt
index 3b1b9ae..3baa43b 100644
--- a/runtime/syntax/testdir/README.txt
+++ b/runtime/syntax/testdir/README.txt
@@ -124,8 +124,8 @@
test.
-Viewing generated screendumps
------------------------------
+Viewing generated screendumps (local)
+-------------------------------------
You may also wish to look at the whole batch of failed screendumps after
running "make test". Source the "viewdumps.vim" script for this task:
@@ -151,5 +151,60 @@
The listing of argument commands can be found under :help buffer-list.
+Viewing generated screendumps (from a CI-uploaded artifact)
+-----------------------------------------------------------
+
+After you have downloaded an artifact archive containing failed screendumps
+and extracted its files in a temporary directory, you need to set up a "dumps"
+directory by creating a symlink:
+
+ cd /path/to/fork
+ ln -s $(pwd)/runtime/syntax/testdir/dumps \
+ /tmp/runtime/syntax/testdir/dumps
+
+You can now examine the extracted screendumps:
+
+ ./src/vim --clean -S runtime/syntax/testdir/viewdumps.vim \
+ /tmp/runtime/syntax/testdir/failed/*.dump
+
+
+Viewing generated screendumps (submitted for a pull request)
+------------------------------------------------------------
+
+First, you need to check out the topic branch with the proposed changes and
+write down a difference list between the HEAD commit (index) and its parent
+commit with respect to the changed "dumps" filenames:
+
+ cd /path/to/fork
+ git switch prs/1234
+ git diff-index --relative=runtime/syntax/testdir/dumps/ \
+ --name-only prs/1234~1 > /tmp/filelist
+
+Then, you need to check out the master branch, change the current working
+directory to reconcile relative filepaths written in the filenames list, copy
+in the "failed" directory the old "dumps" files, whose names are on the same
+list, and follow it by checking out the topic branch:
+
+ git switch master
+ cd runtime/syntax/testdir/dumps
+ cp -t ../failed $(cat /tmp/filelist)
+ git switch prs/1234
+
+Make note of any missing new screendumps. Please remember about the
+introduced INVERTED relation between "dumps" and "failed", i.e. the files to
+be committed are in "dumps" already and their old versions are in "failed".
+Therefore, you need to copy the missing new screendumps from "dumps" to
+"failed":
+
+ cp -t ../failed foo_10.dump foo_11.dump foo_12.dump
+
+After you have changed the current working directory to its parent directory,
+you can now examine the screendumps from the "failed" directory (note that new
+screendumps will be shown with no difference between their versions):
+
+ cd ..
+ ../../../src/vim --clean -S viewdumps.vim
+
+
TODO: run test for one specific filetype
TODO: test syncing by jumping around
diff --git a/runtime/syntax/testdir/viewdumps.vim b/runtime/syntax/testdir/viewdumps.vim
index 0fbfade..e1624ff 100644
--- a/runtime/syntax/testdir/viewdumps.vim
+++ b/runtime/syntax/testdir/viewdumps.vim
@@ -1,44 +1,9 @@
vim9script
-# Support sourcing this script from this directory or any other directory in
-# the direct path that leads to the project's root directory.
-const failed_path: string = finddir('failed', getcwd() .. '/**', -1)
- ->filter(((cwdpath: string) => (_: number, dirpath: string) =>
- cwdpath =~ '\<syntax\>' || dirpath =~ '\<syntax\>')(getcwd()))
- ->get(-1, '') .. '/'
-var error: string = null_string
-
-if failed_path == '/'
- error = 'No such directory: "failed"'
-else
- const failed_fnames: string = failed_path .. readdir(failed_path,
- (fname: string) => fname =~ '^.\+\.dump$')
- ->join(' ' .. failed_path)
-
- if failed_fnames =~ 'failed/$'
- error = 'No such file: "*.dump"'
- else
- exec ':0argedit ' .. failed_fnames
- buffers
- endif
-endif
-
-def Render()
- const failed_fname: string = bufname()
- try
- setlocal suffixesadd=.dump
- const dumps_fname: string = findfile(
- fnamemodify(failed_fname, ':p:t'),
- fnamemodify(failed_fname, ':p:h:h') .. '/dumps')
- if filereadable(dumps_fname)
- term_dumpdiff(failed_fname, dumps_fname)
- else
- term_dumpload(failed_fname)
- endif
- finally
- exec 'bwipeout ' .. failed_fname
- endtry
-enddef
+exec 'source ' .. (((cwdpath: string) => cwdpath
+ ->strpart(0, cwdpath->strridx('/vim')))(getcwd()))
+ .. '/vim/src/testdir/commondumps.vim'
+g:Init('\<syntax\>', -1)
# THE FOLLOWING SETTINGS PERTAIN TO "input/" FILES THAT ARE LIKELY TO BE
# LOADED SIDE BY SIDE WHENEVER BATCHES OF NEW SCREENDUMPS ARE GENERATED.
@@ -56,22 +21,4 @@
# Anticipate non-Latin-1 characters in "input/" files.
set encoding=utf-8 termencoding=utf-8
-autocmd_add([{
- replace: true,
- group: 'viewdumps',
- event: 'BufRead',
- pattern: '*.dump',
- cmd: 'Render()',
-}])
-
-# Unconditionally help, in case a list of filenames is passed to the command,
-# the first terminal window with its BufRead event.
-silent doautocmd viewdumps BufRead
-
-if error != null_string
- # Instead of sleeping, fill half a window with blanks and prompt hit-enter.
- echom error .. repeat("\x20", (winwidth(0) * (winheight(0) / 2) - strlen(error)))
- error = null_string
-endif
-
# vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta: