patch 9.1.0404: [security] xxd: buffer-overflow with specific flags

Problem:  [security] xxd: buffer-overflow with specific flags
Solution: Correctly calculate the required buffer space
          (Lennard Hofmann)

xxd writes each output line into a global buffer before printing.
The maximum size of that buffer was not calculated correctly.

This command was crashing in AddressSanitizer:
$ xxd -Ralways -g1 -c256 -d -o 9223372036854775808 /etc/passwd

This prints a line of 6680 bytes but the buffer only had room for 6549 bytes.
If the output from "-b" was colored, the line could be even longer.

closes: #14738

Co-authored-by: K.Takata <kentkt@csc.jp>
Signed-off-by: Lennard Hofmann <lennard.hofmann@web.de>
Signed-off-by: Christian Brabandt <cb@256bit.org>
diff --git a/src/testdir/test_xxd.vim b/src/testdir/test_xxd.vim
index 7a2771e..642f5e7 100644
--- a/src/testdir/test_xxd.vim
+++ b/src/testdir/test_xxd.vim
@@ -411,6 +411,19 @@
   endfor
 endfunc
 
+
+" Try to trigger a buffer overflow (#14738)
+func Test_xxd_buffer_overflow()
+  CheckUnix
+  new
+  let input = repeat('A', 256)
+  call writefile(['-9223372036854775808: ' . repeat("\e[1;32m41\e[0m ", 256) . ' ' . repeat("\e[1;32mA\e[0m", 256)], 'Xxdexpected', 'D')
+  exe 'r! printf ' . input . '| ' . s:xxd_cmd . ' -Ralways -g1 -c256 -d -o 9223372036854775808 > Xxdout'
+  call assert_equalfile('Xxdexpected', 'Xxdout')
+  call delete('Xxdout')
+  bwipe!
+endfunc
+
 " -c0 selects the format specific default column value, as if no -c was given
 " except for -ps, where it disables extra newlines
 func Test_xxd_c0_is_def_cols()