patch 9.1.0404: [security] xxd: buffer-overflow with specific flags

Problem:  [security] xxd: buffer-overflow with specific flags
Solution: Correctly calculate the required buffer space
          (Lennard Hofmann)

xxd writes each output line into a global buffer before printing.
The maximum size of that buffer was not calculated correctly.

This command was crashing in AddressSanitizer:
$ xxd -Ralways -g1 -c256 -d -o 9223372036854775808 /etc/passwd

This prints a line of 6680 bytes but the buffer only had room for 6549 bytes.
If the output from "-b" was colored, the line could be even longer.

closes: #14738

Co-authored-by: K.Takata <kentkt@csc.jp>
Signed-off-by: Lennard Hofmann <lennard.hofmann@web.de>
Signed-off-by: Christian Brabandt <cb@256bit.org>
diff --git a/src/testdir/test_xxd.vim b/src/testdir/test_xxd.vim
index 7a2771e..642f5e7 100644
--- a/src/testdir/test_xxd.vim
+++ b/src/testdir/test_xxd.vim
@@ -411,6 +411,19 @@
   endfor
 endfunc
 
+
+" Try to trigger a buffer overflow (#14738)
+func Test_xxd_buffer_overflow()
+  CheckUnix
+  new
+  let input = repeat('A', 256)
+  call writefile(['-9223372036854775808: ' . repeat("\e[1;32m41\e[0m ", 256) . ' ' . repeat("\e[1;32mA\e[0m", 256)], 'Xxdexpected', 'D')
+  exe 'r! printf ' . input . '| ' . s:xxd_cmd . ' -Ralways -g1 -c256 -d -o 9223372036854775808 > Xxdout'
+  call assert_equalfile('Xxdexpected', 'Xxdout')
+  call delete('Xxdout')
+  bwipe!
+endfunc
+
 " -c0 selects the format specific default column value, as if no -c was given
 " except for -ps, where it disables extra newlines
 func Test_xxd_c0_is_def_cols()
diff --git a/src/version.c b/src/version.c
index 954b747..6bb8be6 100644
--- a/src/version.c
+++ b/src/version.c
@@ -705,6 +705,8 @@
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    404,
+/**/
     403,
 /**/
     402,
diff --git a/src/xxd/xxd.c b/src/xxd/xxd.c
index cf8b4ea..7a3d36a 100644
--- a/src/xxd/xxd.c
+++ b/src/xxd/xxd.c
@@ -62,6 +62,7 @@
  * 17.01.2024  use size_t instead of usigned int for code-generation (-i), #13876
  * 25.01.2024  revert the previous patch (size_t instead of unsigned int)
  * 10.02.2024  fix buffer-overflow when writing color output to buffer, #14003
+ * 10.05.2024  fix another buffer-overflow when writing colored output to buffer, #14738
  *
  * (c) 1990-1998 by Juergen Weigert (jnweiger@gmail.com)
  *
@@ -142,7 +143,7 @@
 # endif
 #endif
 
-char version[] = "xxd 2024-02-10 by Juergen Weigert et al.";
+char version[] = "xxd 2024-05-10 by Juergen Weigert et al.";
 #ifdef WIN32
 char osver[] = " (Win32)";
 #else
@@ -205,29 +206,16 @@
 /*
  * LLEN is the maximum length of a line; other than the visible characters
  * we need to consider also the escape color sequence prologue/epilogue ,
- * (11 bytes for each character). The most larger format is the default one:
- * addr + 1 word for each col/2 + 1 char for each col
- *
- *   addr        1st group       2nd group
- * +-------+ +-----------------+ +------+
- * 01234567: 1234 5678 9abc def0 12345678
- *
- * - addr: typically 012345678:    -> from 10 up to 18 bytes (including trailing
- *                                    space)
- * - 1st group: 1234 5678 9abc ... -> each byte may be colored, so add 11
- *                                    for each byte
- * - space                         -> 1 byte
- * - 2nd group: 12345678           -> each char may be colore so add 11
- *                                    for each byte
- * - new line                      -> 1 byte
- * - zero (end line)               -> 1 byte
+ * (11 bytes for each character).
  */
-#define LLEN (2*(int)sizeof(unsigned long) + 2 +     /* addr + ": " */ \
-             (11 * 2 + 4 + 1) * (COLS / 2) +         /* 1st group */   \
-	     1 +                                     /* space */       \
-	     (1 + 11) * COLS +                       /* 2nd group */   \
-	     1 +                                     /* new line */    \
-	     1)                                      /* zero */
+#define LLEN \
+    (39            /* addr: ⌈log10(ULONG_MAX)⌉ if "-d" flag given. We assume ULONG_MAX = 2**128 */ \
+    + 2            /* ": " */ \
+    + 13 * COLS    /* hex dump with colors */ \
+    + (COLS - 1)   /* whitespace between groups if "-g1" option given and "-c" maxed out */ \
+    + 2            /* whitespace */ \
+    + 12 * COLS    /* ASCII dump with colors */ \
+    + 2)           /* "\n\0" */
 
 char hexxa[] = "0123456789abcdef0123456789ABCDEF", *hexx = hexxa;