patch 9.1.0441: getregionpos() can't properly indicate positions beyond eol

Problem:  getregionpos() can't properly indicate positions beyond eol.
Solution: Add an "eol" flag that enables handling positions beyond end
          of line like getpos() does (zeertzjq).

Also fix the problem that a position still has the coladd beyond the end
of the line when its column has been clamped.  In the last test case
with TABs at the end of the line the old behavior is obviously wrong.

I decided to gate this behind a flag because returning positions that
don't correspond to actual characters in the line may lead to mistakes
for callers that want to calculate the length of the selected text, so
the behavior is only enabled if the caller wants it.

closes: #14838

Signed-off-by: zeertzjq <zeertzjq@outlook.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>
diff --git a/runtime/doc/builtin.txt b/runtime/doc/builtin.txt
index 4177a01..afd84c1 100644
--- a/runtime/doc/builtin.txt
+++ b/runtime/doc/builtin.txt
@@ -1,4 +1,4 @@
-*builtin.txt*	For Vim version 9.1.  Last change: 2024 May 22
+*builtin.txt*	For Vim version 9.1.  Last change: 2024 May 24
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -4348,6 +4348,19 @@
 		the offset of the character's first cell not included in the
 		selection, otherwise all its cells are included.
 
+		Apart from the options supported by |getregion()|, {opts} also
+		supports the following:
+
+			eol		If |TRUE|, indicate positions beyond
+					the end of a line with "col" values
+					one more than the length of the line.
+					If |FALSE|, positions are limited
+					within their lines, and if a line is
+					empty or the selection is entirely
+					beyond the end of a line, a "col"
+					value of 0 is used for both positions.
+					(default: |FALSE|)
+
 		Can also be used as a |method|: >
 			getpos('.')->getregionpos(getpos("'a"))
 <
diff --git a/src/evalfunc.c b/src/evalfunc.c
index f70b032..a1f8a8f 100644
--- a/src/evalfunc.c
+++ b/src/evalfunc.c
@@ -5695,7 +5695,6 @@
 add_regionpos_range(typval_T *rettv, pos_T p1, pos_T p2)
 {
     list_T	*l1, *l2, *l3;
-    int		max_col1, max_col2;
 
     l1 = list_alloc();
     if (l1 == NULL)
@@ -5737,16 +5736,14 @@
 	return;
     }
 
-    max_col1 = ml_get_len(p1.lnum);
     list_append_number(l2, curbuf->b_fnum);
     list_append_number(l2, p1.lnum);
-    list_append_number(l2, p1.col > max_col1 ? max_col1 : p1.col);
+    list_append_number(l2, p1.col);
     list_append_number(l2, p1.coladd);
 
-    max_col2 = ml_get_len(p2.lnum);
     list_append_number(l3, curbuf->b_fnum);
     list_append_number(l3, p2.lnum);
-    list_append_number(l3, p2.col > max_col2 ? max_col2 : p2.col);
+    list_append_number(l3, p2.col);
     list_append_number(l3, p2.coladd);
 }
 
@@ -5759,6 +5756,7 @@
     pos_T	p1, p2;
     int		inclusive = TRUE;
     int		region_type = -1;
+    int		allow_eol = FALSE;
     oparg_T	oa;
     int		lnum;
 
@@ -5772,9 +5770,13 @@
 		&p1, &p2, &inclusive, &region_type, &oa) == FAIL)
 	return;
 
+    if (argvars[2].v_type == VAR_DICT)
+	allow_eol = dict_get_bool(argvars[2].vval.v_dict, "eol", FALSE);
+
     for (lnum = p1.lnum; lnum <= p2.lnum; lnum++)
     {
-	pos_T			ret_p1, ret_p2;
+	pos_T		ret_p1, ret_p2;
+	colnr_T		line_len = ml_get_len(lnum);
 
 	if (region_type == MLINE)
 	{
@@ -5806,6 +5808,13 @@
 		    ret_p1.coladd = p1.coladd;
 		}
 	    }
+	    else if (region_type == MBLOCK && oa.start_vcol > bd.start_vcol)
+	    {
+		// blockwise selection entirely beyond end of line
+		ret_p1.col = MAXCOL;
+		ret_p1.coladd = oa.start_vcol - bd.start_vcol;
+		bd.is_oneChar = TRUE;
+	    }
 	    else if (bd.startspaces > 0)
 	    {
 		ret_p1.col = bd.textcol;
@@ -5820,7 +5829,7 @@
 	    if (bd.is_oneChar)  // selection entirely inside one char
 	    {
 		ret_p2.col = ret_p1.col;
-		ret_p2.coladd = ret_p1.coladd + bd.startspaces;
+		ret_p2.coladd = ret_p1.coladd + bd.startspaces + bd.endspaces;
 	    }
 	    else if (bd.endspaces > 0)
 	    {
@@ -5834,6 +5843,22 @@
 	    }
 	}
 
+	if (!allow_eol && ret_p1.col > line_len)
+	{
+	    ret_p1.col = 0;
+	    ret_p1.coladd = 0;
+	}
+	else if (ret_p1.col > line_len + 1)
+	    ret_p1.col = line_len + 1;
+
+	if (!allow_eol && ret_p2.col > line_len)
+	{
+	    ret_p2.col = ret_p1.col == 0 ? 0 : line_len;
+	    ret_p2.coladd = 0;
+	}
+	else if (ret_p2.col > line_len + 1)
+	    ret_p2.col = line_len + 1;
+
 	ret_p1.lnum = lnum;
 	ret_p2.lnum = lnum;
 	add_regionpos_range(rettv, ret_p1, ret_p2);
diff --git a/src/testdir/test_visual.vim b/src/testdir/test_visual.vim
index febf678..d1b9c71 100644
--- a/src/testdir/test_visual.vim
+++ b/src/testdir/test_visual.vim
@@ -1770,24 +1770,185 @@
     call feedkeys("\<ESC>Vjj", 'tx')
     call assert_equal(['one', 'two', 'three'],
           \ getregion(getpos('v'), getpos('.'), {'type': 'V' }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 0], [bufnr('%'), 1, 3, 0]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 3, 0]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': 'V' }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 0], [bufnr('%'), 1, 4, 0]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 4, 0]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 6, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': 'V', 'eol': v:true }))
 
     #" Multiline with block visual mode
     call cursor(1, 1)
     call feedkeys("\<ESC>\<C-v>jj", 'tx')
     call assert_equal(['o', 't', 't'],
           \ getregion(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 0], [bufnr('%'), 1, 1, 0]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 1, 0]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 1, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
 
     call cursor(1, 1)
     call feedkeys("\<ESC>\<C-v>jj$", 'tx')
     call assert_equal(['one', 'two', 'three'],
           \ getregion(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 0], [bufnr('%'), 1, 3, 0]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 3, 0]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 0], [bufnr('%'), 1, 3, 0]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 3, 0]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': "\<C-v>", 'eol': v:true }))
 
     #" 'virtualedit'
     set virtualedit=all
+
     call cursor(1, 1)
     call feedkeys("\<ESC>\<C-v>10ljj$", 'tx')
     call assert_equal(['one   ', 'two   ', 'three '],
           \ getregion(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 0], [bufnr('%'), 1, 3, 0]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 3, 0]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 0], [bufnr('%'), 1, 4, 3]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 4, 3]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 6, 1]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': "\<C-v>", 'eol': v:true }))
+
+    call cursor(3, 5)
+    call feedkeys("\<ESC>\<C-v>hkk", 'tx')
+    call assert_equal(['  ', '  ', 'ee'],
+          \ getregion(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 0, 0], [bufnr('%'), 1, 0, 0]],
+          \   [[bufnr('%'), 2, 0, 0], [bufnr('%'), 2, 0, 0]],
+          \   [[bufnr('%'), 3, 4, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 4, 0], [bufnr('%'), 1, 4, 2]],
+          \   [[bufnr('%'), 2, 4, 0], [bufnr('%'), 2, 4, 2]],
+          \   [[bufnr('%'), 3, 4, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': "\<C-v>", 'eol': v:true }))
+
+    call cursor(3, 5)
+    call feedkeys("\<ESC>\<C-v>kk", 'tx')
+    call assert_equal([' ', ' ', 'e'],
+          \ getregion(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 0, 0], [bufnr('%'), 1, 0, 0]],
+          \   [[bufnr('%'), 2, 0, 0], [bufnr('%'), 2, 0, 0]],
+          \   [[bufnr('%'), 3, 5, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 4, 1], [bufnr('%'), 1, 4, 2]],
+          \   [[bufnr('%'), 2, 4, 1], [bufnr('%'), 2, 4, 2]],
+          \   [[bufnr('%'), 3, 5, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': "\<C-v>", 'eol': v:true }))
+
+    call cursor(1, 3)
+    call feedkeys("\<ESC>vjj4l", 'tx')
+    call assert_equal(['e', 'two', 'three  '],
+          \ getregion(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 3, 0], [bufnr('%'), 1, 3, 0]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 3, 0]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 3, 0], [bufnr('%'), 1, 4, 0]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 4, 0]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 6, 2]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': 'v', 'eol': v:true }))
+
+    call cursor(1, 3)
+    call feedkeys("\<ESC>lvjj3l", 'tx')
+    call assert_equal(['', 'two', 'three  '],
+          \ getregion(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 0, 0], [bufnr('%'), 1, 0, 0]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 3, 0]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 4, 0], [bufnr('%'), 1, 4, 0]],
+          \   [[bufnr('%'), 2, 1, 0], [bufnr('%'), 2, 4, 0]],
+          \   [[bufnr('%'), 3, 1, 0], [bufnr('%'), 3, 6, 2]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': 'v', 'eol': v:true }))
+
+    call cursor(3, 5)
+    call feedkeys("\<ESC>v3l", 'tx')
+    call assert_equal(['e   '],
+          \ getregion(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 3, 5, 0], [bufnr('%'), 3, 5, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 3, 5, 0], [bufnr('%'), 3, 6, 3]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': 'v', 'eol': v:true }))
+
+    call cursor(3, 5)
+    call feedkeys("\<ESC>lv3l", 'tx')
+    call assert_equal(['    '],
+          \ getregion(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 3, 0, 0], [bufnr('%'), 3, 0, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 3, 6, 0], [bufnr('%'), 3, 6, 4]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': 'v', 'eol': v:true }))
+
+    call cursor(3, 5)
+    call feedkeys("\<ESC>3lv3l", 'tx')
+    call assert_equal(['    '],
+          \ getregion(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 3, 0, 0], [bufnr('%'), 3, 0, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 3, 6, 2], [bufnr('%'), 3, 6, 6]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': 'v', 'eol': v:true }))
+
     set virtualedit&
 
     #" using wrong types for positions
@@ -1863,11 +2024,10 @@
 
     exe $':{g:buf}bwipe!'
     unlet g:buf
+    bwipe!
   END
   call v9.CheckLegacyAndVim9Success(lines)
 
-  bwipe!
-
   let lines =<< trim END
     #" Selection in starts or ends in the middle of a multibyte character
     new
@@ -1987,12 +2147,12 @@
     call assert_equal(['abcdefghijk«'],
           \ getregion(getpos("'a"), getpos("'b"),
           \ {'type': 'V', 'exclusive': 1 }))
-    :set selection&
+
+    set selection&
+    bwipe!
   END
   call v9.CheckLegacyAndVim9Success(lines)
 
-  bwipe!
-
   let lines =<< trim END
     #" Exclusive selection
     new
@@ -2179,9 +2339,16 @@
     call assert_equal([
           \   [[bufnr('%'), 1, 2, 0], [bufnr('%'), 1, 2, 2]],
           \   [[bufnr('%'), 2, 2, 0], [bufnr('%'), 2, 2, 2]],
-          \   [[bufnr('%'), 3, 0, 0], [bufnr('%'), 3, 0, 2]],
+          \   [[bufnr('%'), 3, 0, 0], [bufnr('%'), 3, 0, 0]],
           \ ],
           \ getregionpos(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 2, 0], [bufnr('%'), 1, 2, 2]],
+          \   [[bufnr('%'), 2, 2, 0], [bufnr('%'), 2, 2, 2]],
+          \   [[bufnr('%'), 3, 1, 1], [bufnr('%'), 3, 1, 3]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': "\<C-v>", "eol": v:true }))
 
     call cursor(1, 1)
     call feedkeys("\<Esc>2l\<C-v>2l2j", 'xt')
@@ -2190,9 +2357,16 @@
     call assert_equal([
           \   [[bufnr('%'), 1, 2, 1], [bufnr('%'), 1, 2, 3]],
           \   [[bufnr('%'), 2, 2, 1], [bufnr('%'), 2, 2, 3]],
-          \   [[bufnr('%'), 3, 0, 0], [bufnr('%'), 3, 0, 2]],
+          \   [[bufnr('%'), 3, 0, 0], [bufnr('%'), 3, 0, 0]],
           \ ],
           \ getregionpos(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 2, 1], [bufnr('%'), 1, 2, 3]],
+          \   [[bufnr('%'), 2, 2, 1], [bufnr('%'), 2, 2, 3]],
+          \   [[bufnr('%'), 3, 1, 2], [bufnr('%'), 3, 1, 4]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': "\<C-v>", "eol": v:true }))
 
     #" 'virtualedit' with inclusive selection
     set selection&
@@ -2278,9 +2452,16 @@
     call assert_equal([
           \   [[bufnr('%'), 1, 2, 0], [bufnr('%'), 1, 2, 3]],
           \   [[bufnr('%'), 2, 2, 0], [bufnr('%'), 2, 2, 3]],
-          \   [[bufnr('%'), 3, 0, 0], [bufnr('%'), 3, 0, 3]],
+          \   [[bufnr('%'), 3, 0, 0], [bufnr('%'), 3, 0, 0]],
           \ ],
           \ getregionpos(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 2, 0], [bufnr('%'), 1, 2, 3]],
+          \   [[bufnr('%'), 2, 2, 0], [bufnr('%'), 2, 2, 3]],
+          \   [[bufnr('%'), 3, 1, 1], [bufnr('%'), 3, 1, 4]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': "\<C-v>", "eol": v:true }))
 
     call cursor(1, 1)
     call feedkeys("\<Esc>2l\<C-v>2l2j", 'xt')
@@ -2289,9 +2470,57 @@
     call assert_equal([
           \   [[bufnr('%'), 1, 2, 1], [bufnr('%'), 1, 2, 4]],
           \   [[bufnr('%'), 2, 2, 1], [bufnr('%'), 2, 2, 4]],
-          \   [[bufnr('%'), 3, 0, 0], [bufnr('%'), 3, 0, 3]],
+          \   [[bufnr('%'), 3, 0, 0], [bufnr('%'), 3, 0, 0]],
           \ ],
           \ getregionpos(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 2, 1], [bufnr('%'), 1, 2, 4]],
+          \   [[bufnr('%'), 2, 2, 1], [bufnr('%'), 2, 2, 4]],
+          \   [[bufnr('%'), 3, 1, 2], [bufnr('%'), 3, 1, 5]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': "\<C-v>", "eol": v:true }))
+
+    set virtualedit&
+    bwipe!
+  END
+  call v9.CheckLegacyAndVim9Success(lines)
+
+  let lines =<< trim END
+    #" 'virtualedit' with TABs at end of line
+    new
+    set virtualedit=all
+    call setline(1, ["\t", "a\t", "aa\t"])
+
+    call feedkeys("gg06l\<C-v>3l2j", 'xt')
+    call assert_equal(['    ', '    ', '    '],
+          \ getregion(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 6], [bufnr('%'), 1, 1, 0]],
+          \   [[bufnr('%'), 2, 2, 5], [bufnr('%'), 2, 2, 0]],
+          \   [[bufnr('%'), 3, 3, 4], [bufnr('%'), 3, 3, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': "\<C-v>" }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 6], [bufnr('%'), 1, 2, 2]],
+          \   [[bufnr('%'), 2, 2, 5], [bufnr('%'), 2, 3, 2]],
+          \   [[bufnr('%'), 3, 3, 4], [bufnr('%'), 3, 4, 2]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': "\<C-v>", "eol": v:true }))
+
+    call feedkeys("gg06lv3l", 'xt')
+    call assert_equal(['    '],
+          \ getregion(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 6], [bufnr('%'), 1, 1, 0]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'), {'type': 'v' }))
+    call assert_equal([
+          \   [[bufnr('%'), 1, 1, 6], [bufnr('%'), 1, 2, 2]],
+          \ ],
+          \ getregionpos(getpos('v'), getpos('.'),
+          \              {'type': 'v', "eol": v:true }))
 
     set virtualedit&
     bwipe!
diff --git a/src/version.c b/src/version.c
index f05081a..46cdb05 100644
--- a/src/version.c
+++ b/src/version.c
@@ -705,6 +705,8 @@
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    441,
+/**/
     440,
 /**/
     439,