diff --git a/src/testdir/test_prompt_buffer.vim b/src/testdir/test_prompt_buffer.vim
index 32ed21e..8918bd4 100644
--- a/src/testdir/test_prompt_buffer.vim
+++ b/src/testdir/test_prompt_buffer.vim
@@ -39,6 +39,10 @@
 	\ '  set nomodified',
 	\ 'endfunc',
 	\ '',
+	\ 'func SwitchWindows()',
+	\ '  call timer_start(0, {-> execute("wincmd p|wincmd p", "")})',
+	\ 'endfunc',
+	\ '',
 	\ 'call setline(1, "other buffer")',
 	\ 'set nomodified',
 	\ 'new',
@@ -99,6 +103,27 @@
   call delete(scriptName)
 endfunc
 
+func Test_prompt_switch_windows()
+  call CanTestPromptBuffer()
+  let scriptName = 'XpromptSwitchWindows'
+  call WriteScript(scriptName)
+
+  let buf = RunVimInTerminal('-S ' . scriptName, {'rows': 12})
+  call WaitForAssert({-> assert_equal('cmd:', term_getline(buf, 1))})
+  call WaitForAssert({-> assert_match('-- INSERT --', term_getline(buf, 12))})
+
+  call term_sendkeys(buf, "\<C-O>:call SwitchWindows()\<CR>")
+  call term_wait(buf, 50)
+  call WaitForAssert({-> assert_match('-- INSERT --', term_getline(buf, 12))})
+
+  call term_sendkeys(buf, "\<Esc>")
+  call term_wait(buf, 50)
+  call WaitForAssert({-> assert_match('^ *$', term_getline(buf, 12))})
+
+  call StopVimInTerminal(buf)
+  call delete(scriptName)
+endfunc
+
 func Test_prompt_garbage_collect()
   func MyPromptCallback(x, text)
     " NOP
diff --git a/src/version.c b/src/version.c
index 1918890..0686cfa 100644
--- a/src/version.c
+++ b/src/version.c
@@ -758,6 +758,8 @@
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    3671,
+/**/
     3670,
 /**/
     3669,
diff --git a/src/window.c b/src/window.c
index d28962a..4fbf944 100644
--- a/src/window.c
+++ b/src/window.c
@@ -2268,8 +2268,9 @@
 	stop_insert_mode = FALSE;
 
     // When entering the prompt window restart Insert mode if we were in Insert
-    // mode when we left it.
-    restart_edit = win->w_buffer->b_prompt_insert;
+    // mode when we left it and not already in Insert mode.
+    if ((State & INSERT) == 0)
+	restart_edit = win->w_buffer->b_prompt_insert;
 }
 #endif