diff --git a/src/window.c b/src/window.c
index 27ef9d5..74e3c04 100644
--- a/src/window.c
+++ b/src/window.c
@@ -2107,17 +2107,22 @@
     static void
 leaving_window(win_T *win)
 {
+    // Only matters for a prompt window.
+    if (!bt_prompt(win->w_buffer))
+	return;
+
     // When leaving a prompt window stop Insert mode and perhaps restart
     // it when entering that window again.
     win->w_buffer->b_prompt_insert = restart_edit;
     restart_edit = NUL;
 
     // When leaving the window (or closing the window) was done from a
-    // callback we need to break out of the Insert mode loop.
+    // callback we need to break out of the Insert mode loop and restart Insert
+    // mode when entering the window again.
     if (State & INSERT)
     {
 	stop_insert_mode = TRUE;
-	if (bt_prompt(win->w_buffer) && win->w_buffer->b_prompt_insert == NUL)
+	if (win->w_buffer->b_prompt_insert == NUL)
 	    win->w_buffer->b_prompt_insert = 'A';
     }
 }
@@ -2125,12 +2130,17 @@
     static void
 entering_window(win_T *win)
 {
+    // Only matters for a prompt window.
+    if (!bt_prompt(win->w_buffer))
+	return;
+
     // When switching to a prompt buffer that was in Insert mode, don't stop
     // Insert mode, it may have been set in leaving_window().
-    if (bt_prompt(win->w_buffer) && win->w_buffer->b_prompt_insert != NUL)
+    if (win->w_buffer->b_prompt_insert != NUL)
 	stop_insert_mode = FALSE;
 
-    // When entering the prompt window may restart Insert mode.
+    // When entering the prompt window restart Insert mode if we were in Insert
+    // mode when we left it.
     restart_edit = win->w_buffer->b_prompt_insert;
 }
 #endif