patch 8.0.1758: open_line() returns TRUE/FALSE for success/failure

Problem:    open_line() returns TRUE/FALSE for success/failure.
Solution:   Return OK or FAIL.
diff --git a/src/edit.c b/src/edit.c
index 8746436..f29fbc7 100644
--- a/src/edit.c
+++ b/src/edit.c
@@ -1374,7 +1374,7 @@
 		goto doESCkey;
 	    }
 #endif
-	    if (ins_eol(c) && !p_im)
+	    if (ins_eol(c) == FAIL && !p_im)
 		goto doESCkey;	    /* out of memory */
 	    auto_format(FALSE, FALSE);
 	    inserted_space = FALSE;
@@ -10201,7 +10201,7 @@
 
 /*
  * Handle CR or NL in insert mode.
- * Return TRUE when out of memory or can't undo.
+ * Return FAIL when out of memory or can't undo.
  */
     static int
 ins_eol(int c)
@@ -10266,7 +10266,7 @@
     foldOpenCursor();
 #endif
 
-    return (!i);
+    return i;
 }
 
 #ifdef FEAT_DIGRAPHS