patch 9.0.1690: popup_create() not aborting on errors

Problem: popup_create() not aborting on errors
Solution: check for errors in arguments given and abort if an error
          occurred

closes: #12711

Signed-off-by: Christian Brabandt <cb@256bit.org>
diff --git a/src/popupwin.c b/src/popupwin.c
index a0d33fd..9f76cb1 100644
--- a/src/popupwin.c
+++ b/src/popupwin.c
@@ -92,19 +92,19 @@
     return n;
 }
 
-    static void
+    static int
 set_padding_border(dict_T *dict, int *array, char *name, int max_val)
 {
     dictitem_T	*di;
 
     di = dict_find(dict, (char_u *)name, -1);
     if (di == NULL)
-	return;
+	return OK;
 
     if (di->di_tv.v_type != VAR_LIST)
     {
 	emsg(_(e_list_required));
-	return;
+	return FAIL;
     }
 
     list_T	*list = di->di_tv.vval.v_list;
@@ -115,7 +115,7 @@
     for (i = 0; i < 4; ++i)
 	array[i] = 1;
     if (list == NULL)
-	return;
+	return OK;
 
     CHECK_LIST_MATERIALIZE(list);
     for (i = 0, li = list->lv_first; i < 4 && i < list->lv_len;
@@ -125,6 +125,8 @@
 	if (nr >= 0)
 	    array[i] = nr > max_val ? max_val : nr;
     }
+
+    return OK;
 }
 
 /*
@@ -713,7 +715,7 @@
 /*
  * Shared between popup_create() and f_popup_setoptions().
  */
-    static void
+    static int
 apply_general_options(win_T *wp, dict_T *dict)
 {
     dictitem_T	*di;
@@ -814,14 +816,18 @@
 #endif
     }
 
-    set_padding_border(dict, wp->w_popup_padding, "padding", 999);
-    set_padding_border(dict, wp->w_popup_border, "border", 1);
+    if (set_padding_border(dict, wp->w_popup_padding, "padding", 999) == FAIL ||
+        set_padding_border(dict, wp->w_popup_border, "border", 1) == FAIL)
+	return FAIL;
 
     di = dict_find(dict, (char_u *)"borderhighlight", -1);
     if (di != NULL)
     {
 	if (di->di_tv.v_type != VAR_LIST || di->di_tv.vval.v_list == NULL)
+	{
 	    emsg(_(e_list_required));
+	    return FAIL;
+	}
 	else
 	{
 	    list_T	*list = di->di_tv.vval.v_list;
@@ -853,7 +859,10 @@
     if (di != NULL)
     {
 	if (di->di_tv.v_type != VAR_LIST)
+	{
 	    emsg(_(e_list_required));
+	    return FAIL;
+	}
 	else
 	{
 	    list_T	*list = di->di_tv.vval.v_list;
@@ -927,7 +936,10 @@
 	    VIM_CLEAR(wp->w_popup_mask_cells);
 	}
 	else
+	{
 	    semsg(_(e_invalid_value_for_argument_str), "mask");
+	    return FAIL;
+	}
     }
 
 #if defined(FEAT_TIMERS)
@@ -993,23 +1005,25 @@
 
     di = dict_find(dict, (char_u *)"callback", -1);
     if (di == NULL)
-	return;
+	return OK;
 
     callback_T	callback = get_callback(&di->di_tv);
     if (callback.cb_name == NULL)
-	return;
+	return OK;
 
     free_callback(&wp->w_close_cb);
     set_callback(&wp->w_close_cb, &callback);
     if (callback.cb_free_name)
 	vim_free(callback.cb_name);
+
+    return OK;
 }
 
 /*
  * Go through the options in "dict" and apply them to popup window "wp".
  * "create" is TRUE when creating a new popup window.
  */
-    static void
+    static int
 apply_options(win_T *wp, dict_T *dict, int create)
 {
     int		nr;
@@ -1020,7 +1034,8 @@
 	set_string_option_direct_in_win(wp, (char_u *)"signcolumn", -1,
 					(char_u *)"no", OPT_FREE|OPT_LOCAL, 0);
 
-    apply_general_options(wp, dict);
+    if (apply_general_options(wp, dict) == FAIL)
+	return FAIL;
 
     nr = dict_get_bool(dict, "hidden", FALSE);
     if (nr > 0)
@@ -1041,6 +1056,8 @@
 
     popup_mask_refresh = TRUE;
     popup_highlight_curline(wp);
+
+    return OK;
 }
 
 /*
@@ -2279,8 +2296,14 @@
     wp->w_filter_mode = MODE_ALL;
 
     if (d != NULL)
+    {
 	// Deal with options.
-	apply_options(wp, d, TRUE);
+	if (apply_options(wp, d, TRUE) == FAIL)
+	{
+	    (void)popup_close(wp->w_id, FALSE);
+	    return NULL;
+	}
+    }
 
 #ifdef FEAT_TIMERS
     if (popup_is_notification(type) && wp->w_popup_timer == NULL)
@@ -2992,7 +3015,7 @@
     dict = argvars[1].vval.v_dict;
     old_firstline = wp->w_firstline;
 
-    apply_options(wp, dict, FALSE);
+    (void)apply_options(wp, dict, FALSE);
 
     if (old_firstline != wp->w_firstline)
 	redraw_win_later(wp, UPD_NOT_VALID);