patch 8.0.1349: options test fails when using Motif or GTK GUI

Problem:    Options test fails when using Motif or GTK GUI.
Solution:   Use "fixed" instead of "fixedsys" for Unix. Don't try "xxx" for
            guifonteset.  Don't set 'termencoding' to anything but "utf-8" for
            GTK.  Give an error if 'termencoding' can't be converted.
diff --git a/src/option.c b/src/option.c
index a867608..c97056b 100644
--- a/src/option.c
+++ b/src/option.c
@@ -6369,8 +6369,13 @@
 	     * display output conversion. */
 	    if (((varp == &p_enc && *p_tenc != NUL) || varp == &p_tenc))
 	    {
-		convert_setup(&input_conv, p_tenc, p_enc);
-		convert_setup(&output_conv, p_enc, p_tenc);
+		if (convert_setup(&input_conv, p_tenc, p_enc) == FAIL
+			|| convert_setup(&output_conv, p_enc, p_tenc) == FAIL)
+		{
+		    EMSG3(_("E950: Cannot convert between %s and %s"),
+			    p_tenc, p_enc);
+		    errmsg = e_invarg;
+		}
 	    }
 
 # if defined(WIN3264) && defined(FEAT_MBYTE)