Turns out that TXImage wasn't just being overly cautious with the custom error
handler; the shared memory extension is actually this broken so it will return
success all the way through even when you're on a remote connection where
things cannot possibly work.
git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4505 3789f03b-4d11-0410-bbf8-ca57d06f2519
diff --git a/vncviewer/X11PixelBuffer.cxx b/vncviewer/X11PixelBuffer.cxx
index 79fa76a..a3f8ebf 100644
--- a/vncviewer/X11PixelBuffer.cxx
+++ b/vncviewer/X11PixelBuffer.cxx
@@ -140,11 +140,20 @@
return xim->bytes_per_line / (getPF().bpp/8);
}
+static bool caughtError;
+
+static int XShmAttachErrorHandler(Display *dpy, XErrorEvent *error)
+{
+ caughtError = true;
+ return 0;
+}
int PlatformPixelBuffer::setupShm()
{
int major, minor;
Bool pixmaps;
+ XErrorHandler old_handler;
+ Status status;
if (!XShmQueryVersion(fl_display, &major, &minor, &pixmaps))
return 0;
@@ -167,12 +176,27 @@
goto free_shm;
shminfo->readOnly = True;
+
+ // This is the only way we can detect that shared memory won't work
+ // (e.g. because we're accessing a remote X11 server)
+ caughtError = false;
+ old_handler = XSetErrorHandler(XShmAttachErrorHandler);
+
XShmAttach(fl_display, shminfo);
+ XSync(fl_display, False);
+
+ XSetErrorHandler(old_handler);
+
+ if (caughtError)
+ goto free_shmaddr;
vlog.debug("Using shared memory XImage");
return 1;
+free_shmaddr:
+ shmdt(shminfo->shmaddr);
+
free_shm:
shmctl(shminfo->shmid, IPC_RMID, 0);