diff --git a/common/rfb/DecodeManager.cxx b/common/rfb/DecodeManager.cxx
index d7cffcf..a655c53 100644
--- a/common/rfb/DecodeManager.cxx
+++ b/common/rfb/DecodeManager.cxx
@@ -56,7 +56,12 @@
     // wasting CPU fighting for locks
     if (cpuCount > 4)
       cpuCount = 4;
-    vlog.info("Creating %d decoder thread(s)", (int)cpuCount);
+    // The overhead of threading is small, but not small enough to
+    // ignore on single CPU systems
+    if (cpuCount == 1)
+      vlog.info("Decoding data on main thread");
+    else
+      vlog.info("Creating %d decoder thread(s)", (int)cpuCount);
   }
 
   while (cpuCount--) {
@@ -116,6 +121,17 @@
 
   decoder = decoders[encoding];
 
+  // Fast path for single CPU machines to avoid the context
+  // switching overhead
+  if (threads.size() == 1) {
+    bufferStream = freeBuffers.front();
+    bufferStream->clear();
+    decoder->readRect(r, conn->getInStream(), conn->cp, bufferStream);
+    decoder->decodeRect(r, bufferStream->data(), bufferStream->length(),
+                        conn->cp, pb);
+    return;
+  }
+
   // Wait for an available memory buffer
   queueMutex->lock();