diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx
index 53e0365..8f5427c 100644
--- a/common/rfb/EncodeManager.cxx
+++ b/common/rfb/EncodeManager.cxx
@@ -1,6 +1,7 @@
 /* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
  * Copyright 2014-2018 Pierre Ossman for Cendio AB
+ * Copyright 2018 Peter Astrand for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -50,6 +51,9 @@
 // Don't bother with blocks smaller than this
 static const int SolidBlockMinArea = 2048;
 
+// How long we consider a region recently changed (in ms)
+static const int RecentChangeTimeout = 50;
+
 namespace rfb {
 
 enum EncoderClass {
@@ -123,7 +127,8 @@
   return "Unknown Encoder Type";
 }
 
-EncodeManager::EncodeManager(SConnection* conn_) : conn(conn_)
+EncodeManager::EncodeManager(SConnection* conn_)
+  : conn(conn_), recentChangeTimer(this)
 {
   StatsVector::iterator iter;
 
@@ -253,23 +258,57 @@
   return !lossyRegion.intersect(req).is_empty();
 }
 
+int EncodeManager::getNextLosslessRefresh(const Region& req)
+{
+  // Do we have something we can send right away?
+  if (!pendingRefreshRegion.intersect(req).is_empty())
+    return 0;
+
+  assert(needsLosslessRefresh(req));
+  assert(recentChangeTimer.isStarted());
+
+  return recentChangeTimer.getNextTimeout();
+}
+
 void EncodeManager::pruneLosslessRefresh(const Region& limits)
 {
   lossyRegion.assign_intersect(limits);
+  pendingRefreshRegion.assign_intersect(limits);
 }
 
 void EncodeManager::writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
                                 const RenderedCursor* renderedCursor)
 {
-    doUpdate(true, ui.changed, ui.copied, ui.copy_delta, pb, renderedCursor);
+  doUpdate(true, ui.changed, ui.copied, ui.copy_delta, pb, renderedCursor);
+
+  recentlyChangedRegion.assign_union(ui.changed);
+  recentlyChangedRegion.assign_union(ui.copied);
+  if (!recentChangeTimer.isStarted())
+    recentChangeTimer.start(RecentChangeTimeout);
 }
 
 void EncodeManager::writeLosslessRefresh(const Region& req, const PixelBuffer* pb,
                                          const RenderedCursor* renderedCursor,
                                          size_t maxUpdateSize)
 {
-    doUpdate(false, getLosslessRefresh(req, maxUpdateSize),
-             Region(), Point(), pb, renderedCursor);
+  doUpdate(false, getLosslessRefresh(req, maxUpdateSize),
+           Region(), Point(), pb, renderedCursor);
+}
+
+bool EncodeManager::handleTimeout(Timer* t)
+{
+  if (t == &recentChangeTimer) {
+    // Any lossy region that wasn't recently updated can
+    // now be scheduled for a refresh
+    pendingRefreshRegion.assign_union(lossyRegion.subtract(recentlyChangedRegion));
+    recentlyChangedRegion.clear();
+
+    // Will there be more to do? (i.e. do we need another round)
+    if (!lossyRegion.subtract(pendingRefreshRegion).is_empty())
+      return true;
+  }
+
+  return false;
 }
 
 void EncodeManager::doUpdate(bool allowLossy, const Region& changed_,
@@ -438,7 +477,7 @@
   maxUpdateSize *= 2;
 
   area = 0;
-  lossyRegion.intersect(req).get_rects(&rects);
+  pendingRefreshRegion.intersect(req).get_rects(&rects);
   while (!rects.empty()) {
     size_t idx;
     Rect rect;
@@ -530,6 +569,10 @@
   else
     lossyRegion.assign_subtract(Region(rect));
 
+  // This was either a rect getting refreshed, or a rect that just got
+  // new content. Either way we should not try to refresh it anymore.
+  pendingRefreshRegion.assign_subtract(Region(rect));
+
   return encoder;
 }
 
@@ -574,6 +617,10 @@
   lossyCopy.translate(delta);
   lossyCopy.assign_intersect(copied);
   lossyRegion.assign_union(lossyCopy);
+
+  // Stop any pending refresh as a copy is enough that we consider
+  // this region to be recently changed
+  pendingRefreshRegion.assign_subtract(copied);
 }
 
 void EncodeManager::writeSolidRects(Region *changed, const PixelBuffer* pb)
diff --git a/common/rfb/EncodeManager.h b/common/rfb/EncodeManager.h
index a91c544..bdae906 100644
--- a/common/rfb/EncodeManager.h
+++ b/common/rfb/EncodeManager.h
@@ -25,6 +25,7 @@
 #include <rdr/types.h>
 #include <rfb/PixelBuffer.h>
 #include <rfb/Region.h>
+#include <rfb/Timer.h>
 
 namespace rfb {
   class SConnection;
@@ -36,7 +37,7 @@
 
   struct RectInfo;
 
-  class EncodeManager {
+  class EncodeManager : public Timer::Callback {
   public:
     EncodeManager(SConnection* conn);
     ~EncodeManager();
@@ -47,6 +48,8 @@
     static bool supported(int encoding);
 
     bool needsLosslessRefresh(const Region& req);
+    int getNextLosslessRefresh(const Region& req);
+
     void pruneLosslessRefresh(const Region& limits);
 
     void writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
@@ -57,6 +60,8 @@
                               size_t maxUpdateSize);
 
   protected:
+    virtual bool handleTimeout(Timer* t);
+
     void doUpdate(bool allowLossy, const Region& changed,
                   const Region& copied, const Point& copy_delta,
                   const PixelBuffer* pb,
@@ -117,6 +122,10 @@
     std::vector<int> activeEncoders;
 
     Region lossyRegion;
+    Region recentlyChangedRegion;
+    Region pendingRefreshRegion;
+
+    Timer recentChangeTimer;
 
     struct EncoderStats {
       unsigned rects;
diff --git a/common/rfb/VNCSConnectionST.cxx b/common/rfb/VNCSConnectionST.cxx
index f22b993..41c6d23 100644
--- a/common/rfb/VNCSConnectionST.cxx
+++ b/common/rfb/VNCSConnectionST.cxx
@@ -1,5 +1,6 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
  * Copyright 2009-2018 Pierre Ossman for Cendio AB
+ * Copyright 2018 Peter Astrand for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -48,7 +49,7 @@
     inProcessMessages(false),
     pendingSyncFence(false), syncFence(false), fenceFlags(0),
     fenceDataLen(0), fenceData(NULL), congestionTimer(this),
-    server(server_), updates(false),
+    losslessTimer(this), server(server_), updates(false),
     updateRenderedCursor(false), removeRenderedCursor(false),
     continuousUpdates(false), encodeManager(this), pointerEventTime(0),
     clientHasCursor(false),
@@ -839,7 +840,8 @@
 bool VNCSConnectionST::handleTimeout(Timer* t)
 {
   try {
-    if (t == &congestionTimer)
+    if ((t == &congestionTimer) ||
+        (t == &losslessTimer))
       writeFramebufferUpdate();
   } catch (rdr::Exception& e) {
     close(e.str());
@@ -1065,10 +1067,20 @@
   }
 
   // Return if there is nothing to send the client.
+  if (ui.is_empty() && !writer()->needFakeUpdate()) {
+    int eta;
 
-  if (ui.is_empty() && !writer()->needFakeUpdate() &&
-      !encodeManager.needsLosslessRefresh(req))
-    return;
+    // Any lossless refresh that needs handling?
+    if (!encodeManager.needsLosslessRefresh(req))
+      return;
+
+    // Now? Or later?
+    eta = encodeManager.getNextLosslessRefresh(req);
+    if (eta > 0) {
+      losslessTimer.start(eta);
+      return;
+    }
+  }
 
   writeRTTPing();
 
diff --git a/common/rfb/VNCSConnectionST.h b/common/rfb/VNCSConnectionST.h
index 2f075a6..dfc8bcb 100644
--- a/common/rfb/VNCSConnectionST.h
+++ b/common/rfb/VNCSConnectionST.h
@@ -191,6 +191,7 @@
 
     Congestion congestion;
     Timer congestionTimer;
+    Timer losslessTimer;
 
     VNCServerST* server;
     SimpleUpdateTracker updates;