diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx
index 53e0365..8f5427c 100644
--- a/common/rfb/EncodeManager.cxx
+++ b/common/rfb/EncodeManager.cxx
@@ -1,6 +1,7 @@
 /* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
  * Copyright 2014-2018 Pierre Ossman for Cendio AB
+ * Copyright 2018 Peter Astrand for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -50,6 +51,9 @@
 // Don't bother with blocks smaller than this
 static const int SolidBlockMinArea = 2048;
 
+// How long we consider a region recently changed (in ms)
+static const int RecentChangeTimeout = 50;
+
 namespace rfb {
 
 enum EncoderClass {
@@ -123,7 +127,8 @@
   return "Unknown Encoder Type";
 }
 
-EncodeManager::EncodeManager(SConnection* conn_) : conn(conn_)
+EncodeManager::EncodeManager(SConnection* conn_)
+  : conn(conn_), recentChangeTimer(this)
 {
   StatsVector::iterator iter;
 
@@ -253,23 +258,57 @@
   return !lossyRegion.intersect(req).is_empty();
 }
 
+int EncodeManager::getNextLosslessRefresh(const Region& req)
+{
+  // Do we have something we can send right away?
+  if (!pendingRefreshRegion.intersect(req).is_empty())
+    return 0;
+
+  assert(needsLosslessRefresh(req));
+  assert(recentChangeTimer.isStarted());
+
+  return recentChangeTimer.getNextTimeout();
+}
+
 void EncodeManager::pruneLosslessRefresh(const Region& limits)
 {
   lossyRegion.assign_intersect(limits);
+  pendingRefreshRegion.assign_intersect(limits);
 }
 
 void EncodeManager::writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
                                 const RenderedCursor* renderedCursor)
 {
-    doUpdate(true, ui.changed, ui.copied, ui.copy_delta, pb, renderedCursor);
+  doUpdate(true, ui.changed, ui.copied, ui.copy_delta, pb, renderedCursor);
+
+  recentlyChangedRegion.assign_union(ui.changed);
+  recentlyChangedRegion.assign_union(ui.copied);
+  if (!recentChangeTimer.isStarted())
+    recentChangeTimer.start(RecentChangeTimeout);
 }
 
 void EncodeManager::writeLosslessRefresh(const Region& req, const PixelBuffer* pb,
                                          const RenderedCursor* renderedCursor,
                                          size_t maxUpdateSize)
 {
-    doUpdate(false, getLosslessRefresh(req, maxUpdateSize),
-             Region(), Point(), pb, renderedCursor);
+  doUpdate(false, getLosslessRefresh(req, maxUpdateSize),
+           Region(), Point(), pb, renderedCursor);
+}
+
+bool EncodeManager::handleTimeout(Timer* t)
+{
+  if (t == &recentChangeTimer) {
+    // Any lossy region that wasn't recently updated can
+    // now be scheduled for a refresh
+    pendingRefreshRegion.assign_union(lossyRegion.subtract(recentlyChangedRegion));
+    recentlyChangedRegion.clear();
+
+    // Will there be more to do? (i.e. do we need another round)
+    if (!lossyRegion.subtract(pendingRefreshRegion).is_empty())
+      return true;
+  }
+
+  return false;
 }
 
 void EncodeManager::doUpdate(bool allowLossy, const Region& changed_,
@@ -438,7 +477,7 @@
   maxUpdateSize *= 2;
 
   area = 0;
-  lossyRegion.intersect(req).get_rects(&rects);
+  pendingRefreshRegion.intersect(req).get_rects(&rects);
   while (!rects.empty()) {
     size_t idx;
     Rect rect;
@@ -530,6 +569,10 @@
   else
     lossyRegion.assign_subtract(Region(rect));
 
+  // This was either a rect getting refreshed, or a rect that just got
+  // new content. Either way we should not try to refresh it anymore.
+  pendingRefreshRegion.assign_subtract(Region(rect));
+
   return encoder;
 }
 
@@ -574,6 +617,10 @@
   lossyCopy.translate(delta);
   lossyCopy.assign_intersect(copied);
   lossyRegion.assign_union(lossyCopy);
+
+  // Stop any pending refresh as a copy is enough that we consider
+  // this region to be recently changed
+  pendingRefreshRegion.assign_subtract(copied);
 }
 
 void EncodeManager::writeSolidRects(Region *changed, const PixelBuffer* pb)