diff --git a/common/rfb/Congestion.cxx b/common/rfb/Congestion.cxx
index 8162808..4a78452 100644
--- a/common/rfb/Congestion.cxx
+++ b/common/rfb/Congestion.cxx
@@ -291,11 +291,20 @@
 
 size_t Congestion::getBandwidth()
 {
+  size_t bandwidth;
+
   // No measurements yet? Guess RTT of 60 ms
   if (safeBaseRTT == (unsigned)-1)
-    return congWindow * 1000 / 60;
+    bandwidth = congWindow * 1000 / 60;
+  else
+    bandwidth = congWindow * 1000 / safeBaseRTT;
 
-  return congWindow * 1000 / safeBaseRTT;
+  // We're still probing so guess actual bandwidth is halfway between
+  // the current guess and the next one (slow start doubles each time)
+  if (inSlowStart)
+    bandwidth = bandwidth + bandwidth / 2;
+
+  return bandwidth;
 }
 
 void Congestion::debugTrace(const char* filename, int fd)
diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx
index 53e0365..02128f7 100644
--- a/common/rfb/EncodeManager.cxx
+++ b/common/rfb/EncodeManager.cxx
@@ -1,6 +1,7 @@
 /* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
  * Copyright 2014-2018 Pierre Ossman for Cendio AB
+ * Copyright 2018 Peter Astrand for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -50,6 +51,9 @@
 // Don't bother with blocks smaller than this
 static const int SolidBlockMinArea = 2048;
 
+// How long we consider a region recently changed (in ms)
+static const int RecentChangeTimeout = 50;
+
 namespace rfb {
 
 enum EncoderClass {
@@ -123,7 +127,8 @@
   return "Unknown Encoder Type";
 }
 
-EncodeManager::EncodeManager(SConnection* conn_) : conn(conn_)
+EncodeManager::EncodeManager(SConnection* conn_)
+  : conn(conn_), recentChangeTimer(this)
 {
   StatsVector::iterator iter;
 
@@ -253,23 +258,57 @@
   return !lossyRegion.intersect(req).is_empty();
 }
 
+int EncodeManager::getNextLosslessRefresh(const Region& req)
+{
+  // Do we have something we can send right away?
+  if (!pendingRefreshRegion.intersect(req).is_empty())
+    return 0;
+
+  assert(needsLosslessRefresh(req));
+  assert(recentChangeTimer.isStarted());
+
+  return recentChangeTimer.getNextTimeout();
+}
+
 void EncodeManager::pruneLosslessRefresh(const Region& limits)
 {
   lossyRegion.assign_intersect(limits);
+  pendingRefreshRegion.assign_intersect(limits);
 }
 
 void EncodeManager::writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
                                 const RenderedCursor* renderedCursor)
 {
-    doUpdate(true, ui.changed, ui.copied, ui.copy_delta, pb, renderedCursor);
+  doUpdate(true, ui.changed, ui.copied, ui.copy_delta, pb, renderedCursor);
+
+  recentlyChangedRegion.assign_union(ui.changed);
+  recentlyChangedRegion.assign_union(ui.copied);
+  if (!recentChangeTimer.isStarted())
+    recentChangeTimer.start(RecentChangeTimeout);
 }
 
 void EncodeManager::writeLosslessRefresh(const Region& req, const PixelBuffer* pb,
                                          const RenderedCursor* renderedCursor,
                                          size_t maxUpdateSize)
 {
-    doUpdate(false, getLosslessRefresh(req, maxUpdateSize),
-             Region(), Point(), pb, renderedCursor);
+  doUpdate(false, getLosslessRefresh(req, maxUpdateSize),
+           Region(), Point(), pb, renderedCursor);
+}
+
+bool EncodeManager::handleTimeout(Timer* t)
+{
+  if (t == &recentChangeTimer) {
+    // Any lossy region that wasn't recently updated can
+    // now be scheduled for a refresh
+    pendingRefreshRegion.assign_union(lossyRegion.subtract(recentlyChangedRegion));
+    recentlyChangedRegion.clear();
+
+    // Will there be more to do? (i.e. do we need another round)
+    if (!lossyRegion.subtract(pendingRefreshRegion).is_empty())
+      return true;
+  }
+
+  return false;
 }
 
 void EncodeManager::doUpdate(bool allowLossy, const Region& changed_,
@@ -325,6 +364,8 @@
   enum EncoderClass solid, bitmap, bitmapRLE;
   enum EncoderClass indexed, indexedRLE, fullColour;
 
+  bool allowJPEG;
+
   rdr::S32 preferred;
 
   std::vector<int>::iterator iter;
@@ -332,6 +373,12 @@
   solid = bitmap = bitmapRLE = encoderRaw;
   indexed = indexedRLE = fullColour = encoderRaw;
 
+  allowJPEG = conn->cp.pf().bpp >= 16;
+  if (!allowLossy) {
+    if (encoders[encoderTightJPEG]->losslessQuality == -1)
+      allowJPEG = false;
+  }
+
   // Try to respect the client's wishes
   preferred = conn->getPreferredEncoding();
   switch (preferred) {
@@ -344,8 +391,7 @@
     bitmapRLE = indexedRLE = fullColour = encoderHextile;
     break;
   case encodingTight:
-    if (encoders[encoderTightJPEG]->isSupported() &&
-        (conn->cp.pf().bpp >= 16) && allowLossy)
+    if (encoders[encoderTightJPEG]->isSupported() && allowJPEG)
       fullColour = encoderTightJPEG;
     else
       fullColour = encoderTight;
@@ -362,8 +408,7 @@
   // Any encoders still unassigned?
 
   if (fullColour == encoderRaw) {
-    if (encoders[encoderTightJPEG]->isSupported() &&
-        (conn->cp.pf().bpp >= 16) && allowLossy)
+    if (encoders[encoderTightJPEG]->isSupported() && allowJPEG)
       fullColour = encoderTightJPEG;
     else if (encoders[encoderZRLE]->isSupported())
       fullColour = encoderZRLE;
@@ -421,9 +466,17 @@
     encoder = encoders[*iter];
 
     encoder->setCompressLevel(conn->cp.compressLevel);
-    encoder->setQualityLevel(conn->cp.qualityLevel);
-    encoder->setFineQualityLevel(conn->cp.fineQualityLevel,
-                                 conn->cp.subsampling);
+
+    if (allowLossy) {
+      encoder->setQualityLevel(conn->cp.qualityLevel);
+      encoder->setFineQualityLevel(conn->cp.fineQualityLevel,
+                                   conn->cp.subsampling);
+    } else {
+      int level = __rfbmax(conn->cp.qualityLevel,
+                           encoder->losslessQuality);
+      encoder->setQualityLevel(level);
+      encoder->setFineQualityLevel(-1, subsampleUndefined);
+    }
   }
 }
 
@@ -437,8 +490,11 @@
   // We make a conservative guess at the compression ratio at 2:1
   maxUpdateSize *= 2;
 
+  // We will measure pixels, not bytes (assume 32 bpp)
+  maxUpdateSize /= 4;
+
   area = 0;
-  lossyRegion.intersect(req).get_rects(&rects);
+  pendingRefreshRegion.intersect(req).get_rects(&rects);
   while (!rects.empty()) {
     size_t idx;
     Rect rect;
@@ -525,11 +581,17 @@
   encoder = encoders[klass];
   conn->writer()->startRect(rect, encoder->encoding);
 
-  if (encoder->flags & EncoderLossy)
+  if ((encoder->flags & EncoderLossy) &&
+      ((encoder->losslessQuality == -1) ||
+       (encoder->getQualityLevel() < encoder->losslessQuality)))
     lossyRegion.assign_union(Region(rect));
   else
     lossyRegion.assign_subtract(Region(rect));
 
+  // This was either a rect getting refreshed, or a rect that just got
+  // new content. Either way we should not try to refresh it anymore.
+  pendingRefreshRegion.assign_subtract(Region(rect));
+
   return encoder;
 }
 
@@ -574,6 +636,10 @@
   lossyCopy.translate(delta);
   lossyCopy.assign_intersect(copied);
   lossyRegion.assign_union(lossyCopy);
+
+  // Stop any pending refresh as a copy is enough that we consider
+  // this region to be recently changed
+  pendingRefreshRegion.assign_subtract(copied);
 }
 
 void EncodeManager::writeSolidRects(Region *changed, const PixelBuffer* pb)
diff --git a/common/rfb/EncodeManager.h b/common/rfb/EncodeManager.h
index a91c544..bdae906 100644
--- a/common/rfb/EncodeManager.h
+++ b/common/rfb/EncodeManager.h
@@ -25,6 +25,7 @@
 #include <rdr/types.h>
 #include <rfb/PixelBuffer.h>
 #include <rfb/Region.h>
+#include <rfb/Timer.h>
 
 namespace rfb {
   class SConnection;
@@ -36,7 +37,7 @@
 
   struct RectInfo;
 
-  class EncodeManager {
+  class EncodeManager : public Timer::Callback {
   public:
     EncodeManager(SConnection* conn);
     ~EncodeManager();
@@ -47,6 +48,8 @@
     static bool supported(int encoding);
 
     bool needsLosslessRefresh(const Region& req);
+    int getNextLosslessRefresh(const Region& req);
+
     void pruneLosslessRefresh(const Region& limits);
 
     void writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
@@ -57,6 +60,8 @@
                               size_t maxUpdateSize);
 
   protected:
+    virtual bool handleTimeout(Timer* t);
+
     void doUpdate(bool allowLossy, const Region& changed,
                   const Region& copied, const Point& copy_delta,
                   const PixelBuffer* pb,
@@ -117,6 +122,10 @@
     std::vector<int> activeEncoders;
 
     Region lossyRegion;
+    Region recentlyChangedRegion;
+    Region pendingRefreshRegion;
+
+    Timer recentChangeTimer;
 
     struct EncoderStats {
       unsigned rects;
diff --git a/common/rfb/Encoder.cxx b/common/rfb/Encoder.cxx
index 18b6680..0e29f1d 100644
--- a/common/rfb/Encoder.cxx
+++ b/common/rfb/Encoder.cxx
@@ -24,9 +24,11 @@
 using namespace rfb;
 
 Encoder::Encoder(SConnection *conn_, int encoding_,
-                 enum EncoderFlags flags_, unsigned int maxPaletteSize_) :
+                 enum EncoderFlags flags_,
+                 unsigned int maxPaletteSize_, int losslessQuality_) :
   encoding(encoding_), flags(flags_),
-  maxPaletteSize(maxPaletteSize_), conn(conn_)
+  maxPaletteSize(maxPaletteSize_), losslessQuality(losslessQuality_),
+  conn(conn_)
 {
 }
 
diff --git a/common/rfb/Encoder.h b/common/rfb/Encoder.h
index 66a10d2..d5a0288 100644
--- a/common/rfb/Encoder.h
+++ b/common/rfb/Encoder.h
@@ -42,7 +42,8 @@
   class Encoder {
   public:
     Encoder(SConnection* conn, int encoding,
-            enum EncoderFlags flags, unsigned int maxPaletteSize);
+            enum EncoderFlags flags, unsigned int maxPaletteSize=-1,
+            int losslessQuality=-1);
     virtual ~Encoder();
 
     // isSupported() should return a boolean indicating if this encoder
@@ -54,6 +55,9 @@
     virtual void setQualityLevel(int level) {};
     virtual void setFineQualityLevel(int quality, int subsampling) {};
 
+    virtual int getCompressLevel() { return -1; };
+    virtual int getQualityLevel() { return -1; };
+
     // writeRect() is the main interface that encodes the given rectangle
     // with data from the PixelBuffer onto the SConnection given at
     // encoder creation.
@@ -92,6 +96,10 @@
     // Maximum size of the palette per rect
     const unsigned int maxPaletteSize;
 
+    // Minimum level where the quality loss will not be noticed by
+    // most users (only relevant with EncoderLossy flag)
+    const int losslessQuality;
+
   protected:
     SConnection* conn;
   };
diff --git a/common/rfb/HextileEncoder.cxx b/common/rfb/HextileEncoder.cxx
index 418a440..47e5251 100644
--- a/common/rfb/HextileEncoder.cxx
+++ b/common/rfb/HextileEncoder.cxx
@@ -45,7 +45,7 @@
 #undef BPP
 
 HextileEncoder::HextileEncoder(SConnection* conn) :
-  Encoder(conn, encodingHextile, EncoderPlain, -1)
+  Encoder(conn, encodingHextile, EncoderPlain)
 {
 }
 
diff --git a/common/rfb/RREEncoder.cxx b/common/rfb/RREEncoder.cxx
index 60a0663..7287e7e 100644
--- a/common/rfb/RREEncoder.cxx
+++ b/common/rfb/RREEncoder.cxx
@@ -37,7 +37,7 @@
 #undef BPP
 
 RREEncoder::RREEncoder(SConnection* conn) :
-  Encoder(conn, encodingRRE, EncoderPlain, -1)
+  Encoder(conn, encodingRRE, EncoderPlain)
 {
 }
 
diff --git a/common/rfb/RawEncoder.cxx b/common/rfb/RawEncoder.cxx
index 4090427..b12cf06 100644
--- a/common/rfb/RawEncoder.cxx
+++ b/common/rfb/RawEncoder.cxx
@@ -25,7 +25,7 @@
 using namespace rfb;
 
 RawEncoder::RawEncoder(SConnection* conn) :
-  Encoder(conn, encodingRaw, EncoderPlain, -1)
+  Encoder(conn, encodingRaw, EncoderPlain)
 {
 }
 
diff --git a/common/rfb/TightJPEGEncoder.cxx b/common/rfb/TightJPEGEncoder.cxx
index 385207f..38cb4eb 100644
--- a/common/rfb/TightJPEGEncoder.cxx
+++ b/common/rfb/TightJPEGEncoder.cxx
@@ -64,7 +64,8 @@
 
 
 TightJPEGEncoder::TightJPEGEncoder(SConnection* conn) :
-  Encoder(conn, encodingTight, (EncoderFlags)(EncoderUseNativePF | EncoderLossy), -1),
+  Encoder(conn, encodingTight,
+          (EncoderFlags)(EncoderUseNativePF | EncoderLossy), -1, 9),
   qualityLevel(-1), fineQuality(-1), fineSubsampling(subsampleUndefined)
 {
 }
@@ -101,6 +102,11 @@
   fineSubsampling = subsampling;
 }
 
+int TightJPEGEncoder::getQualityLevel()
+{
+  return qualityLevel;
+}
+
 void TightJPEGEncoder::writeRect(const PixelBuffer* pb, const Palette& palette)
 {
   const rdr::U8* buffer;
diff --git a/common/rfb/TightJPEGEncoder.h b/common/rfb/TightJPEGEncoder.h
index 458c383..3d8fa8c 100644
--- a/common/rfb/TightJPEGEncoder.h
+++ b/common/rfb/TightJPEGEncoder.h
@@ -35,6 +35,8 @@
     virtual void setQualityLevel(int level);
     virtual void setFineQualityLevel(int quality, int subsampling);
 
+    virtual int getQualityLevel();
+
     virtual void writeRect(const PixelBuffer* pb, const Palette& palette);
     virtual void writeSolidRect(int width, int height,
                                 const PixelFormat& pf,
diff --git a/common/rfb/Timer.cxx b/common/rfb/Timer.cxx
index fd1a87a..b9dd2f6 100644
--- a/common/rfb/Timer.cxx
+++ b/common/rfb/Timer.cxx
@@ -151,7 +151,7 @@
 int Timer::getRemainingMs() {
   timeval now;
   gettimeofday(&now, 0);
-  return __rfbmax(0, diffTimeMillis(pending.front()->dueTime, now));
+  return __rfbmax(0, diffTimeMillis(dueTime, now));
 }
 
 bool Timer::isBefore(timeval other) {
diff --git a/common/rfb/VNCSConnectionST.cxx b/common/rfb/VNCSConnectionST.cxx
index f22b993..f1591f4 100644
--- a/common/rfb/VNCSConnectionST.cxx
+++ b/common/rfb/VNCSConnectionST.cxx
@@ -1,5 +1,6 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
  * Copyright 2009-2018 Pierre Ossman for Cendio AB
+ * Copyright 2018 Peter Astrand for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -48,7 +49,7 @@
     inProcessMessages(false),
     pendingSyncFence(false), syncFence(false), fenceFlags(0),
     fenceDataLen(0), fenceData(NULL), congestionTimer(this),
-    server(server_), updates(false),
+    losslessTimer(this), server(server_), updates(false),
     updateRenderedCursor(false), removeRenderedCursor(false),
     continuousUpdates(false), encodeManager(this), pointerEventTime(0),
     clientHasCursor(false),
@@ -839,7 +840,8 @@
 bool VNCSConnectionST::handleTimeout(Timer* t)
 {
   try {
-    if (t == &congestionTimer)
+    if ((t == &congestionTimer) ||
+        (t == &losslessTimer))
       writeFramebufferUpdate();
   } catch (rdr::Exception& e) {
     close(e.str());
@@ -1065,28 +1067,46 @@
   }
 
   // Return if there is nothing to send the client.
+  if (ui.is_empty() && !writer()->needFakeUpdate()) {
+    int eta;
 
-  if (ui.is_empty() && !writer()->needFakeUpdate() &&
-      !encodeManager.needsLosslessRefresh(req))
-    return;
+    // Any lossless refresh that needs handling?
+    if (!encodeManager.needsLosslessRefresh(req))
+      return;
+
+    // Now? Or later?
+    eta = encodeManager.getNextLosslessRefresh(req);
+    if (eta > 0) {
+      losslessTimer.start(eta);
+      return;
+    }
+  }
 
   writeRTTPing();
 
   if (!ui.is_empty())
     encodeManager.writeUpdate(ui, server->getPixelBuffer(), cursor);
   else {
-    size_t maxUpdateSize;
+    int nextUpdate;
 
     // FIXME: If continuous updates aren't used then the client might
     //        be slower than frameRate in its requests and we could
     //        afford a larger update size
+    nextUpdate = server->msToNextUpdate();
+    if (nextUpdate > 0) {
+      size_t bandwidth, maxUpdateSize;
 
-    // FIXME: Bandwidth estimation without congestion control
-    maxUpdateSize = congestion.getBandwidth() *
-                    server->msToNextUpdate() / 1000;
+      // FIXME: Bandwidth estimation without congestion control
+      bandwidth = congestion.getBandwidth();
 
-    encodeManager.writeLosslessRefresh(req, server->getPixelBuffer(),
-                                       cursor, maxUpdateSize);
+      // FIXME: Hard coded value for maximum CPU throughput
+      if (bandwidth > 5000000)
+        bandwidth = 5000000;
+
+      maxUpdateSize = bandwidth * nextUpdate / 1000;
+      encodeManager.writeLosslessRefresh(req, server->getPixelBuffer(),
+                                         cursor, maxUpdateSize);
+    }
   }
 
   writeRTTPing();
diff --git a/common/rfb/VNCSConnectionST.h b/common/rfb/VNCSConnectionST.h
index 2f075a6..dfc8bcb 100644
--- a/common/rfb/VNCSConnectionST.h
+++ b/common/rfb/VNCSConnectionST.h
@@ -191,6 +191,7 @@
 
     Congestion congestion;
     Timer congestionTimer;
+    Timer losslessTimer;
 
     VNCServerST* server;
     SimpleUpdateTracker updates;