diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx
index 0ceec8f..ca7c730 100644
--- a/common/rfb/EncodeManager.cxx
+++ b/common/rfb/EncodeManager.cxx
@@ -17,6 +17,9 @@
  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
  * USA.
  */
+
+#include <stdlib.h>
+
 #include <rfb/EncodeManager.h>
 #include <rfb/Encoder.h>
 #include <rfb/Palette.h>
@@ -47,6 +50,8 @@
 // Don't bother with blocks smaller than this
 static const int SolidBlockMinArea = 2048;
 
+static const int LosslessRefreshMaxArea = 4096;
+
 namespace rfb {
 
 enum EncoderClass {
@@ -245,17 +250,42 @@
   }
 }
 
+bool EncodeManager::needsLosslessRefresh(const Region& req)
+{
+  return !lossyRegion.intersect(req).is_empty();
+}
+
+void EncodeManager::pruneLosslessRefresh(const Region& limits)
+{
+  lossyRegion.assign_intersect(limits);
+}
+
 void EncodeManager::writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
                                 const RenderedCursor* renderedCursor)
 {
+    doUpdate(true, ui.changed, ui.copied, ui.copy_delta, pb, renderedCursor);
+}
+
+void EncodeManager::writeLosslessRefresh(const Region& req, const PixelBuffer* pb,
+                                         const RenderedCursor* renderedCursor)
+{
+    doUpdate(false, getLosslessRefresh(req),
+             Region(), Point(), pb, renderedCursor);
+}
+
+void EncodeManager::doUpdate(bool allowLossy, const Region& changed_,
+                             const Region& copied, const Point& copyDelta,
+                             const PixelBuffer* pb,
+                             const RenderedCursor* renderedCursor)
+{
     int nRects;
     Region changed, cursorRegion;
 
     updates++;
 
-    prepareEncoders();
+    prepareEncoders(allowLossy);
 
-    changed = ui.changed;
+    changed = changed_;
 
     /*
      * We need to render the cursor seperately as it has its own
@@ -269,14 +299,14 @@
     if (conn->cp.supportsLastRect)
       nRects = 0xFFFF;
     else {
-      nRects = ui.copied.numRects();
+      nRects = copied.numRects();
       nRects += computeNumRects(changed);
       nRects += computeNumRects(cursorRegion);
     }
 
     conn->writer()->writeFramebufferUpdateStart(nRects);
 
-    writeCopyRects(ui);
+    writeCopyRects(copied, copyDelta);
 
     /*
      * We start by searching for solid rects, which are then removed
@@ -291,7 +321,7 @@
     conn->writer()->writeFramebufferUpdateEnd();
 }
 
-void EncodeManager::prepareEncoders()
+void EncodeManager::prepareEncoders(bool allowLossy)
 {
   enum EncoderClass solid, bitmap, bitmapRLE;
   enum EncoderClass indexed, indexedRLE, fullColour;
@@ -316,7 +346,7 @@
     break;
   case encodingTight:
     if (encoders[encoderTightJPEG]->isSupported() &&
-        (conn->cp.pf().bpp >= 16))
+        (conn->cp.pf().bpp >= 16) && allowLossy)
       fullColour = encoderTightJPEG;
     else
       fullColour = encoderTight;
@@ -334,7 +364,7 @@
 
   if (fullColour == encoderRaw) {
     if (encoders[encoderTightJPEG]->isSupported() &&
-        (conn->cp.pf().bpp >= 16))
+        (conn->cp.pf().bpp >= 16) && allowLossy)
       fullColour = encoderTightJPEG;
     else if (encoders[encoderZRLE]->isSupported())
       fullColour = encoderZRLE;
@@ -374,7 +404,7 @@
 
   // JPEG is the only encoder that can reduce things to grayscale
   if ((conn->cp.subsampling == subsampleGray) &&
-      encoders[encoderTightJPEG]->isSupported()) {
+      encoders[encoderTightJPEG]->isSupported() && allowLossy) {
     solid = bitmap = bitmapRLE = encoderTightJPEG;
     indexed = indexedRLE = fullColour = encoderTightJPEG;
   }
@@ -398,6 +428,48 @@
   }
 }
 
+Region EncodeManager::getLosslessRefresh(const Region& req)
+{
+  std::vector<Rect> rects;
+  Region refresh;
+  size_t area;
+
+  area = 0;
+  lossyRegion.intersect(req).get_rects(&rects);
+  while (!rects.empty()) {
+    size_t idx;
+    Rect rect;
+
+    // Grab a random rect so we don't keep damaging and restoring the
+    // same rect over and over
+    idx = rand() % rects.size();
+
+    rect = rects[idx];
+
+    // Add rects until we exceed the threshold, then include as much as
+    // possible of the final rect
+    if ((area + rect.area()) > LosslessRefreshMaxArea) {
+      // Use the narrowest axis to avoid getting to thin rects
+      if (rect.width() > rect.height()) {
+        int width = (LosslessRefreshMaxArea - area) / rect.height();
+        rect.br.x = rect.tl.x + __rfbmax(1, width);
+      } else {
+        int height = (LosslessRefreshMaxArea - area) / rect.width();
+        rect.br.y = rect.tl.y + __rfbmax(1, height);
+      }
+      refresh.assign_union(Region(rect));
+      break;
+    }
+
+    area += rect.area();
+    refresh.assign_union(Region(rect));
+
+    rects.erase(rects.begin() + idx);
+  }
+
+  return refresh;
+}
+
 int EncodeManager::computeNumRects(const Region& changed)
 {
   int numRects;
@@ -450,6 +522,11 @@
   encoder = encoders[klass];
   conn->writer()->startRect(rect, encoder->encoding);
 
+  if (encoder->flags & EncoderLossy)
+    lossyRegion.assign_union(Region(rect));
+  else
+    lossyRegion.assign_subtract(Region(rect));
+
   return encoder;
 }
 
@@ -466,14 +543,16 @@
   stats[klass][activeType].bytes += length;
 }
 
-void EncodeManager::writeCopyRects(const UpdateInfo& ui)
+void EncodeManager::writeCopyRects(const Region& copied, const Point& delta)
 {
   std::vector<Rect> rects;
   std::vector<Rect>::const_iterator rect;
 
+  Region lossyCopy;
+
   beforeLength = conn->getOutStream()->length();
 
-  ui.copied.get_rects(&rects, ui.copy_delta.x <= 0, ui.copy_delta.y <= 0);
+  copied.get_rects(&rects, delta.x <= 0, delta.y <= 0);
   for (rect = rects.begin(); rect != rects.end(); ++rect) {
     int equiv;
 
@@ -482,11 +561,16 @@
     equiv = 12 + rect->area() * conn->cp.pf().bpp/8;
     copyStats.equivalent += equiv;
 
-    conn->writer()->writeCopyRect(*rect, rect->tl.x - ui.copy_delta.x,
-                                   rect->tl.y - ui.copy_delta.y);
+    conn->writer()->writeCopyRect(*rect, rect->tl.x - delta.x,
+                                   rect->tl.y - delta.y);
   }
 
   copyStats.bytes += conn->getOutStream()->length() - beforeLength;
+
+  lossyCopy = lossyRegion;
+  lossyCopy.translate(delta);
+  lossyCopy.assign_intersect(copied);
+  lossyRegion.assign_union(lossyCopy);
 }
 
 void EncodeManager::writeSolidRects(Region *changed, const PixelBuffer* pb)
diff --git a/common/rfb/EncodeManager.h b/common/rfb/EncodeManager.h
index 4319b42..bdff04b 100644
--- a/common/rfb/EncodeManager.h
+++ b/common/rfb/EncodeManager.h
@@ -1,6 +1,6 @@
 /* Copyright (C) 2000-2003 Constantin Kaplinsky.  All Rights Reserved.
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
- * Copyright 2014 Pierre Ossman for Cendio AB
+ * Copyright 2014-2018 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@
 
 #include <rdr/types.h>
 #include <rfb/PixelBuffer.h>
+#include <rfb/Region.h>
 
 namespace rfb {
   class SConnection;
@@ -31,7 +32,6 @@
   class UpdateInfo;
   class PixelBuffer;
   class RenderedCursor;
-  class Region;
   struct Rect;
 
   struct RectInfo;
@@ -46,18 +46,30 @@
     // Hack to let ConnParams calculate the client's preferred encoding
     static bool supported(int encoding);
 
+    bool needsLosslessRefresh(const Region& req);
+    void pruneLosslessRefresh(const Region& limits);
+
     void writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
                      const RenderedCursor* renderedCursor);
 
+    void writeLosslessRefresh(const Region& req, const PixelBuffer* pb,
+                              const RenderedCursor* renderedCursor);
+
   protected:
-    void prepareEncoders();
+    void doUpdate(bool allowLossy, const Region& changed,
+                  const Region& copied, const Point& copy_delta,
+                  const PixelBuffer* pb,
+                  const RenderedCursor* renderedCursor);
+    void prepareEncoders(bool allowLossy);
+
+    Region getLosslessRefresh(const Region& req);
 
     int computeNumRects(const Region& changed);
 
     Encoder *startRect(const Rect& rect, int type);
     void endRect();
 
-    void writeCopyRects(const UpdateInfo& ui);
+    void writeCopyRects(const Region& copied, const Point& delta);
     void writeSolidRects(Region *changed, const PixelBuffer* pb);
     void findSolidRect(const Rect& rect, Region *changed, const PixelBuffer* pb);
     void writeRects(const Region& changed, const PixelBuffer* pb);
@@ -103,6 +115,8 @@
     std::vector<Encoder*> encoders;
     std::vector<int> activeEncoders;
 
+    Region lossyRegion;
+
     struct EncoderStats {
       unsigned rects;
       unsigned long long bytes;
diff --git a/common/rfb/Encoder.h b/common/rfb/Encoder.h
index a8a447e..66a10d2 100644
--- a/common/rfb/Encoder.h
+++ b/common/rfb/Encoder.h
@@ -35,6 +35,8 @@
     // Give us the raw frame buffer, and not something converted to
     // the what the client is asking for.
     EncoderUseNativePF = 1 << 0,
+    // Encoder does not encode pixels perfectly accurate
+    EncoderLossy = 1 << 1,
   };
 
   class Encoder {
diff --git a/common/rfb/TightJPEGEncoder.cxx b/common/rfb/TightJPEGEncoder.cxx
index 7bb6126..385207f 100644
--- a/common/rfb/TightJPEGEncoder.cxx
+++ b/common/rfb/TightJPEGEncoder.cxx
@@ -64,7 +64,7 @@
 
 
 TightJPEGEncoder::TightJPEGEncoder(SConnection* conn) :
-  Encoder(conn, encodingTight, EncoderUseNativePF, -1),
+  Encoder(conn, encodingTight, (EncoderFlags)(EncoderUseNativePF | EncoderLossy), -1),
   qualityLevel(-1), fineQuality(-1), fineSubsampling(subsampleUndefined)
 {
 }
diff --git a/common/rfb/VNCSConnectionST.cxx b/common/rfb/VNCSConnectionST.cxx
index 0b79dc1..77a6058 100644
--- a/common/rfb/VNCSConnectionST.cxx
+++ b/common/rfb/VNCSConnectionST.cxx
@@ -223,6 +223,9 @@
           }
         }
       }
+
+      // Drop any lossy tracking that is now outside the framebuffer
+      encodeManager.pruneLosslessRefresh(Region(server->pb->getRect()));
     }
     // Just update the whole screen at the moment because we're too lazy to
     // work out what's actually changed.
@@ -1026,7 +1029,8 @@
 
   // Return if there is nothing to send the client.
 
-  if (updates.is_empty() && !writer()->needFakeUpdate())
+  if (updates.is_empty() && !writer()->needFakeUpdate() &&
+      !encodeManager.needsLosslessRefresh(req))
     return;
 
   // The `updates' object could change, make sure we have valid update info.
@@ -1055,12 +1059,16 @@
     damagedCursorRegion.assign_union(ui.changed.intersect(renderedCursorRect));
   }
 
-  if (ui.is_empty() && !writer()->needFakeUpdate())
+  if (ui.is_empty() && !writer()->needFakeUpdate() &&
+      !encodeManager.needsLosslessRefresh(req))
     return;
 
   writeRTTPing();
 
-  encodeManager.writeUpdate(ui, server->getPixelBuffer(), cursor);
+  if (!ui.is_empty())
+    encodeManager.writeUpdate(ui, server->getPixelBuffer(), cursor);
+  else
+    encodeManager.writeLosslessRefresh(req, server->getPixelBuffer(), cursor);
 
   writeRTTPing();